From efa9c38d4dc942ee3dca902dcefba2e5bdc9c58f Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Mon, 23 Oct 2023 11:46:20 +0800 Subject: [PATCH 01/10] feat: add support pvc deletion when need --- .../templates/minio.min.io_tenants.yaml | 2 ++ pkg/apis/minio.min.io/v2/types.go | 5 +++++ pkg/controller/main-controller.go | 20 +++++++++++++++++++ .../statefulsets/minio-statefulset.go | 9 ++++++++- resources/base/crds/minio.min.io_tenants.yaml | 2 ++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index 4dec342a7c8..ba2af272300 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -2868,6 +2868,8 @@ spec: servers: format: int32 type: integer + stroageDeletion: + type: boolean tolerations: items: properties: diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 8c6ed8c6ec2..ebc8b636ff1 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -702,6 +702,11 @@ type Pool struct { // If provided, each pod on the Statefulset will run with the specified RuntimeClassName, for more info https://kubernetes.io/docs/concepts/containers/runtime-class/ // +optional RuntimeClassName *string `json:"runtimeClassName,omitempty"` + // *Optional* + + // + // If true. Will delete the stroage when tenant has been deleted. + // +optional + StorageDeletion *bool `json:"stroageDeletion,omitempty"` } // EqualImage returns true if config image and current input image are same diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index 6ca10b6d7ac..9e354ccf3d0 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -752,6 +752,26 @@ func (c *Controller) syncHandler(key string) (Result, error) { // Just output the error. Will not retry. runtime.HandleError(fmt.Errorf("DeletePrometheusAddlConfig '%s/%s' error:%s", namespace, tenantName, err.Error())) } + // try to delete pvc if set StroageDeletionLabel:true + pvcList := corev1.PersistentVolumeClaimList{} + listOpt := client.ListOptions{ + Namespace: namespace, + } + client.MatchingLabels{ + "v1.min.io/tenant": tenantName, + }.ApplyToList(&listOpt) + err := c.k8sClient.List(ctx, &pvcList, &listOpt) + if err != nil { + runtime.HandleError(fmt.Errorf("PersistentVolumeClaimList '%s/%s' error:%s", namespace, tenantName, err.Error())) + } + for _, pvc := range pvcList.Items { + if pvc.Labels[statefulsets.StroageDeletionLabel] == "true" { + err := c.k8sClient.Delete(ctx, &pvc) + if err != nil { + runtime.HandleError(fmt.Errorf("Delete PersistentVolumeClaim '%s/%s/%s' error:%s", namespace, tenantName, pvc.Name, err.Error())) + } + } + } return WrapResult(Result{}, nil) } // will retry after 5sec diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 0856e0fdbac..5a684250f3f 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -33,7 +33,8 @@ import ( ) const ( - bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" + bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" + StroageDeletionLabel = "stroageDeletion" ) // Returns the MinIO environment variables set in configuration. @@ -854,6 +855,12 @@ func NewPool(args *NewPoolArgs) *appsv1.StatefulSet { if pool.VolumeClaimTemplate != nil { pvClaim := *pool.VolumeClaimTemplate + if pool.StorageDeletion != nil && *pool.StorageDeletion { + if len(pvClaim.Labels) == 0 { + pvClaim.Labels = make(map[string]string) + } + pvClaim.Labels[StroageDeletionLabel] = "true" + } name := pvClaim.Name for i := 0; i < int(pool.VolumesPerServer); i++ { pvClaim.Name = name + strconv.Itoa(i) diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index 4dec342a7c8..ba2af272300 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -2868,6 +2868,8 @@ spec: servers: format: int32 type: integer + stroageDeletion: + type: boolean tolerations: items: properties: From a3dd7d68e7aa97a3b6006fcb3b4a2d5aab136f89 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Mon, 23 Oct 2023 11:50:50 +0800 Subject: [PATCH 02/10] add role --- helm/operator/templates/operator-clusterrole.yaml | 1 + resources/base/cluster-role.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/helm/operator/templates/operator-clusterrole.yaml b/helm/operator/templates/operator-clusterrole.yaml index b5a9ee97923..3e58817c15b 100644 --- a/helm/operator/templates/operator-clusterrole.yaml +++ b/helm/operator/templates/operator-clusterrole.yaml @@ -18,6 +18,7 @@ rules: - get - update - list + - delete - apiGroups: - "" resources: diff --git a/resources/base/cluster-role.yaml b/resources/base/cluster-role.yaml index 22a8dcad694..2a4df1dab66 100644 --- a/resources/base/cluster-role.yaml +++ b/resources/base/cluster-role.yaml @@ -18,6 +18,7 @@ rules: - get - update - list + - delete - apiGroups: - "" resources: From 94cba44e2c967bc09a729b432011ca0750e5b258 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Mon, 23 Oct 2023 14:20:00 +0800 Subject: [PATCH 03/10] add comments --- pkg/resources/statefulsets/minio-statefulset.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 5a684250f3f..04eecb6b250 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -33,7 +33,8 @@ import ( ) const ( - bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" + bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" + // StroageDeletionLabel - pvc with this label and the value is `true` means when tenant is being deleted the pvc will be deleted. StroageDeletionLabel = "stroageDeletion" ) From c4f28444b2f729fcbe0cef7b1d07d1ba5a3da700 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Mon, 23 Oct 2023 14:44:47 +0800 Subject: [PATCH 04/10] rename --- helm/operator/templates/minio.min.io_tenants.yaml | 2 +- pkg/apis/minio.min.io/v2/types.go | 2 +- resources/base/crds/minio.min.io_tenants.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index ba2af272300..11589944c84 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -2868,7 +2868,7 @@ spec: servers: format: int32 type: integer - stroageDeletion: + storageDeletion: type: boolean tolerations: items: diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index ebc8b636ff1..d7a59b436d1 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -706,7 +706,7 @@ type Pool struct { // // If true. Will delete the stroage when tenant has been deleted. // +optional - StorageDeletion *bool `json:"stroageDeletion,omitempty"` + StorageDeletion *bool `json:"storageDeletion,omitempty"` } // EqualImage returns true if config image and current input image are same diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index ba2af272300..11589944c84 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -2868,7 +2868,7 @@ spec: servers: format: int32 type: integer - stroageDeletion: + storageDeletion: type: boolean tolerations: items: From 0a53be3eac2e52fb07ec63e2be0d55bc356d3720 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Mon, 23 Oct 2023 15:04:49 +0800 Subject: [PATCH 05/10] rename --- pkg/resources/statefulsets/minio-statefulset.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 04eecb6b250..371eb88d4ec 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -34,8 +34,8 @@ import ( const ( bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" - // StroageDeletionLabel - pvc with this label and the value is `true` means when tenant is being deleted the pvc will be deleted. - StroageDeletionLabel = "stroageDeletion" + // StorageDeletionLabel - pvc with this label and the value is `true` means when tenant is being deleted the pvc will be deleted. + StorageDeletionLabel = "storageDeletion" ) // Returns the MinIO environment variables set in configuration. @@ -860,7 +860,7 @@ func NewPool(args *NewPoolArgs) *appsv1.StatefulSet { if len(pvClaim.Labels) == 0 { pvClaim.Labels = make(map[string]string) } - pvClaim.Labels[StroageDeletionLabel] = "true" + pvClaim.Labels[StorageDeletionLabel] = "true" } name := pvClaim.Name for i := 0; i < int(pool.VolumesPerServer); i++ { From b307ff254677744d411fe00c259924d89926ef35 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Mon, 23 Oct 2023 15:05:00 +0800 Subject: [PATCH 06/10] rename --- pkg/controller/main-controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index 9e354ccf3d0..145c5a94c2d 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -752,7 +752,7 @@ func (c *Controller) syncHandler(key string) (Result, error) { // Just output the error. Will not retry. runtime.HandleError(fmt.Errorf("DeletePrometheusAddlConfig '%s/%s' error:%s", namespace, tenantName, err.Error())) } - // try to delete pvc if set StroageDeletionLabel:true + // try to delete pvc if set StorageDeletionLabel:true pvcList := corev1.PersistentVolumeClaimList{} listOpt := client.ListOptions{ Namespace: namespace, @@ -765,7 +765,7 @@ func (c *Controller) syncHandler(key string) (Result, error) { runtime.HandleError(fmt.Errorf("PersistentVolumeClaimList '%s/%s' error:%s", namespace, tenantName, err.Error())) } for _, pvc := range pvcList.Items { - if pvc.Labels[statefulsets.StroageDeletionLabel] == "true" { + if pvc.Labels[statefulsets.StorageDeletionLabel] == "true" { err := c.k8sClient.Delete(ctx, &pvc) if err != nil { runtime.HandleError(fmt.Errorf("Delete PersistentVolumeClaim '%s/%s/%s' error:%s", namespace, tenantName, pvc.Name, err.Error())) From bf195a814694162b876712c182426931b6590542 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Tue, 24 Oct 2023 09:20:18 +0800 Subject: [PATCH 07/10] Update pkg/apis/minio.min.io/v2/types.go Co-authored-by: Harshavardhana --- pkg/apis/minio.min.io/v2/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index d7a59b436d1..44599566e5e 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -704,7 +704,7 @@ type Pool struct { RuntimeClassName *string `json:"runtimeClassName,omitempty"` // *Optional* + // - // If true. Will delete the stroage when tenant has been deleted. + // If true. Will delete the storage when tenant has been deleted. // +optional StorageDeletion *bool `json:"storageDeletion,omitempty"` } From 7cef4c82de2100a52a91f1dd6e3940a0a47bffb4 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Tue, 24 Oct 2023 09:30:42 +0800 Subject: [PATCH 08/10] docs --- docs/tenant-storage-deletion.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/tenant-storage-deletion.md diff --git a/docs/tenant-storage-deletion.md b/docs/tenant-storage-deletion.md new file mode 100644 index 00000000000..9ba1fd3e08d --- /dev/null +++ b/docs/tenant-storage-deletion.md @@ -0,0 +1,10 @@ +# When one-time storage is required + +There are times when we need to do tests, and once the tests are done, we need to clean up the storage immediately. We can do the following configuration + +```$xslt + - name: pool-0 + storageDeletion: true +``` + +When a tenant is deleted, the associated pvc is also deleted immediately. \ No newline at end of file From 184e4bf726c38e49059f519e61b9cbae198b90d4 Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Tue, 24 Oct 2023 09:33:01 +0800 Subject: [PATCH 09/10] docs --- docs/tenant_crd.adoc | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index 2e1466b210b..407c3707788 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -101,16 +101,6 @@ CertificateStatus keeps track of all the certificates managed by the operator |=== -[id="{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificateconfig"] -==== CustomCertificateConfig - -CustomCertificateConfig (`customCertificateConfig`) provides attributes associated of the TLS certificates manually added to the Operator as part of tenant creation. These fields contain no data if there are no custom TLS certificates. - -.Appears In: -**** -- xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificates[$$CustomCertificates$$] -**** - [id="{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificates"] @@ -454,6 +444,10 @@ Pool (`pools`) defines a MinIO server pool on a Tenant. Each pool consists of a |*Optional* + If provided, each pod on the Statefulset will run with the specified RuntimeClassName, for more info https://kubernetes.io/docs/concepts/containers/runtime-class/ +|*`storageDeletion`* __boolean__ +|*Optional* + + If true. Will delete the storage when tenant has been deleted. + |=== From 34679f27e26a18d7d47daa58b81e3ef45a3b999e Mon Sep 17 00:00:00 2001 From: "guozhi.li" Date: Tue, 24 Oct 2023 14:39:27 +0800 Subject: [PATCH 10/10] rename --- docs/tenant-storage-deletion.md | 2 +- docs/tenant_crd.adoc | 2 +- helm/operator/templates/minio.min.io_tenants.yaml | 4 ++-- pkg/apis/minio.min.io/v2/types.go | 2 +- pkg/controller/main-controller.go | 4 ++-- pkg/resources/statefulsets/minio-statefulset.go | 8 ++++---- resources/base/crds/minio.min.io_tenants.yaml | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/tenant-storage-deletion.md b/docs/tenant-storage-deletion.md index 9ba1fd3e08d..007e3a70d3f 100644 --- a/docs/tenant-storage-deletion.md +++ b/docs/tenant-storage-deletion.md @@ -4,7 +4,7 @@ There are times when we need to do tests, and once the tests are done, we need t ```$xslt - name: pool-0 - storageDeletion: true + reclaimStorage: true ``` When a tenant is deleted, the associated pvc is also deleted immediately. \ No newline at end of file diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index 407c3707788..f44aa17a2f6 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -444,7 +444,7 @@ Pool (`pools`) defines a MinIO server pool on a Tenant. Each pool consists of a |*Optional* + If provided, each pod on the Statefulset will run with the specified RuntimeClassName, for more info https://kubernetes.io/docs/concepts/containers/runtime-class/ -|*`storageDeletion`* __boolean__ +|*`reclaimStorage`* __boolean__ |*Optional* + If true. Will delete the storage when tenant has been deleted. diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index 11589944c84..24331b5b942 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -2768,6 +2768,8 @@ spec: additionalProperties: type: string type: object + reclaimStorage: + type: boolean resources: properties: claims: @@ -2868,8 +2870,6 @@ spec: servers: format: int32 type: integer - storageDeletion: - type: boolean tolerations: items: properties: diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 44599566e5e..ae3202e4025 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -706,7 +706,7 @@ type Pool struct { // // If true. Will delete the storage when tenant has been deleted. // +optional - StorageDeletion *bool `json:"storageDeletion,omitempty"` + ReclaimStorage *bool `json:"reclaimStorage,omitempty"` } // EqualImage returns true if config image and current input image are same diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index 145c5a94c2d..92018247f40 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -752,7 +752,7 @@ func (c *Controller) syncHandler(key string) (Result, error) { // Just output the error. Will not retry. runtime.HandleError(fmt.Errorf("DeletePrometheusAddlConfig '%s/%s' error:%s", namespace, tenantName, err.Error())) } - // try to delete pvc if set StorageDeletionLabel:true + // try to delete pvc if set ReclaimStorageLabel:true pvcList := corev1.PersistentVolumeClaimList{} listOpt := client.ListOptions{ Namespace: namespace, @@ -765,7 +765,7 @@ func (c *Controller) syncHandler(key string) (Result, error) { runtime.HandleError(fmt.Errorf("PersistentVolumeClaimList '%s/%s' error:%s", namespace, tenantName, err.Error())) } for _, pvc := range pvcList.Items { - if pvc.Labels[statefulsets.StorageDeletionLabel] == "true" { + if pvc.Labels[statefulsets.ReclaimStorageLabel] == "true" { err := c.k8sClient.Delete(ctx, &pvc) if err != nil { runtime.HandleError(fmt.Errorf("Delete PersistentVolumeClaim '%s/%s/%s' error:%s", namespace, tenantName, pvc.Name, err.Error())) diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 371eb88d4ec..a563708dbc1 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -34,8 +34,8 @@ import ( const ( bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT" - // StorageDeletionLabel - pvc with this label and the value is `true` means when tenant is being deleted the pvc will be deleted. - StorageDeletionLabel = "storageDeletion" + // ReclaimStorageLabel - pvc with this label and the value is `true` means when tenant is being deleted the pvc will be deleted. + ReclaimStorageLabel = "reclaimStorage" ) // Returns the MinIO environment variables set in configuration. @@ -856,11 +856,11 @@ func NewPool(args *NewPoolArgs) *appsv1.StatefulSet { if pool.VolumeClaimTemplate != nil { pvClaim := *pool.VolumeClaimTemplate - if pool.StorageDeletion != nil && *pool.StorageDeletion { + if pool.ReclaimStorage != nil && *pool.ReclaimStorage { if len(pvClaim.Labels) == 0 { pvClaim.Labels = make(map[string]string) } - pvClaim.Labels[StorageDeletionLabel] = "true" + pvClaim.Labels[ReclaimStorageLabel] = "true" } name := pvClaim.Name for i := 0; i < int(pool.VolumesPerServer); i++ { diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index 11589944c84..24331b5b942 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -2768,6 +2768,8 @@ spec: additionalProperties: type: string type: object + reclaimStorage: + type: boolean resources: properties: claims: @@ -2868,8 +2870,6 @@ spec: servers: format: int32 type: integer - storageDeletion: - type: boolean tolerations: items: properties: