feat: add support pvc deletion when need

docs/tenant-storage-deletion.md

@@ -0,0 +1,10 @@
+# When one-time storage is required
+
+There are times when we need to do tests, and once the tests are done, we need to clean up the storage immediately. We can do the following configuration
+
+```$xslt
+ - name: pool-0
+   reclaimStorage: true
+```
+
+When a tenant is deleted, the associated pvc is also deleted immediately.

docs/tenant_crd.adoc

@@ -101,16 +101,6 @@ CertificateStatus keeps track of all the certificates managed by the operator
 |===
 
 
-[id="{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificateconfig"]
-==== CustomCertificateConfig 
-
-CustomCertificateConfig (`customCertificateConfig`) provides attributes associated of the TLS certificates manually added to the Operator as part of tenant creation. These fields contain no data if there are no custom TLS certificates.
-
-.Appears In:
-****
-- xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificates[$$CustomCertificates$$]
-****
-
 
 
 [id="{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificates"]
@@ -454,6 +444,10 @@ Pool (`pools`) defines a MinIO server pool on a Tenant. Each pool consists of a
 |*Optional* + 
  If provided, each pod on the Statefulset will run with the specified RuntimeClassName, for more info https://kubernetes.io/docs/concepts/containers/runtime-class/
 
+|*`reclaimStorage`* __boolean__ 
+|*Optional* + 
+ If true. Will delete the storage when tenant has been deleted.
+
 |===
 
 

helm/operator/templates/minio.min.io_tenants.yaml

@@ -2768,6 +2768,8 @@ spec:
                       additionalProperties:
                         type: string
                       type: object
+                    reclaimStorage:
+                      type: boolean
                     resources:
                       properties:
                         claims:

helm/operator/templates/operator-clusterrole.yaml

@@ -18,6 +18,7 @@ rules:
       - get
       - update
       - list
+      - delete
   - apiGroups:
       - ""
     resources:

pkg/apis/minio.min.io/v2/types.go

@@ -702,6 +702,11 @@ type Pool struct {
 	// If provided, each pod on the Statefulset will run with the specified RuntimeClassName, for more info https://kubernetes.io/docs/concepts/containers/runtime-class/
 	// +optional
 	RuntimeClassName *string `json:"runtimeClassName,omitempty"`
+	// *Optional* +
+	//
+	// If true. Will delete the storage when tenant has been deleted.
+	// +optional
+	ReclaimStorage *bool `json:"reclaimStorage,omitempty"`
 }
 
 // EqualImage returns true if config image and current input image are same

pkg/controller/main-controller.go

@@ -752,6 +752,26 @@ func (c *Controller) syncHandler(key string) (Result, error) {
 				// Just output the error. Will not retry.
 				runtime.HandleError(fmt.Errorf("DeletePrometheusAddlConfig '%s/%s' error:%s", namespace, tenantName, err.Error()))
 			}
+			// try to delete pvc if set ReclaimStorageLabel:true
+			pvcList := corev1.PersistentVolumeClaimList{}
+			listOpt := client.ListOptions{
+				Namespace: namespace,
+			}
+			client.MatchingLabels{
+				"v1.min.io/tenant": tenantName,
+			}.ApplyToList(&listOpt)
+			err := c.k8sClient.List(ctx, &pvcList, &listOpt)
+			if err != nil {
+				runtime.HandleError(fmt.Errorf("PersistentVolumeClaimList  '%s/%s' error:%s", namespace, tenantName, err.Error()))
+			}
+			for _, pvc := range pvcList.Items {
+				if pvc.Labels[statefulsets.ReclaimStorageLabel] == "true" {
+					err := c.k8sClient.Delete(ctx, &pvc)
+					if err != nil {
+						runtime.HandleError(fmt.Errorf("Delete PersistentVolumeClaim '%s/%s/%s' error:%s", namespace, tenantName, pvc.Name, err.Error()))
+					}
+				}
+			}
 			return WrapResult(Result{}, nil)
 		}
 		// will retry after 5sec

pkg/resources/statefulsets/minio-statefulset.go

@@ -34,6 +34,8 @@ import (
 
 const (
 	bucketDNSEnv = "MINIO_DNS_WEBHOOK_ENDPOINT"
+	// ReclaimStorageLabel - pvc with this label and the value is `true` means when tenant is being deleted the pvc will be deleted.
+	ReclaimStorageLabel = "reclaimStorage"
 )
 
 // Returns the MinIO environment variables set in configuration.
@@ -854,6 +856,12 @@ func NewPool(args *NewPoolArgs) *appsv1.StatefulSet {
 
 	if pool.VolumeClaimTemplate != nil {
 		pvClaim := *pool.VolumeClaimTemplate
+		if pool.ReclaimStorage != nil && *pool.ReclaimStorage {
+			if len(pvClaim.Labels) == 0 {
+				pvClaim.Labels = make(map[string]string)
+			}
+			pvClaim.Labels[ReclaimStorageLabel] = "true"
+		}
 		name := pvClaim.Name
 		for i := 0; i < int(pool.VolumesPerServer); i++ {
 			pvClaim.Name = name + strconv.Itoa(i)

resources/base/cluster-role.yaml

@@ -18,6 +18,7 @@ rules:
       - get
       - update
       - list
+      - delete
   - apiGroups:
       - ""
     resources:

resources/base/crds/minio.min.io_tenants.yaml

@@ -2768,6 +2768,8 @@ spec:
                       additionalProperties:
                         type: string
                       type: object
+                    reclaimStorage:
+                      type: boolean
                     resources:
                       properties:
                         claims: