From 3365f8e26827486874f970463318ca7aabdf81f5 Mon Sep 17 00:00:00 2001 From: cesnietor Date: Thu, 6 Jun 2024 11:08:04 -0700 Subject: [PATCH 1/4] Fix Tenant PodInformer for Tenant Status and remove recurrent job --- api/pod-handlers.go | 4 +- api/volumes.go | 10 ++--- operator-integration/tenant_test.go | 5 ++- pkg/controller/main-controller.go | 47 +++++++++++++------- pkg/controller/monitoring.go | 41 ----------------- pkg/controller/pods.go | 46 ++++++------------- pkg/utils/utils.go | 68 +++++++++++++++++++++++++++++ 7 files changed, 124 insertions(+), 97 deletions(-) diff --git a/api/pod-handlers.go b/api/pod-handlers.go index 1626fa99a65..25e87d8b1a2 100644 --- a/api/pod-handlers.go +++ b/api/pod-handlers.go @@ -169,7 +169,7 @@ func getDeletePodResponse(session *models.Principal, params operator_api.DeleteP return ErrorWithContext(ctx, err) } listOpts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("v1.min.io/tenant=%s", params.Tenant), + LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, params.Tenant), FieldSelector: fmt.Sprintf("metadata.name=%s%s", params.Tenant, params.PodName[len(params.Tenant):]), } if err = clientset.CoreV1().Pods(params.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts); err != nil { @@ -531,7 +531,7 @@ func generateTenantLogReport(ctx context.Context, coreInterface v1.CoreV1Interfa return []byte{}, ErrorWithContext(ctx, errors.New("Namespace and Tenant name cannot be empty")) } podListOpts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("v1.min.io/tenant=%s", tenantName), + LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, tenantName), } pods, err := coreInterface.Pods(namespace).List(ctx, podListOpts) if err != nil { diff --git a/api/volumes.go b/api/volumes.go index d6bc62e2ae2..3cdfe7abd14 100644 --- a/api/volumes.go +++ b/api/volumes.go @@ -120,7 +120,7 @@ func getPVCsResponse(session *models.Principal, params operator_api.ListPVCsPara Status: status, StorageClass: *pvc.Spec.StorageClassName, Volume: pvc.Spec.VolumeName, - Tenant: pvc.Labels["v1.min.io/tenant"], + Tenant: pvc.Labels[miniov2.TenantLabel], } ListPVCs = append(ListPVCs, &pvcResponse) } @@ -142,7 +142,7 @@ func getPVCsForTenantResponse(session *models.Principal, params operator_api.Lis // Filter Tenant PVCs. They keep their v1 tenant annotation listOpts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("v1.min.io/tenant=%s", params.Tenant), + LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, params.Tenant), } // List all PVCs @@ -167,7 +167,7 @@ func getPVCsForTenantResponse(session *models.Principal, params operator_api.Lis Status: status, StorageClass: *pvc.Spec.StorageClassName, Volume: pvc.Spec.VolumeName, - Tenant: pvc.Labels["v1.min.io/tenant"], + Tenant: pvc.Labels[miniov2.TenantLabel], } ListPVCs = append(ListPVCs, &pvcResponse) } @@ -188,7 +188,7 @@ func getDeletePVCResponse(session *models.Principal, params operator_api.DeleteP return ErrorWithContext(ctx, err) } listOpts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("v1.min.io/tenant=%s", params.Tenant), + LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, params.Tenant), FieldSelector: fmt.Sprintf("metadata.name=%s", params.PVCName), } if err = clientset.CoreV1().PersistentVolumeClaims(params.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts); err != nil { @@ -237,7 +237,7 @@ func getTenantCSResponse(session *models.Principal, params operator_api.ListTena } // Get CSRs by Label "v1.min.io/tenant=" + params.Tenant - listByTenantLabel := metav1.ListOptions{LabelSelector: "v1.min.io/tenant=" + params.Tenant} + listByTenantLabel := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, params.Tenant)} listResult, listError := clientset.CertificatesV1().CertificateSigningRequests().List(ctx, listByTenantLabel) if listError != nil { return nil, ErrorWithContext(ctx, listError) diff --git a/operator-integration/tenant_test.go b/operator-integration/tenant_test.go index 42895f997a4..2fac994748c 100644 --- a/operator-integration/tenant_test.go +++ b/operator-integration/tenant_test.go @@ -37,6 +37,7 @@ import ( "github.com/go-openapi/loads" "github.com/minio/operator/api/operations" "github.com/minio/operator/models" + miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" "github.com/stretchr/testify/assert" ) @@ -350,7 +351,7 @@ func TestCreateTenant(t *testing.T) { pools := make([]map[string]interface{}, 1) matchExpressions := make([]map[string]interface{}, 2) matchExpressions[0] = map[string]interface{}{ - "key": "v1.min.io/tenant", + "key": miniov2.TenantLabel, "operator": "In", "values": values, } @@ -489,7 +490,7 @@ func TestDeleteTenant(t *testing.T) { pools := make([]map[string]interface{}, 1) matchExpressions := make([]map[string]interface{}, 2) matchExpressions[0] = map[string]interface{}{ - "key": "v1.min.io/tenant", + "key": miniov2.TenantLabel, "operator": "In", "values": values, } diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index afac5743e72..f34bc228034 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -236,7 +236,6 @@ func NewController( ) *Controller { statefulSetInformer := kubeInformerFactory.Apps().V1().StatefulSets() deploymentInformer := kubeInformerFactory.Apps().V1().Deployments() - podInformer := kubeInformerFactory.Core().V1().Pods() serviceInformer := kubeInformerFactory.Core().V1().Services() jobInformer := kubeInformerFactory.Batch().V1().Jobs() secretInformer := kubeInformerFactoryInOperatorNamespace.Core().V1().Secrets() @@ -251,6 +250,22 @@ func NewController( eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeClientSet.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) + // Create PodInformer for Tenant Pods + labelSelector := metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: miniov2.TenantLabel, + Operator: metav1.LabelSelectorOpExists, + }, + }, + } + labelSelectorString, err := utils.LabelSelectorToString(labelSelector) + if err != nil { + klog.Errorf("bad label: %s for podInformer", labelSelectorString) + } + + podInformer := utils.NewPodInformer(kubeClientSet, labelSelectorString) + controller := &Controller{ podName: podName, namespacesToWatch: namespacesToWatch, @@ -260,7 +275,7 @@ func NewController( promClient: promClient, statefulSetLister: statefulSetInformer.Lister(), statefulSetListerSynced: statefulSetInformer.Informer().HasSynced, - podInformer: podInformer.Informer(), + podInformer: podInformer, deploymentLister: deploymentInformer.Lister(), deploymentListerSynced: deploymentInformer.Informer().HasSynced, tenantsSynced: tenantInformer.Informer().HasSynced, @@ -345,14 +360,13 @@ func NewController( DeleteFunc: controller.handleObject, }) - podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.handlePodChange, UpdateFunc: func(old, new interface{}) { - newDepl := new.(*corev1.Pod) - oldDepl := old.(*corev1.Pod) - if newDepl.ResourceVersion == oldDepl.ResourceVersion { - // Periodic resync will send update events for all known Deployments. - // Two different versions of the same Pods will always have different RVs. + newPod := new.(*corev1.Pod) + oldPod := old.(*corev1.Pod) + // Ignore Pod changes if same ResourceVersion + if newPod.ResourceVersion == oldPod.ResourceVersion { return } controller.handlePodChange(new) @@ -379,8 +393,13 @@ func NewController( return controller } +// StartPodInformer runs PodInformer +func (c *Controller) StartPodInformer(stopCh <-chan struct{}) { + c.podInformer.Run(stopCh) +} + // startUpgradeServer Starts the Upgrade tenant API server and notifies the start and stop via notificationChannel returned -func (c *Controller) startUpgradeServer(ctx context.Context) <-chan error { +func (c *Controller) startUpgradeServer() <-chan error { notificationChannel := make(chan error) go func() { defer close(notificationChannel) @@ -429,7 +448,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha var upgradeServerChannel <-chan error klog.Info("Waiting for Upgrade Server to start") - upgradeServerChannel = c.startUpgradeServer(ctx) + upgradeServerChannel = c.startUpgradeServer() // Start the informer factories to begin populating the informer caches klog.Info("Starting Tenant controller") @@ -448,7 +467,6 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha klog.Info("Starting workers and Job workers") JobController := c.controllers[0] - // fmt.Println(controller.SyncHandler()) // Launch two workers to process Job resources for i := 0; i < threadiness; i++ { go wait.Until(JobController.runJobWorker, time.Second, stopCh) @@ -458,8 +476,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha // Launch a single worker for Health Check reacting to Pod Changes go wait.Until(c.runHealthCheckWorker, time.Second, stopCh) - // Launch a goroutine to monitor all Tenants - go c.recurrentTenantStatusMonitor(stopCh) + go c.StartPodInformer(stopCh) // 1) we need to make sure we have console TLS certificates (if enabled) if isOperatorConsoleTLS() { @@ -502,7 +519,7 @@ func leaderRun(ctx context.Context, c *Controller, threadiness int, stopCh <-cha case err := <-upgradeServerChannel: if err != http.ErrServerClosed { klog.Errorf("Upgrade Server stopped: %v, going to restart", err) - upgradeServerChannel = c.startUpgradeServer(ctx) + upgradeServerChannel = c.startUpgradeServer() } // webserver was instructed to stop, do not attempt to restart continue @@ -740,7 +757,7 @@ func (c *Controller) syncHandler(key string) (Result, error) { Namespace: namespace, } client.MatchingLabels{ - "v1.min.io/tenant": tenantName, + miniov2.TenantLabel: tenantName, }.ApplyToList(&listOpt) err := c.k8sClient.List(ctx, &pvcList, &listOpt) if err != nil { diff --git a/pkg/controller/monitoring.go b/pkg/controller/monitoring.go index fcbddc29225..f1d6e499d7e 100644 --- a/pkg/controller/monitoring.go +++ b/pkg/controller/monitoring.go @@ -17,7 +17,6 @@ package controller import ( "context" "fmt" - "log" "time" "github.com/minio/madmin-go/v3" @@ -40,46 +39,6 @@ const ( HealthReduceAvailabilityMessage = "Reduced Availability" ) -// recurrentTenantStatusMonitor loop that checks every 3 minutes for tenants health -func (c *Controller) recurrentTenantStatusMonitor(stopCh <-chan struct{}) { - // do an initial check, then start the periodic check - if err := c.tenantsHealthMonitor(); err != nil { - log.Println(err) - } - // How often will this function run - interval := miniov2.GetMonitoringInterval() - ticker := time.NewTicker(time.Duration(interval) * time.Minute) - defer func() { - log.Println("recurrent pod status monitor closed") - }() - for { - select { - case <-ticker.C: - if err := c.tenantsHealthMonitor(); err != nil { - klog.Infof("%v", err) - } - case <-stopCh: - ticker.Stop() - return - } - } -} - -func (c *Controller) tenantsHealthMonitor() error { - // list all tenants and get their cluster health - tenants, err := c.minioClientSet.MinioV2().Tenants("").List(context.Background(), metav1.ListOptions{}) - if err != nil { - return err - } - for _, tenant := range tenants.Items { - if err = c.updateHealthStatusForTenant(&tenant); err != nil { - klog.Errorf("%v", err) - return err - } - } - return nil -} - func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { // don't get the tenant cluster health if it doesn't have at least 1 pool initialized oneInitialized := false diff --git a/pkg/controller/pods.go b/pkg/controller/pods.go index 3d065e1bead..fdf92d08447 100644 --- a/pkg/controller/pods.go +++ b/pkg/controller/pods.go @@ -17,46 +17,28 @@ package controller import ( - "context" "fmt" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/client-go/tools/cache" - "k8s.io/klog/v2" + "github.com/minio/operator/pkg/utils" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) -// handlePodChange will handle changes in pods and see if they are a MinIO pod, if so, queue it for processing +// handlePodChange will handle changes in pods and queue it for processing, pods are already filtered by PodInformer func (c *Controller) handlePodChange(obj interface{}) { - var object metav1.Object - var ok bool - if object, ok = obj.(metav1.Object); !ok { - tombstone, ok := obj.(cache.DeletedFinalStateUnknown) - if !ok { - runtime.HandleError(fmt.Errorf("error decoding object, invalid type")) - return - } - object, ok = tombstone.Obj.(metav1.Object) - if !ok { - runtime.HandleError(fmt.Errorf("error decoding object tombstone, invalid type")) - return - } - klog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) + // NOTE: currently only Tenant pods are being monitored by the Pod Informer + object, err := utils.CastObjectToMetaV1(obj) + if err != nil { + utilruntime.HandleError(err) + return } - // if the pod is a MinIO pod, we do process it - if tenantName, ok := object.GetLabels()[miniov2.TenantLabel]; ok { - // check this is still a valid tenant - _, err := c.minioClientSet.MinioV2().Tenants(object.GetNamespace()).Get(context.Background(), tenantName, metav1.GetOptions{}) - if err != nil { - klog.V(4).Infof("ignoring orphaned object '%s' of tenant '%s'", object.GetSelfLink(), tenantName) - return - } - key := fmt.Sprintf("%s/%s", object.GetNamespace(), tenantName) - // check if already in queue before re-queueing - c.healthCheckQueue.Add(key) + instanceName, ok := object.GetLabels()[miniov2.TenantLabel] + if !ok { + utilruntime.HandleError(fmt.Errorf("label: %s not found in %s", miniov2.TenantLabel, object.GetName())) return } + + key := fmt.Sprintf("%s/%s", object.GetNamespace(), instanceName) + c.healthCheckQueue.Add(key) } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index c9b96583940..c1abcf4976f 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -17,13 +17,24 @@ package utils import ( + "context" "encoding/base64" + "fmt" "os" "strings" + "time" "github.com/minio/operator/pkg/common" "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" ) // NewUUID - get a random UUID. @@ -71,3 +82,60 @@ func GetOperatorRuntime() common.Runtime { } return runtimeReturn } + +// NewPodInformer creates a Shared Index Pod Informer matching the labelSelector string +func NewPodInformer(kubeClientSet kubernetes.Interface, labelSelectorString string) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return kubeClientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelectorString, + }) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return kubeClientSet.CoreV1().Pods("").Watch(context.Background(), metav1.ListOptions{ + LabelSelector: labelSelectorString, + }) + }, + }, + &corev1.Pod{}, // Resource type + time.Second*30, // Resync period + cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, // Index by namespace + }, + ) +} + +// LabelSelectorToString gets a string from a labelSelector +func LabelSelectorToString(labelSelector metav1.LabelSelector) (string, error) { + var matchExpressions []string + for _, expr := range labelSelector.MatchExpressions { + // Handle only Exists expressions + matchExpressions = append(matchExpressions, expr.Key) + } + // Join match labels and match expressions into a single string with a comma separator. + labelSelectorString := strings.Join(matchExpressions, ",") + // Validate labelSelectorString + if _, err := labels.Parse(labelSelectorString); err != nil { + return "", err + } + return labelSelectorString, nil +} + +// CastObjectToMetaV1 gets a metav1.Object from an interface +func CastObjectToMetaV1(obj interface{}) (metav1.Object, error) { + var object metav1.Object + var ok bool + if object, ok = obj.(metav1.Object); !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + return nil, fmt.Errorf("error decoding object, invalid type") + } + object, ok = tombstone.Obj.(metav1.Object) + if !ok { + return nil, fmt.Errorf("error decoding object tombstone, invalid type") + } + klog.V(4).Infof("Recovered deleted object '%s' from tombstone", object.GetName()) + } + return object, nil +} From 571aa1aad6284690f229c6ecf580d989c4c208e2 Mon Sep 17 00:00:00 2001 From: cesnietor Date: Thu, 6 Jun 2024 12:23:15 -0700 Subject: [PATCH 2/4] remove LabelSelectorString --- pkg/controller/main-controller.go | 14 ++++---------- pkg/utils/utils.go | 17 ----------------- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index f34bc228034..91f7a5e0a1d 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -37,6 +37,7 @@ import ( "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/operator/pkg/controller/certificates" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/labels" "k8s.io/klog/v2" "k8s.io/client-go/tools/leaderelection" @@ -251,17 +252,10 @@ func NewController( recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName}) // Create PodInformer for Tenant Pods - labelSelector := metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: miniov2.TenantLabel, - Operator: metav1.LabelSelectorOpExists, - }, - }, - } - labelSelectorString, err := utils.LabelSelectorToString(labelSelector) - if err != nil { + labelSelectorString := miniov2.TenantLabel // "" -> "Key exists" + if _, err := labels.Parse(labelSelectorString); err != nil { klog.Errorf("bad label: %s for podInformer", labelSelectorString) + labelSelectorString = "" // falback value } podInformer := utils.NewPodInformer(kubeClientSet, labelSelectorString) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index c1abcf4976f..c2e287634d1 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -29,7 +29,6 @@ import ( "github.com/google/uuid" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" @@ -106,22 +105,6 @@ func NewPodInformer(kubeClientSet kubernetes.Interface, labelSelectorString stri ) } -// LabelSelectorToString gets a string from a labelSelector -func LabelSelectorToString(labelSelector metav1.LabelSelector) (string, error) { - var matchExpressions []string - for _, expr := range labelSelector.MatchExpressions { - // Handle only Exists expressions - matchExpressions = append(matchExpressions, expr.Key) - } - // Join match labels and match expressions into a single string with a comma separator. - labelSelectorString := strings.Join(matchExpressions, ",") - // Validate labelSelectorString - if _, err := labels.Parse(labelSelectorString); err != nil { - return "", err - } - return labelSelectorString, nil -} - // CastObjectToMetaV1 gets a metav1.Object from an interface func CastObjectToMetaV1(obj interface{}) (metav1.Object, error) { var object metav1.Object From 3281e38042d22199fbf03f68802d04e2272da680 Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Thu, 6 Jun 2024 14:16:26 -0700 Subject: [PATCH 3/4] requeue tenant health check Signed-off-by: pjuarezd --- pkg/controller/monitoring.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/controller/monitoring.go b/pkg/controller/monitoring.go index f1d6e499d7e..e12237463ca 100644 --- a/pkg/controller/monitoring.go +++ b/pkg/controller/monitoring.go @@ -205,6 +205,11 @@ func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { klog.Infof("'%s/%s' Can't update tenant status with tiers: %v", tenant.Namespace, tenant.Name, err) } } + // Add tenant to the health check queue again until is green again + if tenant.Status.HealthStatus != miniov2.HealthStatusGreen { + key := fmt.Sprintf("%s/%s", tenant.GetNamespace(), tenant.Name) + c.healthCheckQueue.Add(key) + } return nil } From 1f20ff74445e39015ceb666c4e48be9ef5878a95 Mon Sep 17 00:00:00 2001 From: cesnietor Date: Thu, 6 Jun 2024 14:37:41 -0700 Subject: [PATCH 4/4] move requeueing to syncHealthCheckHandler function --- pkg/controller/monitoring.go | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/pkg/controller/monitoring.go b/pkg/controller/monitoring.go index e12237463ca..47ef7db701f 100644 --- a/pkg/controller/monitoring.go +++ b/pkg/controller/monitoring.go @@ -39,7 +39,7 @@ const ( HealthReduceAvailabilityMessage = "Reduced Availability" ) -func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { +func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) (*miniov2.Tenant, error) { // don't get the tenant cluster health if it doesn't have at least 1 pool initialized oneInitialized := false for _, pool := range tenant.Status.Pools { @@ -49,25 +49,25 @@ func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { } if !oneInitialized { klog.Infof("'%s/%s' no pool is initialized", tenant.Namespace, tenant.Name) - return nil + return tenant, nil } tenantConfiguration, err := c.getTenantCredentials(context.Background(), tenant) if err != nil { - return err + return nil, err } adminClnt, err := tenant.NewMinIOAdmin(tenantConfiguration, c.getTransport()) if err != nil { klog.Errorf("Error instantiating adminClnt '%s/%s': %v", tenant.Namespace, tenant.Name, err) - return err + return nil, err } aClnt, err := madmin.NewAnonymousClient(tenant.MinIOServerHostAddress(), tenant.TLS()) if err != nil { // show the error and continue klog.Infof("'%s/%s': %v", tenant.Namespace, tenant.Name, err) - return nil + return tenant, nil } aClnt.SetCustomTransport(c.getTransport()) @@ -79,7 +79,7 @@ func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { if err != nil { // show the error and continue klog.Infof("'%s/%s' Failed to get cluster health: %v", tenant.Namespace, tenant.Name, err) - return nil + return tenant, nil } tenant.Status.DrivesHealing = int32(healthResult.HealingDrives) @@ -98,7 +98,7 @@ func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, tenant.Name), }) if err != nil { - return err + return nil, err } allPodsRunning := true @@ -122,7 +122,7 @@ func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { if err != nil { // show the error and continue klog.Infof("'%s/%s' Failed to get storage info: %v", tenant.Namespace, tenant.Name, err) - return nil + return tenant, nil } // Add back "Usable Capacity" & "Internal" values in Tenant Status and in the UI @@ -205,13 +205,8 @@ func (c *Controller) updateHealthStatusForTenant(tenant *miniov2.Tenant) error { klog.Infof("'%s/%s' Can't update tenant status with tiers: %v", tenant.Namespace, tenant.Name, err) } } - // Add tenant to the health check queue again until is green again - if tenant.Status.HealthStatus != miniov2.HealthStatusGreen { - key := fmt.Sprintf("%s/%s", tenant.GetNamespace(), tenant.Name) - c.healthCheckQueue.Add(key) - } - return nil + return tenant, nil } // HealthResult holds the results from cluster/health query into MinIO @@ -244,10 +239,16 @@ func (c *Controller) syncHealthCheckHandler(key string) (Result, error) { tenant.EnsureDefaults() - if err = c.updateHealthStatusForTenant(tenant); err != nil { + tenant, err = c.updateHealthStatusForTenant(tenant) + if err != nil { klog.Errorf("%v", err) return WrapResult(Result{}, err) } + // Add tenant to the health check queue again until is green again + if tenant != nil && tenant.Status.HealthStatus != miniov2.HealthStatusGreen { + c.healthCheckQueue.Add(key) + } + return WrapResult(Result{}, nil) }