From 5b71f042c55de607a845a64cf6be40925cf921e7 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Tue, 27 Jun 2023 15:03:17 +0800 Subject: [PATCH] fix:hang for serverUpdate failed (#1659) Co-authored-by: guozhi.li Co-authored-by: Harshavardhana --- .../tenant-certmanager-kes/vault.yaml | 2 +- examples/vault/deployment.yaml | 2 +- pkg/controller/main-controller.go | 77 +++++++++---------- 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/examples/kustomization/tenant-certmanager-kes/vault.yaml b/examples/kustomization/tenant-certmanager-kes/vault.yaml index e906e5a53b5..16c28b70c31 100644 --- a/examples/kustomization/tenant-certmanager-kes/vault.yaml +++ b/examples/kustomization/tenant-certmanager-kes/vault.yaml @@ -30,7 +30,7 @@ spec: spec: containers: - name: vault - image: vault:latest + image: vault:1.13.3 imagePullPolicy: "IfNotPresent" env: - name: SECRET_SHARES diff --git a/examples/vault/deployment.yaml b/examples/vault/deployment.yaml index 7209bed3c4d..7ac946cd433 100644 --- a/examples/vault/deployment.yaml +++ b/examples/vault/deployment.yaml @@ -30,7 +30,7 @@ spec: spec: containers: - name: vault - image: vault:latest + image: vault:1.13.3 imagePullPolicy: "IfNotPresent" env: - name: SECRET_SHARES diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go index eb4554d4f37..a23acbb22e1 100644 --- a/pkg/controller/main-controller.go +++ b/pkg/controller/main-controller.go @@ -26,8 +26,8 @@ import ( "syscall" "time" + "github.com/minio/madmin-go/v2" "github.com/minio/operator/pkg/common" - xcerts "github.com/minio/pkg/certs" "github.com/minio/minio-go/v7/pkg/set" @@ -1090,11 +1090,10 @@ func (c *Controller) syncHandler(key string) error { tenantName, images[0], tenant.Spec.Image) latest, err := c.fetchArtifacts(tenant) + defer c.removeArtifacts() if err != nil { - _ = c.removeArtifacts() return err } - updateURL, err := tenant.UpdateURL(latest, fmt.Sprintf("http://operator.%s.svc.%s:%s%s", miniov2.GetNSFromFile(), miniov2.GetClusterDomain(), @@ -1102,8 +1101,6 @@ func (c *Controller) syncHandler(key string) error { common.WebhookAPIUpdate, )) if err != nil { - _ = c.removeArtifacts() - err = fmt.Errorf("Unable to get canonical update URL for Tenant '%s', failed with %v", tenantName, err) if _, terr := c.updateTenantStatus(ctx, tenant, err.Error(), totalAvailableReplicas); terr != nil { return terr @@ -1118,49 +1115,45 @@ func (c *Controller) syncHandler(key string) error { us, err := adminClnt.ServerUpdate(ctx, updateURL) if err != nil { - _ = c.removeArtifacts() - - err = fmt.Errorf("Tenant '%s' MinIO update failed with %w", tenantName, err) - if _, terr := c.updateTenantStatus(ctx, tenant, err.Error(), totalAvailableReplicas); terr != nil { - return terr - } - - // Update failed, nothing needs to be changed in the container - return err - } - - if us.CurrentVersion != us.UpdatedVersion { - // In case the upgrade is from an older version to RELEASE.2021-07-27T02-40-15Z (which introduced - // MinIO server integrated with Console), we need to delete the old console deployment and service. - // We do this only when MinIO server is successfully updated. - unifiedConsoleReleaseTime, _ := miniov2.ReleaseTagToReleaseTime("RELEASE.2021-07-27T02-40-15Z") - newVer, err := miniov2.ReleaseTagToReleaseTime(us.UpdatedVersion) - if err != nil { - klog.Errorf("Unsupported release tag on new image, server updated but might leave dangling console deployment %v", err) + if madmin.ToErrorResponse(err).Code != "MethodNotAllowed" { + if _, terr := c.updateTenantStatus(ctx, tenant, err.Error(), totalAvailableReplicas); terr != nil { + return terr + } + // Update failed, nothing needs to be changed in the container return err } - consoleDeployment, err := c.deploymentLister.Deployments(tenant.Namespace).Get(tenant.ConsoleDeploymentName()) - if unifiedConsoleReleaseTime.Before(newVer) && consoleDeployment != nil && err == nil { - if err := c.deleteOldConsoleDeployment(ctx, tenant, consoleDeployment.Name); err != nil { + c.RegisterEvent(ctx, tenant, corev1.EventTypeWarning, "Inplace update is disabled, falling back to performing only statefulset update.", fmt.Sprintf("Tenant %s", tenant.Name)) + } + if err == nil { + if us.CurrentVersion != us.UpdatedVersion { + // In case the upgrade is from an older version to RELEASE.2021-07-27T02-40-15Z (which introduced + // MinIO server integrated with Console), we need to delete the old console deployment and service. + // We do this only when MinIO server is successfully updated. + unifiedConsoleReleaseTime, _ := miniov2.ReleaseTagToReleaseTime("RELEASE.2021-07-27T02-40-15Z") + newVer, err := miniov2.ReleaseTagToReleaseTime(us.UpdatedVersion) + if err != nil { + klog.Errorf("Unsupported release tag on new image, server updated but might leave dangling console deployment %v", err) return err } + consoleDeployment, err := c.deploymentLister.Deployments(tenant.Namespace).Get(tenant.ConsoleDeploymentName()) + if unifiedConsoleReleaseTime.Before(newVer) && consoleDeployment != nil && err == nil { + if err := c.deleteOldConsoleDeployment(ctx, tenant, consoleDeployment.Name); err != nil { + return err + } + } + klog.Infof("Tenant '%s' MinIO updated successfully from: %s, to: %s successfully", + tenantName, us.CurrentVersion, us.UpdatedVersion) + } else { + msg := fmt.Sprintf("Tenant '%s' MinIO is already running the most recent version of %s", + tenantName, + us.CurrentVersion) + klog.Info(msg) + if _, terr := c.updateTenantStatus(ctx, tenant, msg, totalAvailableReplicas); terr != nil { + return err + } + return nil } - klog.Infof("Tenant '%s' MinIO updated successfully from: %s, to: %s successfully", - tenantName, us.CurrentVersion, us.UpdatedVersion) - } else { - msg := fmt.Sprintf("Tenant '%s' MinIO is already running the most recent version of %s", - tenantName, - us.CurrentVersion) - klog.Info(msg) - if _, terr := c.updateTenantStatus(ctx, tenant, msg, totalAvailableReplicas); terr != nil { - return err - } - return nil } - - // clean the local directory - _ = c.removeArtifacts() - for i, pool := range tenant.Spec.Pools { // Now proceed to make the yaml changes for the tenant statefulset. ss := statefulsets.NewPool(&statefulsets.NewPoolArgs{