Readiness Endpoint support

UPGRADE.md

@@ -3,6 +3,16 @@ Upgrades
 
 In this document we will try to document relevant upgrade notes for the MinIO Operator.
 
+v6.0.0
+---
+
+This release is focused on a variety of improvements and bug fixes. Mainly reducing the number of times we need to do a
+rolling restart for a MinIO Tenant, for example when the MinIO Operator is upgraded or downgraded.
+
+This release introduces a readiness probe to prevent kubernetes from routing traffic to a MinIO pod that is not ready
+
+> ⚠️ Upgrading to v6.0.0 will cause all pods to restart upon upgrade.
+
 v5.0.0
 ---
 
@@ -40,7 +50,8 @@ kubectl -n $NAMESPACE get svc $TENANT_NAME-prometheus-hl-svc -o yaml > $TENANT_N
 
 After exporting these objects, remove `.metadata.ownerReferences` for all these files.
 
-After upgrading, to have the MinIO Tenant keep using these services, just add the following environment variables to `.spec.env`
+After upgrading, to have the MinIO Tenant keep using these services, just add the following environment variables
+to `.spec.env`
 
 ```yaml
 - name: MINIO_LOG_QUERY_AUTH_TOKEN
@@ -56,7 +67,6 @@ After upgrading, to have the MinIO Tenant keep using these services, just add th
   value: http://<TENANT_NAME>-prometheus-hl-svc:9090
 ```
 
-
 v4.4.5
 ---
 
@@ -101,7 +111,7 @@ securityContext:
 
 This scenario is automatically handled by the operator, however if the tenant is updated from a pre-stored source (i.e:
 a yaml file) which is missing the added `securityContext` this problem may arise again, so update your stored yamls
-respectively. 
+respectively.
 
 v4.2.2 to v4.2.3
 ---
@@ -110,7 +120,8 @@ Before upgrading the `MinIO Operator` you need to make the following changes to
 
 - Update your current `MinIO image` to the latest version in the tenant spec.
 - Make sure every `pool` in `tenant.spec.pools` explicitly set a `securityContext` if not configured already, if this is
-  the first time you are configuring a `securityContext` then your `MinIO` pods are running as root, and you need to use:
+  the first time you are configuring a `securityContext` then your `MinIO` pods are running as root, and you need to
+  use:
 
 ```yaml
       securityContext:
@@ -124,7 +135,7 @@ Before upgrading the `MinIO Operator` you need to make the following changes to
 
 ```yaml
   image: "minio/minio:$(LATEST-VERSION)"
-  ...
+           ...
   pools:
     - servers: 4
       name: "pool-0"
@@ -176,7 +187,7 @@ Before upgrading the `MinIO Operator` you need to make the following changes to
 
 ```yaml
   image: "minio/minio:$(LATEST-VERSION)"
-  ...
+           ...
   zones:
     - servers: 4
       name: "zone-0"
@@ -195,29 +206,30 @@ Before upgrading the `MinIO Operator` you need to make the following changes to
         runAsGroup: 0
         runAsNonRoot: false
         fsGroup: 0
-  - servers: 4
-      name: "zone-1"
-      volumesPerServer: 4
-      volumeClaimTemplate:
-        metadata:
-          name: data
-        spec:
-          accessModes:
-            - ReadWriteOnce
-          resources:
-            requests:
-              storage: 1Ti
-      securityContext:
-        runAsUser: 0
-        runAsGroup: 0
-        runAsNonRoot: false
-        fsGroup: 0
+    - servers: 4
+        name: "zone-1"
+        volumesPerServer: 4
+        volumeClaimTemplate:
+          metadata:
+            name: data
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 1Ti
+        securityContext:
+          runAsUser: 0
+          runAsGroup: 0
+          runAsNonRoot: false
+          fsGroup: 0
 ```
 
 You can make all the changes directly via `kubectl edit tenants $(TENANT-NAME) -n $(NAMESPACE)` or edit your
 `tenant.yaml` and apply the changes: `kubectl apply -f tenant.yaml`.
 
-Failing to apply this changes will cause some issues during the upgrade such as the tenants not able to provision because
+Failing to apply this changes will cause some issues during the upgrade such as the tenants not able to provision
+because
 of wrong `persistent volume claims` (this happens if you don't add the zone name) or MinIO not able to `read/write` on
 existing volumes (this happens if you don't add the right `securityContext`) or they will take too long to start.
 
@@ -236,7 +248,8 @@ existing tenant.
 
 # Upgrade MinIO Operator via Helm Charts
 
-Make sure your current version of the `tenants.minio.min.io` `CRD` includes the necessary `labels` and `annotations` for `Helm`
+Make sure your current version of the `tenants.minio.min.io` `CRD` includes the necessary `labels` and `annotations`
+for `Helm`
 to perform the upgrade:
 
 ```bash

pkg/controller/minio-services.go

@@ -160,6 +160,7 @@ func (c *Controller) checkMinIOHLSvc(ctx context.Context, tenant *miniov2.Tenant
 		hlSvc.ObjectMeta.Annotations = expectedHlSvc.ObjectMeta.Annotations
 		hlSvc.ObjectMeta.Labels = expectedHlSvc.ObjectMeta.Labels
 		hlSvc.Spec.Ports = expectedHlSvc.Spec.Ports
+		hlSvc.Spec.PublishNotReadyAddresses = expectedHlSvc.Spec.PublishNotReadyAddresses
 
 		// update the selector
 		hlSvc.Spec.Selector = expectedHlSvc.Spec.Selector

pkg/controller/upgrades.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/minio/operator/pkg/controller/legacy"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/blang/semver/v4"
 	"github.com/hashicorp/go-version"
@@ -41,8 +42,9 @@ const (
 	version430 = "v4.3.0"
 	version45  = "v4.5"
 	version500 = "v5.0.0"
+	version600 = "v6.0.0"
 	// currentVersion will point to the latest released update version
-	currentVersion = version500
+	currentVersion = version600
 )
 
 // Legacy const
@@ -62,12 +64,13 @@ func (c *Controller) checkForUpgrades(ctx context.Context, tenant *miniov2.Tenan
 		version430: c.upgrade430,
 		version45:  c.upgrade45,
 		version500: c.upgrade500,
+		version600: c.upgrade600,
 	}
 
 	// if tenant has no version we mark it with latest version upgrade released
 	if tenant.Status.SyncVersion == "" {
 		tenant.Status.SyncVersion = currentVersion
-		return c.updateTenantSyncVersion(ctx, tenant, version500)
+		return c.updateTenantSyncVersion(ctx, tenant, version600)
 	}
 
 	// if the version is empty, upgrades might not been applied, we apply them all
@@ -84,6 +87,7 @@ func (c *Controller) checkForUpgrades(ctx context.Context, tenant *miniov2.Tenan
 			version430,
 			version45,
 			version500,
+			version600,
 		}
 		for _, v := range versionsThatNeedUpgrades {
 			vp, _ := version.NewVersion(v)
@@ -414,3 +418,25 @@ func (c *Controller) upgrade500(ctx context.Context, tenant *miniov2.Tenant) (*m
 	}
 	return c.updateTenantSyncVersion(ctx, tenant, version500)
 }
+
+// Upgrades the sync version to v6.0.0
+// since we are adding `publishNotReadyAddresses` to the headless service, we need to restart all pods
+func (c *Controller) upgrade600(ctx context.Context, tenant *miniov2.Tenant) (*miniov2.Tenant, error) {
+	nsName := types.NamespacedName{Namespace: tenant.Namespace, Name: tenant.Name}
+	// Check MinIO Headless Service used for internode communication
+	err := c.checkMinIOHLSvc(ctx, tenant, nsName)
+	if err != nil {
+		klog.V(2).Infof("error consolidating headless service: %s", err.Error())
+		return nil, err
+	}
+	// restart all pods for this tenant
+	listOpts := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("%s=%s", miniov2.TenantLabel, tenant.Name),
+	}
+	err = c.kubeClientSet.CoreV1().Pods(tenant.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts)
+	if err != nil {
+		klog.V(2).Infof("error deleting pods: %s", err.Error())
+		return nil, err
+	}
+	return c.updateTenantSyncVersion(ctx, tenant, version600)
+}

pkg/resources/services/service.go

@@ -58,9 +58,10 @@ func NewClusterIPForMinIO(t *miniov2.Tenant) *corev1.Service {
 			Annotations:     annotations,
 		},
 		Spec: corev1.ServiceSpec{
-			Ports:    []corev1.ServicePort{minioPort},
-			Selector: t.MinIOPodLabels(),
-			Type:     corev1.ServiceTypeClusterIP,
+			Ports:                    []corev1.ServicePort{minioPort},
+			Selector:                 t.MinIOPodLabels(),
+			Type:                     corev1.ServiceTypeClusterIP,
+			PublishNotReadyAddresses: false,
 		},
 	}
 	// check if the service is meant to be exposed
@@ -171,10 +172,11 @@ func NewHeadlessForMinIO(t *miniov2.Tenant) *corev1.Service {
 			OwnerReferences: t.OwnerRef(),
 		},
 		Spec: corev1.ServiceSpec{
-			Ports:     ports,
-			Selector:  t.MinIOPodLabels(),
-			Type:      corev1.ServiceTypeClusterIP,
-			ClusterIP: corev1.ClusterIPNone,
+			Ports:                    ports,
+			Selector:                 t.MinIOPodLabels(),
+			Type:                     corev1.ServiceTypeClusterIP,
+			ClusterIP:                corev1.ClusterIPNone,
+			PublishNotReadyAddresses: true,
 		},
 	}
 

pkg/resources/statefulsets/minio-statefulset.go

@@ -21,6 +21,8 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/util/intstr"
+
 	"github.com/minio/operator/pkg/certs"
 	"github.com/minio/operator/pkg/common"
 
@@ -54,10 +56,6 @@ func minioEnvironmentVars(t *miniov2.Tenant, skipEnvVars map[string][]byte, opVe
 			Name:  "MINIO_UPDATE_MINISIGN_PUBKEY",
 			Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav",
 		},
-		"MINIO_OPERATOR_VERSION": {
-			Name:  "MINIO_OPERATOR_VERSION",
-			Value: opVersion,
-		},
 		"MINIO_PROMETHEUS_JOB_ID": {
 			Name:  "MINIO_PROMETHEUS_JOB_ID",
 			Value: t.PrometheusConfigJobName(),
@@ -909,6 +907,25 @@ func getInitContainer(t *miniov2.Tenant, pool *miniov2.Pool) corev1.Container {
 }
 
 func getSideCarContainer(t *miniov2.Tenant, pool *miniov2.Pool) corev1.Container {
+	scheme := corev1.URISchemeHTTP
+
+	readinessProbe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path: "/ready",
+				Port: intstr.IntOrString{
+					IntVal: 4444,
+				},
+				// Host:        "localhost",
+				Scheme:      scheme,
+				HTTPHeaders: nil,
+			},
+		},
+		InitialDelaySeconds: 5,
+		PeriodSeconds:       1,
+		FailureThreshold:    1,
+	}
+
 	sidecarContainer := corev1.Container{
 		Name:  "sidecar",
 		Image: getSidecarImage(),
@@ -929,6 +946,7 @@ func getSideCarContainer(t *miniov2.Tenant, pool *miniov2.Pool) corev1.Container
 			CfgVolumeMount,
 		},
 		SecurityContext: poolContainerSecurityContext(pool),
+		ReadinessProbe:  readinessProbe,
 	}
 	if t.Spec.SideCars != nil && t.Spec.SideCars.Resources != nil {
 		sidecarContainer.Resources = *t.Spec.SideCars.Resources

sidecar/DEVELOPMENT.md

@@ -0,0 +1,18 @@
+# MinIO Operator Sidecar
+
+This document provides information on how to build and test the sidecar container.
+
+# Testing
+
+Build this project into a container image and run it with the following command:
+
+```shell
+TAG=miniodev/operator-sidecar:sc GOOS=linux  make docker
+```
+
+Patch the MinIO Operator deployment to include the sidecar container via the `OPERATOR_SIDECAR_IMAGE` environment
+variable:
+
+```shell
+kubectl patch deployment minio-operator -n minio-operator --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/1", "value": {"name": "sidecar", "image": "miniodev/operator-sidecar:sc"}}]'
+```

sidecar/pkg/common/generate_config.go

@@ -0,0 +1,54 @@
+// This file is part of MinIO Operator
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package common
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
+	"github.com/minio/operator/pkg/resources/statefulsets"
+)
+
+// AttachGeneratedConfig attaches the generated config to the file contents which will be stored in /tmp/minio/config.env
+func AttachGeneratedConfig(tenant *miniov2.Tenant, fileContents string) string {
+	args, err := GetTenantArgs(tenant)
+	if err != nil {
+		log.Println(err)
+		os.Exit(1)
+	}
+
+	fileContents = fileContents + fmt.Sprintf("export MINIO_ARGS=\"%s\"\n", args)
+
+	return fileContents
+}
+
+// GetTenantArgs returns the arguments for the tenant based on the tenants they have
+func GetTenantArgs(tenant *miniov2.Tenant) (string, error) {
+	if tenant == nil {
+		return "", fmt.Errorf("tenant is nil")
+	}
+	// Validate the MinIO Tenant
+	if err := tenant.Validate(); err != nil {
+		log.Println(err)
+		return "", err
+	}
+	args := strings.Join(statefulsets.GetContainerArgs(tenant, ""), " ")
+	return args, nil
+}