Allow user to customize all tenant environment variables

examples/kustomization/base/tenant-config.yaml

@@ -1,13 +1,12 @@
 apiVersion: v1
-data:
-  ## Tenant credentials, base64 encoded (cat config.env | base64)
-  ##  export MINIO_ROOT_USER="minio"
-  ##  export MINIO_ROOT_PASSWORD="minio123"
-  ##  export MINIO_STORAGE_CLASS_STANDARD="EC:2"
-  ##  export MINIO_BROWSER="on"
-  config.env: ZXhwb3J0IE1JTklPX1JPT1RfVVNFUj0ibWluaW8iCmV4cG9ydCBNSU5JT19ST09UX1BBU1NXT1JEPSJtaW5pbzEyMyIKZXhwb3J0IE1JTklPX1NUT1JBR0VfQ0xBU1NfU1RBTkRBUkQ9IkVDOjIiCmV4cG9ydCBNSU5JT19CUk9XU0VSPSJvbiI=
 kind: Secret
 metadata:
   name: storage-configuration
   namespace: minio-tenant
-type: Opaque
+type: Opaque
+stringData:
+  config.env: |-
+    export MINIO_ROOT_USER="minio"
+    export MINIO_ROOT_PASSWORD="minio123"
+    export MINIO_STORAGE_CLASS_STANDARD="EC:2"
+    export MINIO_BROWSER="on"

examples/kustomization/base/tenant-minio-creds-secret_deprecated.yaml

@@ -1,9 +1,9 @@
 apiVersion: v1
-data:
-  accessKey: ""
-  secretKey: ""
 kind: Secret
 metadata:
   name: storage-creds-secret
   namespace: minio-tenant
-type: Opaque
+type: Opaque
+stringData:
+  accessKey: ""
+  secretKey: ""

pkg/controller/cluster/main-controller.go

@@ -614,10 +614,16 @@ func (c *Controller) syncHandler(key string) error {
 	// Set any required default values and init Global variables
 	nsName := types.NamespacedName{Namespace: namespace, Name: tenantName}
 
+	// get combined configurations (tenant.env, tenant.credsSecret and tenant.Configuration) for tenant
 	tenantConfiguration, err := c.getTenantCredentials(ctx, tenant)
 	if err != nil {
 		return err
 	}
+	// get existing configuration from config.env
+	skipEnvVars, err := c.getTenantConfiguration(ctx, tenant)
+	if err != nil {
+		return err
+	}
 
 	// Check if we are decommissioning a pool before we ensure defaults, as that would populate a defaulted pool name
 	tenant, err = c.checkForPoolDecommission(ctx, key, tenant, tenantConfiguration)
@@ -829,8 +835,7 @@ func (c *Controller) syncHandler(key string) error {
 			if tenant, err = c.updateTenantStatus(ctx, tenant, StatusProvisioningStatefulSet, 0); err != nil {
 				return err
 			}
-
-			ss = statefulsets.NewPool(tenant, secret, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS())
+			ss = statefulsets.NewPool(tenant, secret, skipEnvVars, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS())
 			ss, err = c.kubeClientSet.AppsV1().StatefulSets(tenant.Namespace).Create(ctx, ss, cOpts)
 			if err != nil {
 				return err
@@ -1040,7 +1045,7 @@ func (c *Controller) syncHandler(key string) error {
 
 		for i, pool := range tenant.Spec.Pools {
 			// Now proceed to make the yaml changes for the tenant statefulset.
-			ss := statefulsets.NewPool(tenant, secret, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS())
+			ss := statefulsets.NewPool(tenant, secret, skipEnvVars, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS())
 			if _, err = c.kubeClientSet.AppsV1().StatefulSets(tenant.Namespace).Update(ctx, ss, uOpts); err != nil {
 				return err
 			}
@@ -1092,7 +1097,7 @@ func (c *Controller) syncHandler(key string) error {
 				carryOverLabels[miniov1.ZoneLabel] = val
 			}
 
-			nss := statefulsets.NewPool(tenant, secret, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS())
+			nss := statefulsets.NewPool(tenant, secret, skipEnvVars, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS())
 			ssCopy := ss.DeepCopy()
 
 			ssCopy.Spec.Template = nss.Spec.Template

pkg/controller/cluster/tenants.go

@@ -24,6 +24,24 @@ import (
 	miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
 )
 
+func (c *Controller) getTenantConfiguration(ctx context.Context, tenant *miniov2.Tenant) (map[string][]byte, error) {
+	tenantConfiguration := map[string][]byte{}
+	// Load tenant configuration from file
+	if tenant.HasConfigurationSecret() {
+		minioConfigurationSecretName := tenant.Spec.Configuration.Name
+		minioConfigurationSecret, err := c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Get(ctx, minioConfigurationSecretName, metav1.GetOptions{})
+		if err != nil {
+			return nil, err
+		}
+		configFromFile := miniov2.ParseRawConfiguration(minioConfigurationSecret.Data["config.env"])
+		for key, val := range configFromFile {
+			tenantConfiguration[key] = val
+		}
+	}
+	return tenantConfiguration, nil
+}
+
+// getTenantCredentials returns a combination of env, credsSecret and Configuration tenant credentials
 func (c *Controller) getTenantCredentials(ctx context.Context, tenant *miniov2.Tenant) (map[string][]byte, error) {
 	// Configuration for tenant can be passed using 3 different sources, tenant.spec.env, k8s credsSecret and config.env secret
 	// If the user provides duplicated configuration the override order will be:
@@ -47,16 +65,13 @@ func (c *Controller) getTenantCredentials(ctx context.Context, tenant *miniov2.T
 	}
 
 	// Load tenant configuration from file
-	if tenant.HasConfigurationSecret() {
-		minioConfigurationSecretName := tenant.Spec.Configuration.Name
-		minioConfigurationSecret, err := c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Get(ctx, minioConfigurationSecretName, metav1.GetOptions{})
-		if err != nil {
-			return nil, err
-		}
-		configFromFile := miniov2.ParseRawConfiguration(minioConfigurationSecret.Data["config.env"])
-		for key, val := range configFromFile {
-			tenantConfiguration[key] = val
-		}
+	config, err := c.getTenantConfiguration(ctx, tenant)
+	if err != nil {
+		return nil, err
+	}
+	for key, val := range config {
+		tenantConfiguration[key] = val
 	}
+
 	return tenantConfiguration, nil
 }

pkg/resources/statefulsets/minio-statefulset.go

@@ -16,6 +16,7 @@ package statefulsets
 
 import (
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -63,22 +64,21 @@ func consoleEnvVars(t *miniov2.Tenant) []corev1.EnvVar {
 // Returns the MinIO environment variables set in configuration.
 // If a user specifies a secret in the spec (for MinIO credentials) we use
 // that to set MINIO_ROOT_USER & MINIO_ROOT_PASSWORD.
-func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate string, opVersion string) []corev1.EnvVar {
+func minioEnvironmentVars(t *miniov2.Tenant, skipEnvVars map[string][]byte, opVersion string) []corev1.EnvVar {
 	var envVars []corev1.EnvVar
-	// Add all the environment variables
-	envVars = append(envVars, t.GetEnvVars()...)
-
 	// Enable `mc admin update` style updates to MinIO binaries
 	// within the container, only operator is supposed to perform
 	// these operations.
-	envVars = append(envVars,
-		corev1.EnvVar{
+	envVarsMap := map[string]corev1.EnvVar{
+		"MINIO_UPDATE": {
 			Name:  "MINIO_UPDATE",
 			Value: "on",
-		}, corev1.EnvVar{
+		},
+		"MINIO_UPDATE_MINISIGN_PUBKEY": {
 			Name:  "MINIO_UPDATE_MINISIGN_PUBKEY",
 			Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav",
-		}, corev1.EnvVar{
+		},
+		miniov2.WebhookMinIOArgs: {
 			Name: miniov2.WebhookMinIOArgs,
 			ValueFrom: &corev1.EnvVarSource{
 				SecretKeyRef: &corev1.SecretKeySelector{
@@ -88,19 +88,22 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate
 					Key: miniov2.WebhookMinIOArgs,
 				},
 			},
-		}, corev1.EnvVar{
+		},
+		"MINIO_OPERATOR_VERSION": {
 			Name:  "MINIO_OPERATOR_VERSION",
 			Value: opVersion,
-		}, corev1.EnvVar{
+		},
+		"MINIO_PROMETHEUS_JOB_ID": {
 			Name:  "MINIO_PROMETHEUS_JOB_ID",
 			Value: t.PrometheusConfigJobName(),
-		})
+		},
+	}
 
 	var domains []string
 	// Enable Bucket DNS only if asked for by default turned off
 	if t.BucketDNS() {
 		domains = append(domains, t.MinIOBucketBaseDomain())
-		envVars = append(envVars, corev1.EnvVar{
+		envVarsMap[miniov2.WebhookMinIOBucket] = corev1.EnvVar{
 			Name: miniov2.WebhookMinIOBucket,
 			ValueFrom: &corev1.EnvVarSource{
 				SecretKeyRef: &corev1.SecretKeySelector{
@@ -110,30 +113,28 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate
 					Key: miniov2.WebhookMinIOArgs,
 				},
 			},
-		})
+		}
 	}
 	// Check if any domains are configured
 	if t.HasMinIODomains() {
 		domains = append(domains, t.GetDomainHosts()...)
 	}
 	// tell MinIO about all the domains meant to hit it if they are not passed manually via .spec.env
-	if !t.HasEnv("MINIO_DOMAIN") && len(domains) > 0 {
-		envVars = append(envVars, corev1.EnvVar{
+	if len(domains) > 0 {
+		envVarsMap["MINIO_DOMAIN"] = corev1.EnvVar{
 			Name:  "MINIO_DOMAIN",
 			Value: strings.Join(domains, ","),
-		})
+		}
 	}
 	// If no specific server URL is specified we will specify the internal k8s url, but if a list of domains was
 	// provided we will use the first domain.
-	if !t.HasEnv("MINIO_SERVER_URL") {
-		serverURL := t.MinIOServerEndpoint()
-		if t.HasMinIODomains() {
-			serverURL = t.Spec.Features.Domains.Minio[0]
-		}
-		envVars = append(envVars, corev1.EnvVar{
-			Name:  "MINIO_SERVER_URL",
-			Value: serverURL,
-		})
+	serverURL := t.MinIOServerEndpoint()
+	if t.HasMinIODomains() {
+		serverURL = t.Spec.Features.Domains.Minio[0]
+	}
+	envVarsMap["MINIO_SERVER_URL"] = corev1.EnvVar{
+		Name:  "MINIO_SERVER_URL",
+		Value: serverURL,
 	}
 
 	// Set the redirect url for console
@@ -146,17 +147,16 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate
 			}
 			consoleDomain = fmt.Sprintf("%s://%s", useSchema, t.Spec.Features.Domains.Console)
 		}
-		envVars = append(envVars, corev1.EnvVar{
+		envVarsMap["MINIO_BROWSER_REDIRECT_URL"] = corev1.EnvVar{
 			Name:  "MINIO_BROWSER_REDIRECT_URL",
 			Value: consoleDomain,
-		})
+		}
 	}
 
-	// Add env variables from credentials secret, if no secret provided, dont use
-	// env vars. MinIO server automatically creates default credentials
-	if !t.HasConfigurationSecret() && t.HasCredsSecret() {
+	// add env variables from tenant.Spec.CredsSecret.Name is deprecated and will be removed in the future
+	if t.HasCredsSecret() {
 		secretName := t.Spec.CredsSecret.Name
-		envVars = append(envVars, corev1.EnvVar{
+		envVarsMap["MINIO_ROOT_USER"] = corev1.EnvVar{
 			Name: "MINIO_ROOT_USER",
 			ValueFrom: &corev1.EnvVarSource{
 				SecretKeyRef: &corev1.SecretKeySelector{
@@ -166,7 +166,8 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate
 					Key: "accesskey",
 				},
 			},
-		}, corev1.EnvVar{
+		}
+		envVarsMap["MINIO_ROOT_PASSWORD"] = corev1.EnvVar{
 			Name: "MINIO_ROOT_PASSWORD",
 			ValueFrom: &corev1.EnvVarSource{
 				SecretKeyRef: &corev1.SecretKeySelector{
@@ -176,35 +177,59 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate
 					Key: "secretkey",
 				},
 			},
-		})
+		}
 	}
 
 	if t.HasKESEnabled() {
-		envVars = append(envVars, corev1.EnvVar{
+		envVarsMap["MINIO_KMS_KES_ENDPOINT"] = corev1.EnvVar{
 			Name:  "MINIO_KMS_KES_ENDPOINT",
 			Value: t.KESServiceEndpoint(),
-		}, corev1.EnvVar{
+		}
+		envVarsMap["MINIO_KMS_KES_CERT_FILE"] = corev1.EnvVar{
 			Name:  "MINIO_KMS_KES_CERT_FILE",
 			Value: miniov2.MinIOCertPath + "/client.crt",
-		}, corev1.EnvVar{
+		}
+		envVarsMap["MINIO_KMS_KES_KEY_FILE"] = corev1.EnvVar{
 			Name:  "MINIO_KMS_KES_KEY_FILE",
 			Value: miniov2.MinIOCertPath + "/client.key",
-		}, corev1.EnvVar{
+		}
+		envVarsMap["MINIO_KMS_KES_CA_PATH"] = corev1.EnvVar{
 			Name:  "MINIO_KMS_KES_CA_PATH",
 			Value: miniov2.MinIOCertPath + "/CAs/kes.crt",
-		}, corev1.EnvVar{
+		}
+		envVarsMap["MINIO_KMS_KES_KEY_NAME"] = corev1.EnvVar{
 			Name:  "MINIO_KMS_KES_KEY_NAME",
 			Value: t.Spec.KES.KeyName,
-		})
+		}
 	}
 
 	if t.HasConfigurationSecret() {
-		envVars = append(envVars, corev1.EnvVar{
+		envVarsMap["MINIO_CONFIG_ENV_FILE"] = corev1.EnvVar{
 			Name:  "MINIO_CONFIG_ENV_FILE",
 			Value: miniov2.TmpPath + "/minio-config/config.env",
-		})
+		}
 	}
 
+	// add console environment variables
+	for _, env := range consoleEnvVars(t) {
+		envVarsMap[env.Name] = env
+	}
+	// Add all the tenant.spec.env environment variables
+	// User defined environment variables will take precedence over default environment variables
+	for _, env := range t.GetEnvVars() {
+		envVarsMap[env.Name] = env
+	}
+
+	// transform map to array and skip configurations from config.env
+	for _, env := range envVarsMap {
+		if _, ok := skipEnvVars[env.Name]; !ok {
+			envVars = append(envVars, env)
+		}
+	}
+	// sort the array to produce the same result everytime
+	sort.Slice(envVars, func(i, j int) bool {
+		return envVars[i].Name < envVars[j].Name
+	})
 	// Return environment variables
 	return envVars
 }
@@ -305,7 +330,7 @@ func volumeMounts(t *miniov2.Tenant, pool *miniov2.Pool, operatorTLS bool, certV
 }
 
 // Builds the MinIO container for a Tenant.
-func poolMinioServerContainer(t *miniov2.Tenant, wsSecret *v1.Secret, pool *miniov2.Pool, hostsTemplate string, opVersion string, operatorTLS bool, certVolumeSources []v1.VolumeProjection) v1.Container {
+func poolMinioServerContainer(t *miniov2.Tenant, wsSecret *v1.Secret, skipEnvVars map[string][]byte, pool *miniov2.Pool, hostsTemplate string, opVersion string, operatorTLS bool, certVolumeSources []v1.VolumeProjection) v1.Container {
 	consolePort := miniov2.ConsolePort
 	if t.TLS() {
 		consolePort = miniov2.ConsoleTLSPort
@@ -340,7 +365,7 @@ func poolMinioServerContainer(t *miniov2.Tenant, wsSecret *v1.Secret, pool *mini
 		ImagePullPolicy: t.Spec.ImagePullPolicy,
 		VolumeMounts:    volumeMounts(t, pool, operatorTLS, certVolumeSources),
 		Args:            args,
-		Env:             append(minioEnvironmentVars(t, wsSecret, hostsTemplate, opVersion), consoleEnvVars(t)...),
+		Env:             minioEnvironmentVars(t, skipEnvVars, opVersion),
 		Resources:       pool.Resources,
 		LivenessProbe:   t.Spec.Liveness,
 		ReadinessProbe:  t.Spec.Readiness,
@@ -401,7 +426,7 @@ func poolSecurityContext(pool *miniov2.Pool, status *miniov2.PoolStatus) *v1.Pod
 }
 
 // NewPool creates a new StatefulSet for the given Cluster.
-func NewPool(t *miniov2.Tenant, wsSecret *v1.Secret, pool *miniov2.Pool, poolStatus *miniov2.PoolStatus, serviceName, hostsTemplate, operatorVersion string, operatorTLS bool) *appsv1.StatefulSet {
+func NewPool(t *miniov2.Tenant, wsSecret *v1.Secret, skipEnvVars map[string][]byte, pool *miniov2.Pool, poolStatus *miniov2.PoolStatus, serviceName, hostsTemplate, operatorVersion string, operatorTLS bool) *appsv1.StatefulSet {
 	var podVolumes []corev1.Volume
 	replicas := pool.Servers
 	var certVolumeSources []corev1.VolumeProjection
@@ -677,7 +702,7 @@ func NewPool(t *miniov2.Tenant, wsSecret *v1.Secret, pool *miniov2.Pool, poolSta
 	}
 
 	containers := []corev1.Container{
-		poolMinioServerContainer(t, wsSecret, pool, hostsTemplate, operatorVersion, operatorTLS, certVolumeSources),
+		poolMinioServerContainer(t, wsSecret, skipEnvVars, pool, hostsTemplate, operatorVersion, operatorTLS, certVolumeSources),
 	}
 
 	// attach any sidecar containers and volumes