From b657280ce208ff13091c870d75194d2481a71bdb Mon Sep 17 00:00:00 2001 From: Lenin Alevski Date: Fri, 20 May 2022 14:26:54 -0700 Subject: [PATCH] Allow user to customize all tenant environment variables - variables defined in `tenant.spec.env` will take precedence over default/hardcoded values - if a variable exists on the config.env configuration file operator will not pass it to the pods via container.spec.env Signed-off-by: Lenin Alevski --- .../kustomization/base/tenant-config.yaml | 15 ++- .../tenant-minio-creds-secret_deprecated.yaml | 8 +- pkg/controller/cluster/main-controller.go | 13 +- pkg/controller/cluster/tenants.go | 35 ++++-- .../statefulsets/minio-statefulset.go | 115 +++++++++++------- 5 files changed, 115 insertions(+), 71 deletions(-) diff --git a/examples/kustomization/base/tenant-config.yaml b/examples/kustomization/base/tenant-config.yaml index 3f1348c78b9..cce8c2339ac 100644 --- a/examples/kustomization/base/tenant-config.yaml +++ b/examples/kustomization/base/tenant-config.yaml @@ -1,13 +1,12 @@ apiVersion: v1 -data: - ## Tenant credentials, base64 encoded (cat config.env | base64) - ## export MINIO_ROOT_USER="minio" - ## export MINIO_ROOT_PASSWORD="minio123" - ## export MINIO_STORAGE_CLASS_STANDARD="EC:2" - ## export MINIO_BROWSER="on" - config.env: ZXhwb3J0IE1JTklPX1JPT1RfVVNFUj0ibWluaW8iCmV4cG9ydCBNSU5JT19ST09UX1BBU1NXT1JEPSJtaW5pbzEyMyIKZXhwb3J0IE1JTklPX1NUT1JBR0VfQ0xBU1NfU1RBTkRBUkQ9IkVDOjIiCmV4cG9ydCBNSU5JT19CUk9XU0VSPSJvbiI= kind: Secret metadata: name: storage-configuration namespace: minio-tenant -type: Opaque \ No newline at end of file +type: Opaque +stringData: + config.env: |- + export MINIO_ROOT_USER="minio" + export MINIO_ROOT_PASSWORD="minio123" + export MINIO_STORAGE_CLASS_STANDARD="EC:2" + export MINIO_BROWSER="on" diff --git a/examples/kustomization/base/tenant-minio-creds-secret_deprecated.yaml b/examples/kustomization/base/tenant-minio-creds-secret_deprecated.yaml index aff66339d06..6fd73907ce4 100644 --- a/examples/kustomization/base/tenant-minio-creds-secret_deprecated.yaml +++ b/examples/kustomization/base/tenant-minio-creds-secret_deprecated.yaml @@ -1,9 +1,9 @@ apiVersion: v1 -data: - accessKey: "" - secretKey: "" kind: Secret metadata: name: storage-creds-secret namespace: minio-tenant -type: Opaque \ No newline at end of file +type: Opaque +stringData: + accessKey: "" + secretKey: "" diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index cad4e3740d0..32c52961465 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -614,10 +614,16 @@ func (c *Controller) syncHandler(key string) error { // Set any required default values and init Global variables nsName := types.NamespacedName{Namespace: namespace, Name: tenantName} + // get combined configurations (tenant.env, tenant.credsSecret and tenant.Configuration) for tenant tenantConfiguration, err := c.getTenantCredentials(ctx, tenant) if err != nil { return err } + // get existing configuration from config.env + skipEnvVars, err := c.getTenantConfiguration(ctx, tenant) + if err != nil { + return err + } // Check if we are decommissioning a pool before we ensure defaults, as that would populate a defaulted pool name tenant, err = c.checkForPoolDecommission(ctx, key, tenant, tenantConfiguration) @@ -829,8 +835,7 @@ func (c *Controller) syncHandler(key string) error { if tenant, err = c.updateTenantStatus(ctx, tenant, StatusProvisioningStatefulSet, 0); err != nil { return err } - - ss = statefulsets.NewPool(tenant, secret, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS()) + ss = statefulsets.NewPool(tenant, secret, skipEnvVars, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS()) ss, err = c.kubeClientSet.AppsV1().StatefulSets(tenant.Namespace).Create(ctx, ss, cOpts) if err != nil { return err @@ -1040,7 +1045,7 @@ func (c *Controller) syncHandler(key string) error { for i, pool := range tenant.Spec.Pools { // Now proceed to make the yaml changes for the tenant statefulset. - ss := statefulsets.NewPool(tenant, secret, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS()) + ss := statefulsets.NewPool(tenant, secret, skipEnvVars, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS()) if _, err = c.kubeClientSet.AppsV1().StatefulSets(tenant.Namespace).Update(ctx, ss, uOpts); err != nil { return err } @@ -1092,7 +1097,7 @@ func (c *Controller) syncHandler(key string) error { carryOverLabels[miniov1.ZoneLabel] = val } - nss := statefulsets.NewPool(tenant, secret, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS()) + nss := statefulsets.NewPool(tenant, secret, skipEnvVars, &pool, &tenant.Status.Pools[i], hlSvc.Name, c.hostsTemplate, c.operatorVersion, isOperatorTLS()) ssCopy := ss.DeepCopy() ssCopy.Spec.Template = nss.Spec.Template diff --git a/pkg/controller/cluster/tenants.go b/pkg/controller/cluster/tenants.go index a3033e5ebf7..524016b982d 100644 --- a/pkg/controller/cluster/tenants.go +++ b/pkg/controller/cluster/tenants.go @@ -24,6 +24,24 @@ import ( miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" ) +func (c *Controller) getTenantConfiguration(ctx context.Context, tenant *miniov2.Tenant) (map[string][]byte, error) { + tenantConfiguration := map[string][]byte{} + // Load tenant configuration from file + if tenant.HasConfigurationSecret() { + minioConfigurationSecretName := tenant.Spec.Configuration.Name + minioConfigurationSecret, err := c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Get(ctx, minioConfigurationSecretName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + configFromFile := miniov2.ParseRawConfiguration(minioConfigurationSecret.Data["config.env"]) + for key, val := range configFromFile { + tenantConfiguration[key] = val + } + } + return tenantConfiguration, nil +} + +// getTenantCredentials returns a combination of env, credsSecret and Configuration tenant credentials func (c *Controller) getTenantCredentials(ctx context.Context, tenant *miniov2.Tenant) (map[string][]byte, error) { // Configuration for tenant can be passed using 3 different sources, tenant.spec.env, k8s credsSecret and config.env secret // If the user provides duplicated configuration the override order will be: @@ -47,16 +65,13 @@ func (c *Controller) getTenantCredentials(ctx context.Context, tenant *miniov2.T } // Load tenant configuration from file - if tenant.HasConfigurationSecret() { - minioConfigurationSecretName := tenant.Spec.Configuration.Name - minioConfigurationSecret, err := c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Get(ctx, minioConfigurationSecretName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - configFromFile := miniov2.ParseRawConfiguration(minioConfigurationSecret.Data["config.env"]) - for key, val := range configFromFile { - tenantConfiguration[key] = val - } + config, err := c.getTenantConfiguration(ctx, tenant) + if err != nil { + return nil, err + } + for key, val := range config { + tenantConfiguration[key] = val } + return tenantConfiguration, nil } diff --git a/pkg/resources/statefulsets/minio-statefulset.go b/pkg/resources/statefulsets/minio-statefulset.go index 8f8dd6410e4..1598f150695 100644 --- a/pkg/resources/statefulsets/minio-statefulset.go +++ b/pkg/resources/statefulsets/minio-statefulset.go @@ -16,6 +16,7 @@ package statefulsets import ( "fmt" + "sort" "strconv" "strings" @@ -63,22 +64,21 @@ func consoleEnvVars(t *miniov2.Tenant) []corev1.EnvVar { // Returns the MinIO environment variables set in configuration. // If a user specifies a secret in the spec (for MinIO credentials) we use // that to set MINIO_ROOT_USER & MINIO_ROOT_PASSWORD. -func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate string, opVersion string) []corev1.EnvVar { +func minioEnvironmentVars(t *miniov2.Tenant, skipEnvVars map[string][]byte, opVersion string) []corev1.EnvVar { var envVars []corev1.EnvVar - // Add all the environment variables - envVars = append(envVars, t.GetEnvVars()...) - // Enable `mc admin update` style updates to MinIO binaries // within the container, only operator is supposed to perform // these operations. - envVars = append(envVars, - corev1.EnvVar{ + envVarsMap := map[string]corev1.EnvVar{ + "MINIO_UPDATE": { Name: "MINIO_UPDATE", Value: "on", - }, corev1.EnvVar{ + }, + "MINIO_UPDATE_MINISIGN_PUBKEY": { Name: "MINIO_UPDATE_MINISIGN_PUBKEY", Value: "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav", - }, corev1.EnvVar{ + }, + miniov2.WebhookMinIOArgs: { Name: miniov2.WebhookMinIOArgs, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ @@ -88,19 +88,22 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate Key: miniov2.WebhookMinIOArgs, }, }, - }, corev1.EnvVar{ + }, + "MINIO_OPERATOR_VERSION": { Name: "MINIO_OPERATOR_VERSION", Value: opVersion, - }, corev1.EnvVar{ + }, + "MINIO_PROMETHEUS_JOB_ID": { Name: "MINIO_PROMETHEUS_JOB_ID", Value: t.PrometheusConfigJobName(), - }) + }, + } var domains []string // Enable Bucket DNS only if asked for by default turned off if t.BucketDNS() { domains = append(domains, t.MinIOBucketBaseDomain()) - envVars = append(envVars, corev1.EnvVar{ + envVarsMap[miniov2.WebhookMinIOBucket] = corev1.EnvVar{ Name: miniov2.WebhookMinIOBucket, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ @@ -110,30 +113,28 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate Key: miniov2.WebhookMinIOArgs, }, }, - }) + } } // Check if any domains are configured if t.HasMinIODomains() { domains = append(domains, t.GetDomainHosts()...) } // tell MinIO about all the domains meant to hit it if they are not passed manually via .spec.env - if !t.HasEnv("MINIO_DOMAIN") && len(domains) > 0 { - envVars = append(envVars, corev1.EnvVar{ + if len(domains) > 0 { + envVarsMap["MINIO_DOMAIN"] = corev1.EnvVar{ Name: "MINIO_DOMAIN", Value: strings.Join(domains, ","), - }) + } } // If no specific server URL is specified we will specify the internal k8s url, but if a list of domains was // provided we will use the first domain. - if !t.HasEnv("MINIO_SERVER_URL") { - serverURL := t.MinIOServerEndpoint() - if t.HasMinIODomains() { - serverURL = t.Spec.Features.Domains.Minio[0] - } - envVars = append(envVars, corev1.EnvVar{ - Name: "MINIO_SERVER_URL", - Value: serverURL, - }) + serverURL := t.MinIOServerEndpoint() + if t.HasMinIODomains() { + serverURL = t.Spec.Features.Domains.Minio[0] + } + envVarsMap["MINIO_SERVER_URL"] = corev1.EnvVar{ + Name: "MINIO_SERVER_URL", + Value: serverURL, } // Set the redirect url for console @@ -146,17 +147,16 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate } consoleDomain = fmt.Sprintf("%s://%s", useSchema, t.Spec.Features.Domains.Console) } - envVars = append(envVars, corev1.EnvVar{ + envVarsMap["MINIO_BROWSER_REDIRECT_URL"] = corev1.EnvVar{ Name: "MINIO_BROWSER_REDIRECT_URL", Value: consoleDomain, - }) + } } - // Add env variables from credentials secret, if no secret provided, dont use - // env vars. MinIO server automatically creates default credentials - if !t.HasConfigurationSecret() && t.HasCredsSecret() { + // add env variables from tenant.Spec.CredsSecret.Name is deprecated and will be removed in the future + if t.HasCredsSecret() { secretName := t.Spec.CredsSecret.Name - envVars = append(envVars, corev1.EnvVar{ + envVarsMap["MINIO_ROOT_USER"] = corev1.EnvVar{ Name: "MINIO_ROOT_USER", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ @@ -166,7 +166,8 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate Key: "accesskey", }, }, - }, corev1.EnvVar{ + } + envVarsMap["MINIO_ROOT_PASSWORD"] = corev1.EnvVar{ Name: "MINIO_ROOT_PASSWORD", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ @@ -176,35 +177,59 @@ func minioEnvironmentVars(t *miniov2.Tenant, wsSecret *v1.Secret, hostsTemplate Key: "secretkey", }, }, - }) + } } if t.HasKESEnabled() { - envVars = append(envVars, corev1.EnvVar{ + envVarsMap["MINIO_KMS_KES_ENDPOINT"] = corev1.EnvVar{ Name: "MINIO_KMS_KES_ENDPOINT", Value: t.KESServiceEndpoint(), - }, corev1.EnvVar{ + } + envVarsMap["MINIO_KMS_KES_CERT_FILE"] = corev1.EnvVar{ Name: "MINIO_KMS_KES_CERT_FILE", Value: miniov2.MinIOCertPath + "/client.crt", - }, corev1.EnvVar{ + } + envVarsMap["MINIO_KMS_KES_KEY_FILE"] = corev1.EnvVar{ Name: "MINIO_KMS_KES_KEY_FILE", Value: miniov2.MinIOCertPath + "/client.key", - }, corev1.EnvVar{ + } + envVarsMap["MINIO_KMS_KES_CA_PATH"] = corev1.EnvVar{ Name: "MINIO_KMS_KES_CA_PATH", Value: miniov2.MinIOCertPath + "/CAs/kes.crt", - }, corev1.EnvVar{ + } + envVarsMap["MINIO_KMS_KES_KEY_NAME"] = corev1.EnvVar{ Name: "MINIO_KMS_KES_KEY_NAME", Value: t.Spec.KES.KeyName, - }) + } } if t.HasConfigurationSecret() { - envVars = append(envVars, corev1.EnvVar{ + envVarsMap["MINIO_CONFIG_ENV_FILE"] = corev1.EnvVar{ Name: "MINIO_CONFIG_ENV_FILE", Value: miniov2.TmpPath + "/minio-config/config.env", - }) + } } + // add console environment variables + for _, env := range consoleEnvVars(t) { + envVarsMap[env.Name] = env + } + // Add all the tenant.spec.env environment variables + // User defined environment variables will take precedence over default environment variables + for _, env := range t.GetEnvVars() { + envVarsMap[env.Name] = env + } + + // transform map to array and skip configurations from config.env + for _, env := range envVarsMap { + if _, ok := skipEnvVars[env.Name]; !ok { + envVars = append(envVars, env) + } + } + // sort the array to produce the same result everytime + sort.Slice(envVars, func(i, j int) bool { + return envVars[i].Name < envVars[j].Name + }) // Return environment variables return envVars } @@ -305,7 +330,7 @@ func volumeMounts(t *miniov2.Tenant, pool *miniov2.Pool, operatorTLS bool, certV } // Builds the MinIO container for a Tenant. -func poolMinioServerContainer(t *miniov2.Tenant, wsSecret *v1.Secret, pool *miniov2.Pool, hostsTemplate string, opVersion string, operatorTLS bool, certVolumeSources []v1.VolumeProjection) v1.Container { +func poolMinioServerContainer(t *miniov2.Tenant, wsSecret *v1.Secret, skipEnvVars map[string][]byte, pool *miniov2.Pool, hostsTemplate string, opVersion string, operatorTLS bool, certVolumeSources []v1.VolumeProjection) v1.Container { consolePort := miniov2.ConsolePort if t.TLS() { consolePort = miniov2.ConsoleTLSPort @@ -340,7 +365,7 @@ func poolMinioServerContainer(t *miniov2.Tenant, wsSecret *v1.Secret, pool *mini ImagePullPolicy: t.Spec.ImagePullPolicy, VolumeMounts: volumeMounts(t, pool, operatorTLS, certVolumeSources), Args: args, - Env: append(minioEnvironmentVars(t, wsSecret, hostsTemplate, opVersion), consoleEnvVars(t)...), + Env: minioEnvironmentVars(t, skipEnvVars, opVersion), Resources: pool.Resources, LivenessProbe: t.Spec.Liveness, ReadinessProbe: t.Spec.Readiness, @@ -401,7 +426,7 @@ func poolSecurityContext(pool *miniov2.Pool, status *miniov2.PoolStatus) *v1.Pod } // NewPool creates a new StatefulSet for the given Cluster. -func NewPool(t *miniov2.Tenant, wsSecret *v1.Secret, pool *miniov2.Pool, poolStatus *miniov2.PoolStatus, serviceName, hostsTemplate, operatorVersion string, operatorTLS bool) *appsv1.StatefulSet { +func NewPool(t *miniov2.Tenant, wsSecret *v1.Secret, skipEnvVars map[string][]byte, pool *miniov2.Pool, poolStatus *miniov2.PoolStatus, serviceName, hostsTemplate, operatorVersion string, operatorTLS bool) *appsv1.StatefulSet { var podVolumes []corev1.Volume replicas := pool.Servers var certVolumeSources []corev1.VolumeProjection @@ -677,7 +702,7 @@ func NewPool(t *miniov2.Tenant, wsSecret *v1.Secret, pool *miniov2.Pool, poolSta } containers := []corev1.Container{ - poolMinioServerContainer(t, wsSecret, pool, hostsTemplate, operatorVersion, operatorTLS, certVolumeSources), + poolMinioServerContainer(t, wsSecret, skipEnvVars, pool, hostsTemplate, operatorVersion, operatorTLS, certVolumeSources), } // attach any sidecar containers and volumes