Adding support for Multiple TLS certificate in Tenant

Users can define multiple certificates to be used by MinIO via the TenantSpec.ExternalCertSecret field:

```
  externalCertSecret:
    - name: tls-test-1
      type: kubernetes.io/tls
    - name: tls-test-2
      type: kubernetes.io/tls
    - name: tls-test-3
      type: kubernetes.io/tls
```

Will create the following folder structure inside the MinIO container:

```
certs/
 │
 ├─ public.crt
 ├─ private.key
 ├─ CAs/          // CAs directory is ignored
 │   │
 │    ...
 │
 ├─ example.com/
 │   │
 │   ├─ public.crt
 │   └─ private.key
 └─ foobar.org/
     │
     ├─ public.crt
     └─ private.key
   ...
```

This PR depends on minio/minio#10601

docs/operator-fields.md

@@ -32,7 +32,7 @@ MinIO Operator creates native Kubernetes resources within the cluster. If the Te
 | spec.env | Add MinIO specific environment variables to enable certain features. |
 | spec.requestAutoCert | Enable this to create use your Kubernetes cluster's root Certificate Authority (CA). |
 | spec.certConfig | When `spec.requestAutoCert` is enabled, use this field to pass additional parameters for certificate creation. |
-| spec.externalCertSecret | Set an external secret with private key and certificate to be used to enabled TLS on Tenant pods. Note that only one of `spec.requestAutoCert` or `spec.externalCertSecret` should be enabled at a time. Follow [the document here](https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret) to create the secret to be passed in this section. |
+| spec.externalCertSecret | Set a list of external secrets with private key and certificate to be used to enabled TLS on Tenant pods. Note that only one of `spec.requestAutoCert` or `spec.externalCertSecret` should be enabled at a time. Follow [the document here](https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret) to create the secret to be passed in this section. |
 | spec.resources | Specify CPU and Memory resources for each Tenant container. Refer [this document](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#resource-types) for details. |
 | spec.liveness | Add liveness check for Tenant containers. Refer [this document](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#define-a-liveness-command) for details. |
 | spec.nodeSelector | Add a selector which must be true for the Tenant pod to fit on a node. Refer [this document](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for details.|

docs/tls.md

@@ -34,12 +34,12 @@ Alternatively, it's possible to use a TLS secret. First, create the Kubernetes s
 kubectl create secret tls tls-ssl-minio --key=private.key --cert=public.crt
 ```
 
-Once created, set the name of the Secret (in this example `tls-ssl-minio`) under `spec.externalCertSecret.name`. Also set the type under `spec.externalCertSecret.type` to `kubernetes.io/tls`:
+Once created, set the name of the Secret (in this example `tls-ssl-minio`) under `spec.externalCertSecret[].name`. Also set the type under `spec.externalCertSecret[].type` to `kubernetes.io/tls`:
 
 ```yaml
   externalCertSecret:
-    name: tls-ssl-minio
-    type: kubernetes.io/tls
+    - name: tls-ssl-minio
+      type: kubernetes.io/tls
 ```
 
 ## Using cert-manager
@@ -80,6 +80,6 @@ Finally configure MinIO to use the newly created TLS certificate:
 
 ```yaml
   externalCertSecret:
-    name: tls-minio
-    type: cert-manager.io/v1alpha2
+    - name: tls-minio
+      type: cert-manager.io/v1alpha2
 ```

operator-kustomize/crds/minio.min.io_tenants.yaml

@@ -298,15 +298,18 @@ spec:
                 type: object
               type: array
             externalCertSecret:
-              description: ExternalCertSecret allows a user to specify custom CA certificate, and private key. This is used for enabling TLS support on MinIO Pods.
-              properties:
-                name:
-                  type: string
-                type:
-                  type: string
-              required:
-              - name
-              type: object
+              description: ExternalCertSecret allows a user to specify one or more custom TLS certificates, and private keys. This is used for enabling TLS with SNI support on MinIO Pods.
+              items:
+                description: LocalCertificateReference defines the spec for a local certificate
+                properties:
+                  name:
+                    type: string
+                  type:
+                    type: string
+                required:
+                - name
+                type: object
+              type: array
             externalClientCertSecret:
               description: ExternalClientCertSecret allows a user to specify custom CA client certificate, and private key. This is used for adding client certificates on MinIO Pods --> used for KES authentication.
               properties:

pkg/apis/minio.min.io/v1/types.go

@@ -71,10 +71,10 @@ type TenantSpec struct {
 	// If provided, use these environment variables for Tenant resource
 	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
-	// ExternalCertSecret allows a user to specify custom CA certificate, and private key. This is
-	// used for enabling TLS support on MinIO Pods.
+	// ExternalCertSecret allows a user to specify one or more custom TLS certificates, and private keys. This is
+	// used for enabling TLS with SNI support on MinIO Pods.
 	// +optional
-	ExternalCertSecret *LocalCertificateReference `json:"externalCertSecret,omitempty"`
+	ExternalCertSecret []*LocalCertificateReference `json:"externalCertSecret,omitempty"`
 	// ExternalClientCertSecret allows a user to specify custom CA client certificate, and private key. This is
 	// used for adding client certificates on MinIO Pods --> used for KES authentication.
 	// +optional

pkg/resources/deployments/console-deployment.go

@@ -19,6 +19,7 @@ package deployments
 
 import (
 	"fmt"
+	"strings"
 
 	miniov1 "github.com/minio/operator/pkg/apis/minio.min.io/v1"
 	appsv1 "k8s.io/api/apps/v1"
@@ -35,9 +36,17 @@ func consoleEnvVars(t *miniov1.Tenant) []corev1.EnvVar {
 		},
 	}
 	if t.TLS() {
+		var caCerts []string
+		if t.ExternalCert() {
+			for index := range t.Spec.ExternalCertSecret {
+				caCerts = append(caCerts, fmt.Sprintf("%s/CAs/minio-hostname-%d.crt", miniov1.ConsoleConfigMountPath, index))
+			}
+		} else {
+			caCerts = append(caCerts, fmt.Sprintf("%s/CAs/minio.crt", miniov1.ConsoleConfigMountPath))
+		}
 		envVars = append(envVars, corev1.EnvVar{
 			Name:  "CONSOLE_MINIO_SERVER_TLS_ROOT_CAS",
-			Value: fmt.Sprintf("%s/CAs/minio.crt", miniov1.ConsoleConfigMountPath),
+			Value: strings.Join(caCerts, ","),
 		})
 	}
 	// Add all the environment variables
@@ -129,23 +138,31 @@ func consoleContainer(t *miniov1.Tenant) corev1.Container {
 func NewConsole(t *miniov1.Tenant) *appsv1.Deployment {
 	var certPath = "server.crt"
 	var keyPath = "server.key"
-	var serverCertSecret string
-	var tenantCertSecret string
+
 	var podVolumeSources []corev1.VolumeProjection
+
 	var serverCertPaths = []corev1.KeyToPath{
 		{Key: "public.crt", Path: certPath},
 		{Key: "private.key", Path: keyPath},
 	}
+
 	var tenantCertPath = "CAs/minio.crt"
 	var tenantCertPaths = []corev1.KeyToPath{
 		{Key: "public.crt", Path: tenantCertPath},
 	}
 
 	if t.AutoCert() {
-		serverCertSecret = t.ConsoleTLSSecretName()
-		tenantCertSecret = t.MinIOTLSSecretName()
+		// MinIO tenant certificate generated by AutoCert
+		podVolumeSources = append(podVolumeSources, corev1.VolumeProjection{
+			Secret: &corev1.SecretProjection{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: t.MinIOTLSSecretName(),
+				},
+				Items: tenantCertPaths,
+			},
+		})
 	} else if t.ConsoleExternalCert() {
-		serverCertSecret = t.Spec.Console.ExternalCertSecret.Name
+		serverCertSecret := t.Spec.Console.ExternalCertSecret.Name
 		// This covers both secrets of type "kubernetes.io/tls" and
 		// "cert-manager.io/v1alpha2" because of same keys in both.
 		if t.Spec.Console.ExternalCertSecret.Type == "kubernetes.io/tls" || t.Spec.Console.ExternalCertSecret.Type == "cert-manager.io/v1alpha2" {
@@ -154,21 +171,6 @@ func NewConsole(t *miniov1.Tenant) *appsv1.Deployment {
 				{Key: "tls.key", Path: keyPath},
 			}
 		}
-	}
-
-	// Add MinIO certificate to the CAs pool of Console
-	if t.ExternalCert() {
-		tenantCertSecret = t.Spec.ExternalCertSecret.Name
-		// This covers both secrets of type "kubernetes.io/tls" and
-		// "cert-manager.io/v1alpha2" because of same keys in both.
-		if t.Spec.ExternalCertSecret.Type == "kubernetes.io/tls" || t.Spec.ExternalCertSecret.Type == "cert-manager.io/v1alpha2" {
-			tenantCertPaths = []corev1.KeyToPath{
-				{Key: "tls.crt", Path: tenantCertPath},
-			}
-		}
-	}
-
-	if serverCertSecret != "" {
 		podVolumeSources = append(podVolumeSources, corev1.VolumeProjection{
 			Secret: &corev1.SecretProjection{
 				LocalObjectReference: corev1.LocalObjectReference{
@@ -179,15 +181,38 @@ func NewConsole(t *miniov1.Tenant) *appsv1.Deployment {
 		})
 	}
 
-	if tenantCertSecret != "" {
-		podVolumeSources = append(podVolumeSources, corev1.VolumeProjection{
-			Secret: &corev1.SecretProjection{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: tenantCertSecret,
+	// Add MinIO certificates to the CAs pool of Console
+	if t.ExternalCert() {
+		// Iterate over all provided TLS certificates and store them on the list of
+		// Volumes that will be mounted to the Pod using the following folder structure:
+		//
+		//	certs
+		//		+ CAs
+		//			 + minio-hostname-0.crt
+		//			 + minio-hostname-1.crt
+		//			 + minio-hostname-2.crt
+		//
+		for index, secret := range t.Spec.ExternalCertSecret {
+			// This covers both secrets of type "kubernetes.io/tls" and
+			// "cert-manager.io/v1alpha2" because of same keys in both.
+			if secret.Type == "kubernetes.io/tls" || secret.Type == "cert-manager.io/v1alpha2" {
+				tenantCertPaths = []corev1.KeyToPath{
+					{Key: "tls.crt", Path: fmt.Sprintf("CAs/minio-hostname-%d.crt", index)},
+				}
+			} else {
+				tenantCertPaths = []corev1.KeyToPath{
+					{Key: "public.crt", Path: fmt.Sprintf("CAs/minio-hostname-%d.crt", index)},
+				}
+			}
+			podVolumeSources = append(podVolumeSources, corev1.VolumeProjection{
+				Secret: &corev1.SecretProjection{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: secret.Name,
+					},
+					Items: tenantCertPaths,
 				},
-				Items: tenantCertPaths,
-			},
-		})
+			})
+		}
 	}
 
 	podVolumes := []corev1.Volume{

pkg/resources/jobs/kes-job.go

@@ -41,7 +41,7 @@ func NewForKES(t *miniov1.Tenant) *batchv1.Job {
 		clientCertSecret = t.Spec.ExternalClientCertSecret.Name
 		// This covers both secrets of type "kubernetes.io/tls" and
 		// "cert-manager.io/v1alpha2" because of same keys in both.
-		if t.Spec.ExternalCertSecret.Type == "kubernetes.io/tls" || t.Spec.ExternalCertSecret.Type == "cert-manager.io/v1alpha2" {
+		if t.Spec.ExternalClientCertSecret.Type == "kubernetes.io/tls" || t.Spec.ExternalClientCertSecret.Type == "cert-manager.io/v1alpha2" {
 			clientCertPaths = []corev1.KeyToPath{
 				{Key: "tls.crt", Path: "minio.crt"},
 				{Key: "tls.key", Path: "minio.key"},