Document TLS Certificates flow with Cert Manager (#2079)

* Document TLS Certificates flow with Cert Manager

Signed-off-by: pjuarezd <pjuarezd@users.noreply.github.com>

docs/env-variables.md

@@ -2,16 +2,17 @@
 
 Operator behavior can be customized using environment variables in the `minio-operator` deployment. Here is a list of the available environment variables:
 
-| Variable Name | Description                                                                                                                                                                  | Possible values         | default                         |
-| --- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|---------------------------------|
-|MINIO_OPERATOR_CERTIFICATES_VERSION| This forces which certificate api version to use.                                                                                                                 | `v1`,`v1beta1`              | whichever api k8s provides      |
-|MINIO_OPERATOR_RUNTIME | This tells us which runtime we have i.e. (EKS, Rancher, OpenShift, etc.). For example, if `EKS` is set, the CSR signerName to be used will be `"beta.eks.amazonaws.com/app-serving"` | EKS, Rancher, OpenShift |                                 |
-|MINIO_OPERATOR_CSR_SIGNER_NAME| The name to use for the CSR Signer. It will override the default                                                                                                             |                         | `kubernetes.io/kubelet-serving` |
-|SUBNET_BASE_URL| Subnet base URL                                                                                                                                                              |                         | https://subnet.min.io           |
-|OPERATOR_CERT_PASSWD| This is used to decrypt the private key in the TLS certificate for operator, if needed                                                                               |                         |                                 |
-|MINIO_OPERATOR_DEPLOYMENT_NAME| This specifies a custom deployment name for Operator                                                                                                                |                         | `minio-operator`                |
-|OPERATOR_STS_ENABLED| This toggles the STS Service on or off                                                                                                                         | `on`, `off`                 | `on`                            |
-|MINIO_CONSOLE_DEPLOYMENT_NAME| This is the default name of the console deployment                                                                                                                                |                  | `console`                       |
-|MINIO_CONSOLE_TLS_ENABLE| This toggles the Console TLS on or off                                                                                                                                   | `on`, `off`                 | `off`                           |                                                                             |                         |                               |
-|WATCHED_NAMESPACE| The namespaces which the operator watches for MinIO tenants. Defaults to `""` for all namespaces.                                                                              |                         |                                 |
-|MINIO_OPERATOR_IMAGE| This variable controls the image of the minio instance's sidecar and validate-arguments. if not set, the mirrors of the minio instance's sidecar and validate-arguments will use the operator's image.  | "" | "" |
+| Variable Name | Description                                                                                                                                                                                            | Possible values         | default                         |
+| --- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|---------------------------------|
+|MINIO_OPERATOR_CERTIFICATES_VERSION| This forces which certificate api version to use.                                                                                                                                                      | `v1`,`v1beta1`              | whichever api k8s provides      |
+|MINIO_OPERATOR_RUNTIME | This tells us which runtime we have i.e. (EKS, Rancher, OpenShift, etc.). For example, if `EKS` is set, the CSR signerName to be used will be `"beta.eks.amazonaws.com/app-serving"`                   | EKS, Rancher, OpenShift |                                 |
+|MINIO_OPERATOR_CSR_SIGNER_NAME| The name to use for the CSR Signer. It will override the default                                                                                                                                       |                         | `kubernetes.io/kubelet-serving` |
+|SUBNET_BASE_URL| Subnet base URL                                                                                                                                                                                        |                         | https://subnet.min.io           |
+|OPERATOR_CERT_PASSWD| This is used to decrypt the private key in the TLS certificate for operator, if needed                                                                                                                 |                         |                                 |
+|MINIO_OPERATOR_DEPLOYMENT_NAME| This specifies a custom deployment name for Operator                                                                                                                                                   |                         | `minio-operator`                |
+|OPERATOR_STS_ENABLED| This toggles the STS Service on or off                                                                                                                                                                 | `on`, `off`                 | `on`                            |
+|OPERATOR_STS_AUTO_TLS_ENABLED| Env variable name to turn on and off generating the STS TLS certificate automatically using CSR. If it is disabled, you must provide a certificate issued externally                                                    | `on`, `off`                 | `on`                            |
+|MINIO_CONSOLE_DEPLOYMENT_NAME| This is the default name of the console deployment                                                                                                                                                     |                  | `console`                       |
+|MINIO_CONSOLE_TLS_ENABLE| This toggles the Console TLS on or off                                                                                                                                                                 | `on`, `off`                 | `off`                           |                                                                             |                         |                               |
+|WATCHED_NAMESPACE| The namespaces which the operator watches for MinIO tenants. Defaults to `""` for all namespaces.                                                                                                      |                         |                                 |
+|MINIO_OPERATOR_IMAGE| This variable controls the image of the minio instance's sidecar and validate-arguments. If not set, the mirrors of the minio instance's sidecar and validate-arguments use the operator's image. | "" | "" |

docs/tls.md

@@ -4,23 +4,23 @@ This document explains how to enable TLS on MinIO pods.
 
 ## Automatic TLS
 
-This approach creates TLS certificates automatically using the Kubernetes cluster root Certificate Authority (CA) to establish trust. In this approach, MinIO Operator creates a private key, and a certificate signing request (CSR) which is submitted via the `certificates.k8s.io` API for signing. Automatic TLS approach creates other certificates required for KES as well as explained in [KES document](./kes.md).
+This approach creates TLS certificates automatically using the Kubernetes cluster root Certificate Authority (CA) to establish trust. In this approach, MinIO Operator creates a private key and a certificate signing request (CSR) and submits them via the `certificates.k8s.io` API for signing. Automatic TLS approach creates other certificates required for KES as well as explained in [KES document](./kes.md).
 
 To enable automatic CSR generation on Tenant, set `requestAutoCert` field in the config file to `true`. Optionally you can also pass additional configuration parameters to be used under `certConfig` section. The `certConfig` section currently supports below fields:
 
 - commonName: By default this is set to a wild card domain name as per [Kubernetes StatefulSet Pod Identity](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity). Set it any other value as per your requirements.
 
 - organizationName: By default set to list `["system:nodes"]`. Change it to a list with the name of your organization, e.g., `organizationName: [my-org]`.
 
-- dnsNames: By default set to list of all pod DNS names that are part of current Tenant. Any value added under this section will be appended to the list of existing pod DNS names.
+- dnsNames: By default set to a list of all pod DNS names that are part of current Tenant. Any value added under this section will be appended to the list of existing pod DNS names.
 
-Once you enable `requestAutoCert` field and create the Tenant, MinIO Operator creates a CSR for this instance and sends to the Kubernetes API server. MinIO Operator will then approve the CSR. After the CSR is approved and Certificate available, MinIO operator downloads the certificate and then mounts the Private Key and Certificate within the Tenant pod.
+Once you enable the `requestAutoCert` field and create the Tenant, MinIO Operator creates a CSR for this instance and sends to the Kubernetes API server. MinIO Operator will then approve the CSR. After the CSR is approved and Certificate available, MinIO operator downloads the certificate and then mounts the Private Key and Certificate within the Tenant pod.
 
 ---
 
 ## Pass Certificate Secret to Tenant
 
-This approach involves acquiring a CA signed or self-signed certificate and use a Kubernetes Secret resource to store this information. Once you have the key and certificate file available, create a Kubernetes Secret using
+This approach involves acquiring a CA signed or self-signed certificate and using a Kubernetes Secret resource to store this information. Once you have the key and certificate file available, create a Kubernetes Secret with:
 
 ```bash
 kubectl create secret generic tls-ssl-minio --from-file=path/to/private.key --from-file=path/to/public.crt
@@ -48,49 +48,5 @@ Once created, set the name of the Secret (in this example `tls-ssl-minio`) under
 
 ## Using cert-manager
 
-[Certificate Manager](https://cert-manager.io) is a Kubernetes Operator capable of automatically issuing certificates from multiple Issuers. Integration with MinIO is simple. First, create a new certificate issuer; for this demonstration the issuer certificate will be self-signed:
-
-```yaml
-apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: selfsigning-issuer
-spec:
-  selfSigned: {}
-```
-
-Now it's possible to issue the MinIO certificate using the above issuer:
-
-```yaml
----
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: tls-minio
-spec:
-  commonName: minio.example.com
-  secretName: tls-minio
-  dnsNames:
-    # Ingress domain
-    - minio.example.com
-    # Internal domain
-    - minio-hl-svc
-    - minio-hl-svc.default.svc.cluster.local
-  issuerRef:
-    name: selfsigning-issuer
-```
-
-Finally, configure MinIO to use the newly created TLS certificate:
-
-```yaml
-  externalCertSecret:
-    - name: tls-minio
-      type: kubernetes.io/tls
-```
-
-## Using your own CA Certificate for MinIO Operator
-
-If your `MinIO` tenants are using `custom certificates` or certificates generated by your own internal `certificate authority` (ie: `cert-manager`).
-`MinIO Operator` needs to trust the `TLS` connections in order to talk to the `MinIO tenants`, for that you need to create a new secret in the
-`minio-operator` namespace named `operator-ca-tls`, inside this secret create a new key `ca.crt` that will include the public certificate
-for your internal certificate authority.
+[Certificate Manager](https://cert-manager.io) is a Kubernetes Operator capable of automatically issuing certificates from multiple Issuers. 
+For instructions on using Cert Manager with MinIO please follow the guide in the [cert-manager.md](cert-manager.md) document.

examples/kustomization/cert-manager/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - selfsigned-root-clusterissuer.yaml

examples/kustomization/cert-manager/selfsigned-root-clusterissuer.yaml

@@ -0,0 +1,6 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: selfsigned-root
+spec:
+  selfSigned: {}

examples/kustomization/operator-certmanager/console-tls-certificate.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: console-certmanager-cert
+  namespace: minio-operator
+spec:
+  dnsNames:
+    - console
+    - console.minio-operator.svc
+    - console.minio-operator.svc.cluster.local
+  secretName: console-tls
+  issuerRef:
+    name: minio-operator-ca-issuer

examples/kustomization/operator-certmanager/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../cert-manager
+  - ../../../resources
+  - operator-ca-tls-secret.yaml
+  - sts-tls-certificate.yaml
+  - console-tls-certificate.yaml
+  - minio-operator-ca-issuer.yaml
+
+patches:
+  - path: operator-deployment.yaml

examples/kustomization/operator-certmanager/minio-operator-ca-issuer.yaml

@@ -0,0 +1,8 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: minio-operator-ca-issuer
+  namespace: minio-operator
+spec:
+  ca:
+    secretName: operator-ca-tls

examples/kustomization/operator-certmanager/operator-ca-tls-secret.yaml

@@ -0,0 +1,17 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: minio-operator-ca-certificate
+  namespace: minio-operator
+spec:
+  isCA: true
+  commonName: operator
+  secretName: operator-ca-tls
+  duration: 70128h # 8y
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-root
+    kind: ClusterIssuer
+    group: cert-manager.io

examples/kustomization/operator-certmanager/operator-deployment.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio-operator
+  namespace: minio-operator
+spec:
+  template:
+    spec:
+      containers:
+        - name: minio-operator
+          env:
+            - name: MINIO_CONSOLE_TLS_ENABLE
+              value: "off"
+            - name: OPERATOR_STS_AUTO_TLS_ENABLED
+              value: "off"
+            - name: OPERATOR_STS_ENABLED
+              value: "on"

examples/kustomization/operator-certmanager/sts-tls-certificate.yaml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: sts-certmanager-cert
+  namespace: minio-operator
+spec:
+  dnsNames:
+    - sts
+    - sts.minio-operator.svc
+    - sts.minio-operator.svc.cluster.local
+  secretName: sts-tls
+  issuerRef:
+    name: minio-operator-ca-issuer

examples/kustomization/sts-example/sample-data/mc-job-setup-bucket.yaml

@@ -32,7 +32,7 @@ metadata:
   name: setup-bucket
   namespace: minio-tenant-1
 spec:
-  backoffLimit: 5
+  backoffLimit: 10
   template:
     spec:
       serviceAccountName: mc-job-sa

examples/kustomization/tenant-certmanager-kes/kustomization.yaml

@@ -4,7 +4,8 @@ kind: Kustomization
 namespace: tenant-certmanager-kes
 
 resources:
-  - certificates.yaml
+  - ../cert-manager
+  - myminio-kes-certificate.yaml
   - vault.yaml
   - ../tenant-kes-encryption
 

examples/kustomization/tenant-certmanager-kes/myminio-kes-certificate.yaml

@@ -0,0 +1,15 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: tenant-certmanager-2-cert
+  namespace: tenant-certmanager
+spec:
+  dnsNames:
+    - "minio.tenant-certmanager"
+    - "minio.tenant-certmanager.svc"
+    - "*.tenant-certmanager.svc.cluster.local"
+    - "*.myminio.tenant-certmanager.svc.cluster.local"
+    - "*.myminio-hl.tenant-certmanager.svc.cluster.local"
+  secretName: tenant-certmanager-2-tls
+  issuerRef:
+    name: tenant-certmanager-issuer

examples/kustomization/tenant-certmanager/kustomization.yaml

@@ -1,11 +1,12 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 namespace: tenant-certmanager
 
 resources:
-  - certificates.yaml
+  - tenant-certmanager-issuer.yaml
+  - tenant-certmanager-ca-certificate.yaml
+  - tenant-certmanager-myminio-certificate.yaml
   - ../base
 
-patchesStrategicMerge:
-  - tenant.yaml
+patches:
+  - path: tenant.yaml

examples/kustomization/tenant-certmanager/tenant-certmanager-ca-certificate.yaml

@@ -0,0 +1,17 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: tenant-certmanager-ca-certificate
+  namespace: tenant-certmanager
+spec:
+  isCA: true
+  commonName: tenant-certmanager-ca
+  secretName: tenant-certmanager-ca-tls
+  duration: 70128h # 8y
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-root
+    kind: ClusterIssuer
+    group: cert-manager.io

examples/kustomization/tenant-certmanager/tenant-certmanager-issuer.yaml

@@ -0,0 +1,7 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: tenant-certmanager-issuer
+  namespace: tenant-certmanager
+spec:
+  selfSigned: { }

examples/kustomization/tenant-certmanager/certificates.yaml → examples/kustomization/tenant-certmanager/tenant-certmanager-myminio-certificate.yaml

@@ -1,16 +1,8 @@
 apiVersion: cert-manager.io/v1
-kind: Issuer
-metadata:
-  name: tenant-certmanager-issuer
-  namespace: minio-tenant
-spec:
-  selfSigned: { }
----
-apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: tenant-certmanager-cert
-  namespace: minio-tenant
+  namespace: tenant-certmanager
 spec:
   # If you see "Empty issuer DN not allowed in X509Certificates"
   # You will need to add spec.subject.organizations or spec.commonName
@@ -23,9 +15,12 @@ spec:
   #     - jetstack
   # commonName: example.com
   dnsNames:
+    - "minio.tenant-certmanager"
+    - "minio.tenant-certmanager.svc"
+    - "minio.tenant-certmanager.svc.cluster.local"
+    - "*.myminio-hl.tenant-certmanager.svc.cluster.local"
     - "*.tenant-certmanager.svc.cluster.local"
     - "*.myminio.tenant-certmanager.svc.cluster.local"
-    - "*.myminio-hl.tenant-certmanager.svc.cluster.local"
-  secretName: tenant-certmanager-tls
+  secretName: myminio-tls
   issuerRef:
-    name: tenant-certmanager-issuer
+    name: tenant-certmanager-issuer

examples/kustomization/tenant-certmanager/tenant.yaml

@@ -8,5 +8,5 @@ spec:
   requestAutoCert: false
   ## Use certificates generated by cert-manager.
   externalCertSecret:
-    - name: tenant-certmanager-tls
+    - name: myminio-tls
       type: cert-manager.io/v1