Add support for GCP fleet workload identity in KES (#1501)

This allows KES to use the workload identity to communicate with GCP secret store instead of managing and rotating the GCP service account. This PR provides the support by mounting the default app credentials to the KES pods via k8s secret. For more details, please refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity.
minio · Mar 26, 2023 · c8df636 · c8df636
1 parent 511cfc6
commit c8df636
Show file tree

Hide file tree

Showing 16 changed files with 322 additions and 8 deletions.
diff --git a/examples/kustomization/tenant-kes-encryption-gcp/gcp-default-creds-secret.yaml b/examples/kustomization/tenant-kes-encryption-gcp/gcp-default-creds-secret.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcp-default-creds
+  namespace: tenant-kms-encrypted
+type: Opaque
+stringData:
+  # NOTE: refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity#impersonate_a_service_account for the process to extract application default credentials
+  # Please replace <WORKLOAD_IDENTITY_POOL>,<IDENTITY_PROVIDER>,<GSA_NAME> and <GSA_PROJECT_ID> with the respective values from application default credentials.
+  config: |
+    {
+      "type": "external_account",
+      "audience": "identitynamespace:<WORKLOAD_IDENTITY_POOL>:<IDENTITY_PROVIDER>",
+      "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/<GSA_NAME>@<GSA_PROJECT_ID>.iam.gserviceaccount.com:generateAccessToken",
+      "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+      "token_url": "https://sts.googleapis.com/v1/token",
+      "credential_source": {
+        "file": "/var/run/secrets/tokens/gcp-ksa/token"
+      }
+    }
diff --git a/examples/kustomization/tenant-kes-encryption-gcp/kes-configuration-secret.yaml b/examples/kustomization/tenant-kes-encryption-gcp/kes-configuration-secret.yaml
@@ -0,0 +1,67 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kes-configuration
+  namespace: tenant-kms-encrypted
+type: Opaque
+stringData:
+  server-config.yaml: |-
+    version: v1
+    address: :7373
+    admin:
+      identity: _ # Effectively disabled since no root identity necessary.
+    tls:
+      key: /tmp/kes/server.key   # Path to the TLS private key
+      cert: /tmp/kes/server.crt # Path to the TLS certificate
+      proxy:
+        identities: []
+        header:
+          cert: X-Tls-Client-Cert
+    policy:
+      my-policy:
+        allow:
+        - /v1/api
+        - /v1/key/create/*
+        - /v1/key/import/*
+        - /v1/key/delete/*
+        - /v1/key/list/*
+        - /v1/key/generate/*
+        - /v1/key/decrypt/*
+        - /v1/key/encrypt/*
+        - /v1/key/bulk/decrypt/*
+        - /v1/status
+        - /v1/api
+        - /v1/metrics
+        - /v1/log/audit
+        - /v1/log/error
+        identities:
+        - ${MINIO_KES_IDENTITY}
+    cache:
+      expiry:
+        any: 5m0s
+        unused: 20s
+    log:
+      error: on
+      audit: on
+    keystore:
+      gcp:
+        secretmanager:
+          # The project ID is a unique, user-assigned ID that can be used by Google APIs.
+          # The project ID must be a unique string of 6 to 30 lowercase letters, digits, or hyphens.
+          # It must start with a letter, and cannot have a trailing hyphen.
+          # See: https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin
+          project_id: <PROJECT_ID>"
+          # An optional GCP SecretManager endpoint. If not set, defaults to: secretmanager.googleapis.com:443
+          endpoint: ""
+          # An optional list of GCP OAuth2 scopes. For a list of GCP scopes refer to: https://developers.google.com/identity/protocols/oauth2/scopes
+          # If not set, the GCP default scopes are used.
+          scopes: 
+          - "https://www.googleapis.com/auth/cloud-platform"
+          # The credentials for your GCP service account. If running inside GCP (app engine) the credentials
+          # can be empty and will be fetched from the app engine environment automatically.
+          credentials:
+            client_email:   "" # The service account email                          - e.g. <account>@<project-ID>.iam.gserviceaccount.com
+            client_id:      "" # The service account client ID                      - e.g. 113491952745362495489"
+            private_key_id: "" # The service account private key                    - e.g. 381514ebd3cf45a64ca8adc561f0ce28fca5ec06
+            private_key:    ""
+          ## KES configured with fs (File System mode) doesnt work in Kubernetes environments and it's not recommended
diff --git a/examples/kustomization/tenant-kes-encryption-gcp/kes-service-account.yaml b/examples/kustomization/tenant-kes-encryption-gcp/kes-service-account.yaml
@@ -0,0 +1,9 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  namespace:  tenant-kms-encrypted
+  # This should be the service account which was created in `kes-service-account.yaml`
+  # Please refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity#impersonate_a_service_account to know how
+  # this service account is authorized to use GCP workload identity
+  name: <SERVICE_ACCOUNT>
+automountServiceAccountToken: false
diff --git a/examples/kustomization/tenant-kes-encryption-gcp/kustomization.yaml b/examples/kustomization/tenant-kes-encryption-gcp/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: tenant-kms-encrypted
+
+resources:
+  - ../base
+  - kes-configuration-secret.yaml
+  - gcp-default-creds-secret.yaml
+  - kes-service-account.yaml
+
+patchesStrategicMerge:
+  - tenant.yaml
diff --git a/examples/kustomization/tenant-kes-encryption-gcp/tenant.yaml b/examples/kustomization/tenant-kes-encryption-gcp/tenant.yaml
@@ -0,0 +1,64 @@
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  name: myminio
+  namespace: minio-tenant
+spec:
+  ## Define configuration for KES (stateless and distributed key-management system)
+  ## Refer https://github.com/minio/kes
+  kes:
+    image: "" # minio/kes:v0.22.3
+    env: [ ]
+    replicas: 2
+    kesSecret:
+      name: kes-configuration
+    imagePullPolicy: "IfNotPresent"
+    gcpCredentialSecretName: gcp-default-creds
+    gcpWorkloadIdentityPool: <WORKLOAD_IDENTITY_POOL>
+    ## Use this field to provide external certificates for the KES server. TLS for KES pods will be configured
+    ## by mounting a Kubernetes secret under /tmp/kes folder, supported types:
+    ## Opaque | kubernetes.io/tls | cert-manager.io/v1alpha2 | cert-manager.io/v1
+    ##
+    ## ie:
+    ##
+    ##  externalCertSecret:
+    ##    name: tls-certificates-for-kes
+    ##    type: kubernetes.io/tls
+    ##
+    ## Create secrets as explained here:
+    ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
+    externalCertSecret: null
+    ## Use this field to provide client certificates for KES. This can be used to configure
+    ## mTLS for KES and your KMS. Files will be mounted under /tmp/kes folder, supported types:
+    ## Opaque | kubernetes.io/tls | cert-manager.io/v1alpha2 | cert-manager.io/v1
+    ##
+    ## ie:
+    ##
+    ##  clientCertSecret:
+    ##    name: mtls-certificates-for-kms
+    ##    type: Opaque
+    ##
+    ## Create secrets as explained here:
+    ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret
+    clientCertSecret: null
+    ## Key name to be created on the KMS, default is "my-minio-key"
+    keyName: ""
+
+    resources: { }
+    nodeSelector: { }
+    affinity:
+      nodeAffinity: { }
+      podAffinity: { }
+      podAntiAffinity: { }
+    tolerations: [ ]
+    annotations: { }
+    labels: { }
+    # This should be the service account which was created in `kes-service-account.yaml`
+    # Please refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity#impersonate_a_service_account to know how
+    # this service account is authorized to use GCP workload identity
+    serviceAccountName: "<SERVICE_ACCOUNT>"
+    securityContext:
+      runAsUser: 1000
+      runAsGroup: 1000
+      runAsNonRoot: true
+      fsGroup: 1000
diff --git a/examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml b/examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml
@@ -2,9 +2,11 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: kes-configuration
+  namespace: tenant-kms-encrypted
 type: Opaque
 stringData:
   server-config.yaml: |-
+    version: v1
     address: :7373
     admin:
       identity: _ # Effectively disabled since no root identity necessary.
@@ -17,7 +19,7 @@ stringData:
           cert: X-Tls-Client-Cert
     policy:
       my-policy:
-        paths:
+        allow:
         - /v1/api
         - /v1/key/create/*
         - /v1/key/generate/*
@@ -63,3 +65,24 @@ stringData:
       #       accesskey: ""  # Your AWS Access Key
       #       secretkey: ""  # Your AWS Secret Key
       #       token: ""      # Your AWS session token (usually optional)
+      # gcp:
+      #   secretmanager:
+      #     # The project ID is a unique, user-assigned ID that can be used by Google APIs.
+      #     # The project ID must be a unique string of 6 to 30 lowercase letters, digits, or hyphens.
+      #     # It must start with a letter, and cannot have a trailing hyphen.
+      #     # See: https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin
+      #     project_id: <PROJECT_ID>
+      #     # An optional GCP SecretManager endpoint. If not set, defaults to: secretmanager.googleapis.com:443
+      #     endpoint: ""
+      #     # An optional list of GCP OAuth2 scopes. For a list of GCP scopes refer to: https://developers.google.com/identity/protocols/oauth2/scopes
+      #     # If not set, the GCP default scopes are used.
+      #     scopes: 
+      #     - "https://www.googleapis.com/auth/cloud-platform"
+      #     # The credentials for your GCP service account. If running inside GCP (app engine) the credentials
+      #     # can be empty and will be fetched from the app engine environment automatically.
+      #     credentials:
+      #       client_email:   "" # The service account email                          - e.g. <account>@<project-ID>.iam.gserviceaccount.com
+      #       client_id:      "" # The service account client ID                      - e.g. 113491952745362495489"
+      #       private_key_id: "" # The service account private key                    - e.g. 381514ebd3cf45a64ca8adc561f0ce28fca5ec06
+      #       private_key:    ""
+      #     ## KES configured with fs (File System mode) doesnt work in Kubernetes environments and it's not recommended
diff --git a/examples/kustomization/tenant-kes-encryption/tenant.yaml b/examples/kustomization/tenant-kes-encryption/tenant.yaml
@@ -7,7 +7,7 @@ spec:
   ## Define configuration for KES (stateless and distributed key-management system)
   ## Refer https://github.com/minio/kes
   kes:
-    image: "" # minio/kes:v0.18.0
+    image: "" # minio/kes:2023-02-15T14-54-37Z
     env: [ ]
     replicas: 2
     kesSecret:

diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml
@@ -669,6 +669,10 @@ spec:
                     required:
                     - name
                     type: object
+                  gcpCredentialSecretName:
+                    type: string
+                  gcpWorkloadIdentityPool:
+                    type: string
                   image:
                     type: string
                   imagePullPolicy:

diff --git a/kubectl-minio/cmd/helpers/constants.go b/kubectl-minio/cmd/helpers/constants.go
@@ -42,7 +42,7 @@ const (
 	DefaultTenantImage = "minio/minio:RELEASE.2023-01-12T02-06-16Z"
 
 	// DefaultKESImage is the default KES image used while creating tenant
-	DefaultKESImage = "minio/kes:v0.18.0"
+	DefaultKESImage = "minio/kes:2023-02-15T14-54-37Z"
 )
 
 // KESReplicas is the number of replicas for MinIO KES

diff --git a/pkg/apis/minio.min.io/v2/constants.go b/pkg/apis/minio.min.io/v2/constants.go
@@ -129,7 +129,7 @@ const ConsoleAdminPolicyName = "consoleAdmin"
 // KES Related Constants
 
 // DefaultKESImage specifies the latest KES Docker hub image
-const DefaultKESImage = "minio/kes:v0.18.0"
+const DefaultKESImage = "minio/kes:2023-02-15T14-54-37Z"
 
 // KESInstanceLabel is applied to the KES pods of a Tenant cluster
 const KESInstanceLabel = "v1.min.io/kes"

diff --git a/pkg/apis/minio.min.io/v2/helper.go b/pkg/apis/minio.min.io/v2/helper.go
@@ -93,6 +93,11 @@ var (
 	prometheusName          string
 	prometheusNamespaceOnce sync.Once
 	prometheusNameOnce      sync.Once
+	// gcpAppCredentialENV to denote the GCP APP credential path
+	gcpAppCredentialENV = corev1.EnvVar{
+		Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+		Value: "/var/run/secrets/tokens/gcp-ksa/google-application-credentials.json",
+	}
 )
 
 // GetPodCAFromFile assumes the operator is running inside a k8s pod and extract the
@@ -351,6 +356,9 @@ func (t *Tenant) EnsureDefaults() *Tenant {
 		if t.Spec.KES.KeyName == "" {
 			t.Spec.KES.KeyName = KESMinIOKey
 		}
+		if t.HasGCPCredentialSecretForKES() && t.Spec.KES.ServiceAccountName == "" {
+			t.Spec.KES.ServiceAccountName = "default"
+		}
 	}
 
 	// ServiceAccount
@@ -514,12 +522,26 @@ func (t *Tenant) GetEnvVars() (env []corev1.EnvVar) {
 	return t.Spec.Env
 }
 
+// HasGCPCredentialSecretForKES returns if GCP cred secret is set in KES for fleet workload identity support.
+func (t *Tenant) HasGCPCredentialSecretForKES() bool {
+	return t.HasKESEnabled() && t.Spec.KES.GCPCredentialSecretName != ""
+}
+
+// HasGCPWorkloadIdentityPoolForKES returns if GCP worload identity pool secret is set in KES for fleet workload identity support.
+func (t *Tenant) HasGCPWorkloadIdentityPoolForKES() bool {
+	return t.HasKESEnabled() && t.Spec.KES.GCPWorkloadIdentityPool != ""
+}
+
 // GetKESEnvVars returns the environment variables for the KES deployment.
 func (t *Tenant) GetKESEnvVars() (env []corev1.EnvVar) {
 	if !t.HasKESEnabled() {
 		return env
 	}
-	return t.Spec.KES.Env
+	env = t.Spec.KES.Env
+	if t.HasGCPCredentialSecretForKES() {
+		env = append(env, gcpAppCredentialENV)
+	}
+	return env
 }
 
 // UpdateURL returns the URL for the sha256sum location of the new binary
@@ -795,6 +817,16 @@ func (t *Tenant) Validate() error {
 		return errors.New("please set 'configuration' secret with credentials for Tenant")
 	}
 
+	if t.HasKESEnabled() {
+		switch {
+		case t.HasGCPCredentialSecretForKES() && !t.HasGCPWorkloadIdentityPoolForKES():
+			return errors.New("please set 'gcpWorkloadIdentityPool' to enable fleet workload identity")
+		case t.HasGCPWorkloadIdentityPoolForKES() && !t.HasGCPCredentialSecretForKES():
+			return errors.New("plese set the 'gcpCredentialSecretName' to enable fleet workload identity")
+		default:
+		}
+	}
+
 	// Every pool must contain a Volume Claim Template
 	for zi, pool := range t.Spec.Pools {
 		if err := pool.Validate(zi); err != nil {

diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go
@@ -756,6 +756,18 @@ type KESConfig struct {
 	ClientCertSecret *LocalCertificateReference `json:"clientCertSecret,omitempty"`
 	// *Optional* +
 	//
+	//  Specify the GCP default credentials to be used for KES to authenticate to GCP key store
+	//
+	// +optional
+	GCPCredentialSecretName string `json:"gcpCredentialSecretName,omitempty"`
+	// *Optional* +
+	//
+	//  Specify the name of the workload identity pool (This is required for generating service account token)
+	//
+	// +optional
+	GCPWorkloadIdentityPool string `json:"gcpWorkloadIdentityPool,omitempty"`
+	// *Optional* +
+	//
 	// If provided, use these annotations for KES Object Meta annotations
 	// +optional
 	Annotations map[string]string `json:"annotations,omitempty"`

diff --git a/pkg/controller/kes.go b/pkg/controller/kes.go
@@ -33,6 +33,7 @@ import (
 	"github.com/minio/operator/pkg/resources/services"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
 	"github.com/minio/operator/pkg/resources/statefulsets"
@@ -222,6 +223,17 @@ func (c *Controller) checkKESStatus(ctx context.Context, tenant *miniov2.Tenant,
 			}
 		}
 
+		if tenant.HasGCPCredentialSecretForKES() {
+			kesSA, err := c.kubeClientSet.CoreV1().ServiceAccounts(tenant.Namespace).Get(ctx, tenant.Spec.KES.ServiceAccountName, v1.GetOptions{})
+			if err != nil {
+				klog.Errorf("unable to get the service account %s/%s: %v", tenant.Namespace, tenant.Spec.KES.ServiceAccountName, err)
+				return err
+			}
+			if *kesSA.AutomountServiceAccountToken {
+				return fmt.Errorf("automountServiceAccountToken should be set to false in service account %s/%s to mount the service token", tenant.Namespace, tenant.Spec.KES.ServiceAccountName)
+			}
+		}
+
 		// Get the StatefulSet with the name specified in spec
 		if existingStatefulSet, err := c.statefulSetLister.StatefulSets(tenant.Namespace).Get(tenant.KESStatefulSetName()); err != nil {
 			if k8serrors.IsNotFound(err) {