diff --git a/examples/kustomization/tenant-kes-encryption-gcp/gcp-default-creds-secret.yaml b/examples/kustomization/tenant-kes-encryption-gcp/gcp-default-creds-secret.yaml new file mode 100644 index 00000000000..f134c196691 --- /dev/null +++ b/examples/kustomization/tenant-kes-encryption-gcp/gcp-default-creds-secret.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Secret +metadata: + name: gcp-default-creds + namespace: tenant-kms-encrypted +type: Opaque +stringData: + # NOTE: refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity#impersonate_a_service_account for the process to extract application default credentials + # Please replace ,, and with the respective values from application default credentials. + config: | + { + "type": "external_account", + "audience": "identitynamespace::", + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/@.iam.gserviceaccount.com:generateAccessToken", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "credential_source": { + "file": "/var/run/secrets/tokens/gcp-ksa/token" + } + } diff --git a/examples/kustomization/tenant-kes-encryption-gcp/kes-configuration-secret.yaml b/examples/kustomization/tenant-kes-encryption-gcp/kes-configuration-secret.yaml new file mode 100644 index 00000000000..67296b87e25 --- /dev/null +++ b/examples/kustomization/tenant-kes-encryption-gcp/kes-configuration-secret.yaml @@ -0,0 +1,67 @@ +apiVersion: v1 +kind: Secret +metadata: + name: kes-configuration + namespace: tenant-kms-encrypted +type: Opaque +stringData: + server-config.yaml: |- + version: v1 + address: :7373 + admin: + identity: _ # Effectively disabled since no root identity necessary. + tls: + key: /tmp/kes/server.key # Path to the TLS private key + cert: /tmp/kes/server.crt # Path to the TLS certificate + proxy: + identities: [] + header: + cert: X-Tls-Client-Cert + policy: + my-policy: + allow: + - /v1/api + - /v1/key/create/* + - /v1/key/import/* + - /v1/key/delete/* + - /v1/key/list/* + - /v1/key/generate/* + - /v1/key/decrypt/* + - /v1/key/encrypt/* + - /v1/key/bulk/decrypt/* + - /v1/status + - /v1/api + - /v1/metrics + - /v1/log/audit + - /v1/log/error + identities: + - ${MINIO_KES_IDENTITY} + cache: + expiry: + any: 5m0s + unused: 20s + log: + error: on + audit: on + keystore: + gcp: + secretmanager: + # The project ID is a unique, user-assigned ID that can be used by Google APIs. + # The project ID must be a unique string of 6 to 30 lowercase letters, digits, or hyphens. + # It must start with a letter, and cannot have a trailing hyphen. + # See: https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin + project_id: " + # An optional GCP SecretManager endpoint. If not set, defaults to: secretmanager.googleapis.com:443 + endpoint: "" + # An optional list of GCP OAuth2 scopes. For a list of GCP scopes refer to: https://developers.google.com/identity/protocols/oauth2/scopes + # If not set, the GCP default scopes are used. + scopes: + - "https://www.googleapis.com/auth/cloud-platform" + # The credentials for your GCP service account. If running inside GCP (app engine) the credentials + # can be empty and will be fetched from the app engine environment automatically. + credentials: + client_email: "" # The service account email - e.g. @.iam.gserviceaccount.com + client_id: "" # The service account client ID - e.g. 113491952745362495489" + private_key_id: "" # The service account private key - e.g. 381514ebd3cf45a64ca8adc561f0ce28fca5ec06 + private_key: "" + ## KES configured with fs (File System mode) doesnt work in Kubernetes environments and it's not recommended diff --git a/examples/kustomization/tenant-kes-encryption-gcp/kes-service-account.yaml b/examples/kustomization/tenant-kes-encryption-gcp/kes-service-account.yaml new file mode 100644 index 00000000000..034da30e7b7 --- /dev/null +++ b/examples/kustomization/tenant-kes-encryption-gcp/kes-service-account.yaml @@ -0,0 +1,9 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + namespace: tenant-kms-encrypted + # This should be the service account which was created in `kes-service-account.yaml` + # Please refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity#impersonate_a_service_account to know how + # this service account is authorized to use GCP workload identity + name: +automountServiceAccountToken: false diff --git a/examples/kustomization/tenant-kes-encryption-gcp/kustomization.yaml b/examples/kustomization/tenant-kes-encryption-gcp/kustomization.yaml new file mode 100644 index 00000000000..f75eb697183 --- /dev/null +++ b/examples/kustomization/tenant-kes-encryption-gcp/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: tenant-kms-encrypted + +resources: + - ../base + - kes-configuration-secret.yaml + - gcp-default-creds-secret.yaml + - kes-service-account.yaml + +patchesStrategicMerge: + - tenant.yaml diff --git a/examples/kustomization/tenant-kes-encryption-gcp/tenant.yaml b/examples/kustomization/tenant-kes-encryption-gcp/tenant.yaml new file mode 100644 index 00000000000..80c754fefc2 --- /dev/null +++ b/examples/kustomization/tenant-kes-encryption-gcp/tenant.yaml @@ -0,0 +1,64 @@ +apiVersion: minio.min.io/v2 +kind: Tenant +metadata: + name: myminio + namespace: minio-tenant +spec: + ## Define configuration for KES (stateless and distributed key-management system) + ## Refer https://github.com/minio/kes + kes: + image: "" # minio/kes:v0.22.3 + env: [ ] + replicas: 2 + kesSecret: + name: kes-configuration + imagePullPolicy: "IfNotPresent" + gcpCredentialSecretName: gcp-default-creds + gcpWorkloadIdentityPool: + ## Use this field to provide external certificates for the KES server. TLS for KES pods will be configured + ## by mounting a Kubernetes secret under /tmp/kes folder, supported types: + ## Opaque | kubernetes.io/tls | cert-manager.io/v1alpha2 | cert-manager.io/v1 + ## + ## ie: + ## + ## externalCertSecret: + ## name: tls-certificates-for-kes + ## type: kubernetes.io/tls + ## + ## Create secrets as explained here: + ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + externalCertSecret: null + ## Use this field to provide client certificates for KES. This can be used to configure + ## mTLS for KES and your KMS. Files will be mounted under /tmp/kes folder, supported types: + ## Opaque | kubernetes.io/tls | cert-manager.io/v1alpha2 | cert-manager.io/v1 + ## + ## ie: + ## + ## clientCertSecret: + ## name: mtls-certificates-for-kms + ## type: Opaque + ## + ## Create secrets as explained here: + ## https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + clientCertSecret: null + ## Key name to be created on the KMS, default is "my-minio-key" + keyName: "" + + resources: { } + nodeSelector: { } + affinity: + nodeAffinity: { } + podAffinity: { } + podAntiAffinity: { } + tolerations: [ ] + annotations: { } + labels: { } + # This should be the service account which was created in `kes-service-account.yaml` + # Please refer https://cloud.google.com/anthos/fleet-management/docs/use-workload-identity#impersonate_a_service_account to know how + # this service account is authorized to use GCP workload identity + serviceAccountName: "" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + runAsNonRoot: true + fsGroup: 1000 diff --git a/examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml b/examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml index 435f6376351..aa080a6288b 100644 --- a/examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml +++ b/examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml @@ -2,9 +2,11 @@ apiVersion: v1 kind: Secret metadata: name: kes-configuration + namespace: tenant-kms-encrypted type: Opaque stringData: server-config.yaml: |- + version: v1 address: :7373 admin: identity: _ # Effectively disabled since no root identity necessary. @@ -17,7 +19,7 @@ stringData: cert: X-Tls-Client-Cert policy: my-policy: - paths: + allow: - /v1/api - /v1/key/create/* - /v1/key/generate/* @@ -63,3 +65,24 @@ stringData: # accesskey: "" # Your AWS Access Key # secretkey: "" # Your AWS Secret Key # token: "" # Your AWS session token (usually optional) + # gcp: + # secretmanager: + # # The project ID is a unique, user-assigned ID that can be used by Google APIs. + # # The project ID must be a unique string of 6 to 30 lowercase letters, digits, or hyphens. + # # It must start with a letter, and cannot have a trailing hyphen. + # # See: https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin + # project_id: + # # An optional GCP SecretManager endpoint. If not set, defaults to: secretmanager.googleapis.com:443 + # endpoint: "" + # # An optional list of GCP OAuth2 scopes. For a list of GCP scopes refer to: https://developers.google.com/identity/protocols/oauth2/scopes + # # If not set, the GCP default scopes are used. + # scopes: + # - "https://www.googleapis.com/auth/cloud-platform" + # # The credentials for your GCP service account. If running inside GCP (app engine) the credentials + # # can be empty and will be fetched from the app engine environment automatically. + # credentials: + # client_email: "" # The service account email - e.g. @.iam.gserviceaccount.com + # client_id: "" # The service account client ID - e.g. 113491952745362495489" + # private_key_id: "" # The service account private key - e.g. 381514ebd3cf45a64ca8adc561f0ce28fca5ec06 + # private_key: "" + # ## KES configured with fs (File System mode) doesnt work in Kubernetes environments and it's not recommended diff --git a/examples/kustomization/tenant-kes-encryption/tenant.yaml b/examples/kustomization/tenant-kes-encryption/tenant.yaml index 218ac9ebf72..46fe116ade1 100644 --- a/examples/kustomization/tenant-kes-encryption/tenant.yaml +++ b/examples/kustomization/tenant-kes-encryption/tenant.yaml @@ -7,7 +7,7 @@ spec: ## Define configuration for KES (stateless and distributed key-management system) ## Refer https://github.com/minio/kes kes: - image: "" # minio/kes:v0.18.0 + image: "" # minio/kes:2023-02-15T14-54-37Z env: [ ] replicas: 2 kesSecret: diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index 87119b469a7..6b0eb95b135 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -669,6 +669,10 @@ spec: required: - name type: object + gcpCredentialSecretName: + type: string + gcpWorkloadIdentityPool: + type: string image: type: string imagePullPolicy: diff --git a/kubectl-minio/cmd/helpers/constants.go b/kubectl-minio/cmd/helpers/constants.go index 838ab79fc23..1c781ae54d6 100644 --- a/kubectl-minio/cmd/helpers/constants.go +++ b/kubectl-minio/cmd/helpers/constants.go @@ -42,7 +42,7 @@ const ( DefaultTenantImage = "minio/minio:RELEASE.2023-01-12T02-06-16Z" // DefaultKESImage is the default KES image used while creating tenant - DefaultKESImage = "minio/kes:v0.18.0" + DefaultKESImage = "minio/kes:2023-02-15T14-54-37Z" ) // KESReplicas is the number of replicas for MinIO KES diff --git a/pkg/apis/minio.min.io/v2/constants.go b/pkg/apis/minio.min.io/v2/constants.go index 096e782fc3e..ce78cf53c0b 100644 --- a/pkg/apis/minio.min.io/v2/constants.go +++ b/pkg/apis/minio.min.io/v2/constants.go @@ -129,7 +129,7 @@ const ConsoleAdminPolicyName = "consoleAdmin" // KES Related Constants // DefaultKESImage specifies the latest KES Docker hub image -const DefaultKESImage = "minio/kes:v0.18.0" +const DefaultKESImage = "minio/kes:2023-02-15T14-54-37Z" // KESInstanceLabel is applied to the KES pods of a Tenant cluster const KESInstanceLabel = "v1.min.io/kes" diff --git a/pkg/apis/minio.min.io/v2/helper.go b/pkg/apis/minio.min.io/v2/helper.go index d7fee5a50af..36e7c18af7b 100644 --- a/pkg/apis/minio.min.io/v2/helper.go +++ b/pkg/apis/minio.min.io/v2/helper.go @@ -93,6 +93,11 @@ var ( prometheusName string prometheusNamespaceOnce sync.Once prometheusNameOnce sync.Once + // gcpAppCredentialENV to denote the GCP APP credential path + gcpAppCredentialENV = corev1.EnvVar{ + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: "/var/run/secrets/tokens/gcp-ksa/google-application-credentials.json", + } ) // GetPodCAFromFile assumes the operator is running inside a k8s pod and extract the @@ -351,6 +356,9 @@ func (t *Tenant) EnsureDefaults() *Tenant { if t.Spec.KES.KeyName == "" { t.Spec.KES.KeyName = KESMinIOKey } + if t.HasGCPCredentialSecretForKES() && t.Spec.KES.ServiceAccountName == "" { + t.Spec.KES.ServiceAccountName = "default" + } } // ServiceAccount @@ -514,12 +522,26 @@ func (t *Tenant) GetEnvVars() (env []corev1.EnvVar) { return t.Spec.Env } +// HasGCPCredentialSecretForKES returns if GCP cred secret is set in KES for fleet workload identity support. +func (t *Tenant) HasGCPCredentialSecretForKES() bool { + return t.HasKESEnabled() && t.Spec.KES.GCPCredentialSecretName != "" +} + +// HasGCPWorkloadIdentityPoolForKES returns if GCP worload identity pool secret is set in KES for fleet workload identity support. +func (t *Tenant) HasGCPWorkloadIdentityPoolForKES() bool { + return t.HasKESEnabled() && t.Spec.KES.GCPWorkloadIdentityPool != "" +} + // GetKESEnvVars returns the environment variables for the KES deployment. func (t *Tenant) GetKESEnvVars() (env []corev1.EnvVar) { if !t.HasKESEnabled() { return env } - return t.Spec.KES.Env + env = t.Spec.KES.Env + if t.HasGCPCredentialSecretForKES() { + env = append(env, gcpAppCredentialENV) + } + return env } // UpdateURL returns the URL for the sha256sum location of the new binary @@ -795,6 +817,16 @@ func (t *Tenant) Validate() error { return errors.New("please set 'configuration' secret with credentials for Tenant") } + if t.HasKESEnabled() { + switch { + case t.HasGCPCredentialSecretForKES() && !t.HasGCPWorkloadIdentityPoolForKES(): + return errors.New("please set 'gcpWorkloadIdentityPool' to enable fleet workload identity") + case t.HasGCPWorkloadIdentityPoolForKES() && !t.HasGCPCredentialSecretForKES(): + return errors.New("plese set the 'gcpCredentialSecretName' to enable fleet workload identity") + default: + } + } + // Every pool must contain a Volume Claim Template for zi, pool := range t.Spec.Pools { if err := pool.Validate(zi); err != nil { diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index d1113d551cc..e3c7bcf9b35 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -756,6 +756,18 @@ type KESConfig struct { ClientCertSecret *LocalCertificateReference `json:"clientCertSecret,omitempty"` // *Optional* + // + // Specify the GCP default credentials to be used for KES to authenticate to GCP key store + // + // +optional + GCPCredentialSecretName string `json:"gcpCredentialSecretName,omitempty"` + // *Optional* + + // + // Specify the name of the workload identity pool (This is required for generating service account token) + // + // +optional + GCPWorkloadIdentityPool string `json:"gcpWorkloadIdentityPool,omitempty"` + // *Optional* + + // // If provided, use these annotations for KES Object Meta annotations // +optional Annotations map[string]string `json:"annotations,omitempty"` diff --git a/pkg/controller/kes.go b/pkg/controller/kes.go index 1727b9da7ac..1891c764b57 100644 --- a/pkg/controller/kes.go +++ b/pkg/controller/kes.go @@ -33,6 +33,7 @@ import ( "github.com/minio/operator/pkg/resources/services" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "github.com/minio/operator/pkg/resources/statefulsets" @@ -222,6 +223,17 @@ func (c *Controller) checkKESStatus(ctx context.Context, tenant *miniov2.Tenant, } } + if tenant.HasGCPCredentialSecretForKES() { + kesSA, err := c.kubeClientSet.CoreV1().ServiceAccounts(tenant.Namespace).Get(ctx, tenant.Spec.KES.ServiceAccountName, v1.GetOptions{}) + if err != nil { + klog.Errorf("unable to get the service account %s/%s: %v", tenant.Namespace, tenant.Spec.KES.ServiceAccountName, err) + return err + } + if *kesSA.AutomountServiceAccountToken { + return fmt.Errorf("automountServiceAccountToken should be set to false in service account %s/%s to mount the service token", tenant.Namespace, tenant.Spec.KES.ServiceAccountName) + } + } + // Get the StatefulSet with the name specified in spec if existingStatefulSet, err := c.statefulSetLister.StatefulSets(tenant.Namespace).Get(tenant.KESStatefulSetName()); err != nil { if k8serrors.IsNotFound(err) { diff --git a/pkg/resources/statefulsets/kes-statefulset.go b/pkg/resources/statefulsets/kes-statefulset.go index 769b1a3d61f..8eb777e8608 100644 --- a/pkg/resources/statefulsets/kes-statefulset.go +++ b/pkg/resources/statefulsets/kes-statefulset.go @@ -23,6 +23,23 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + gcpCredentialVolumeMountName = "gcp-ksa" + gcpCredentialVolumeMountPath = "/var/run/secrets/tokens/gcp-ksa" + serviceAccountTokenPath = "token" + gcpAppCredentialsPath = "google-application-credentials.json" +) + +var ( + defaultServiceAccountTokenExpiryInSecs int64 = 172800 // 48 hrs + // gcpCredentialVolumeMount represents the volume mount for GCP creds and service token + gcpCredentialVolumeMount = corev1.VolumeMount{ + Name: gcpCredentialVolumeMountName, + ReadOnly: true, + MountPath: gcpCredentialVolumeMountPath, + } +) + // KESMetadata Returns the KES pods metadata set in configuration. // If a user specifies metadata in the spec we return that // metadata. @@ -48,13 +65,17 @@ func KESSelector(t *miniov2.Tenant) *metav1.LabelSelector { } // KESVolumeMounts builds the volume mounts for MinIO container. -func KESVolumeMounts(t *miniov2.Tenant) []corev1.VolumeMount { - return []corev1.VolumeMount{ +func KESVolumeMounts(t *miniov2.Tenant) (volumeMounts []corev1.VolumeMount) { + volumeMounts = []corev1.VolumeMount{ { Name: t.KESVolMountName(), MountPath: miniov2.KESConfigMountPath, }, } + if t.HasGCPCredentialSecretForKES() { + volumeMounts = append(volumeMounts, gcpCredentialVolumeMount) + } + return } // KESEnvironmentVars returns the KES environment variables set in configuration. @@ -197,6 +218,39 @@ func NewForKES(t *miniov2.Tenant, serviceName string) *appsv1.StatefulSet { }, } + if t.HasGCPCredentialSecretForKES() { + var gcpVolumeDefaultMode int32 = 420 + var isOptional bool + podVolumes = append(podVolumes, corev1.Volume{ + Name: gcpCredentialVolumeMountName, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: &gcpVolumeDefaultMode, + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: t.Spec.KES.GCPCredentialSecretName, + }, + Items: []corev1.KeyToPath{ + {Key: "config", Path: gcpAppCredentialsPath}, + }, + Optional: &isOptional, + }, + }, + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Audience: t.Spec.KES.GCPWorkloadIdentityPool, + ExpirationSeconds: &defaultServiceAccountTokenExpiryInSecs, + Path: serviceAccountTokenPath, + }, + }, + }, + }, + }, + }) + } + containers := []corev1.Container{KESServerContainer(t)} ss := &appsv1.StatefulSet{ diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index 87119b469a7..6b0eb95b135 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -669,6 +669,10 @@ spec: required: - name type: object + gcpCredentialSecretName: + type: string + gcpWorkloadIdentityPool: + type: string image: type: string imagePullPolicy: diff --git a/testing/console-tenant+kes.sh b/testing/console-tenant+kes.sh index 7c6228b9bd0..d195edbaff8 100755 --- a/testing/console-tenant+kes.sh +++ b/testing/console-tenant+kes.sh @@ -113,7 +113,7 @@ function test_kes_tenant() { sed -i -e 's/ROLE_ID/'"$ROLE_ID"'/g' "${SCRIPT_DIR}/kes-config.yaml" sed -i -e 's/SECRET_ID/'"$SECRET_ID"'/g' "${SCRIPT_DIR}/kes-config.yaml" cp "${SCRIPT_DIR}/kes-config.yaml" "${SCRIPT_DIR}/../examples/kustomization/tenant-kes-encryption/kes-configuration-secret.yaml" - yq e -i '.spec.kes.image = "minio/kes:v0.18.0"' "${SCRIPT_DIR}/../examples/kustomization/tenant-kes-encryption/tenant.yaml" + yq e -i '.spec.kes.image = "minio/kes:2023-02-15T14-54-37Z"' "${SCRIPT_DIR}/../examples/kustomization/tenant-kes-encryption/tenant.yaml" kubectl apply -k "${SCRIPT_DIR}/../examples/kustomization/tenant-kes-encryption" echo "Check Tenant Status in tenant-kms-encrypted namespace for myminio:"