diff --git a/flytepropeller/pkg/webhook/config/config.go b/flytepropeller/pkg/webhook/config/config.go index 61c598c1d..c3cf9f2d8 100644 --- a/flytepropeller/pkg/webhook/config/config.go +++ b/flytepropeller/pkg/webhook/config/config.go @@ -33,6 +33,19 @@ var ( }, }, }, + GCPSecretManagerConfig: GCPSecretManagerConfig{ + SidecarImage: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500Mi"), + corev1.ResourceCPU: resource.MustParse("200m"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500Mi"), + corev1.ResourceCPU: resource.MustParse("200m"), + }, + }, + }, VaultSecretManagerConfig: VaultSecretManagerConfig{ Role: "flyte", KVVersion: KVVersion2, @@ -57,6 +70,10 @@ const ( // Manager and mount them to a local file system (in memory) and share that mount with other containers in the pod. SecretManagerTypeAWS + // SecretManagerTypeGCP defines a secret manager webhook that injects a side car to pull secrets from GCP Secret + // Manager and mount them to a local file system (in memory) and share that mount with other containers in the pod. + SecretManagerTypeGCP + // SecretManagerTypeVault defines a secret manager webhook that pulls secrets from Hashicorp Vault. SecretManagerTypeVault ) @@ -81,6 +98,7 @@ type Config struct { SecretName string `json:"secretName" pflag:",Secret name to write generated certs to."` SecretManagerType SecretManagerType `json:"secretManagerType" pflag:"-,Secret manager type to use if secrets are not found in global secrets."` AWSSecretManagerConfig AWSSecretManagerConfig `json:"awsSecretManager" pflag:",AWS Secret Manager config."` + GCPSecretManagerConfig GCPSecretManagerConfig `json:"gcpSecretManager" pflag:",GCP Secret Manager config."` VaultSecretManagerConfig VaultSecretManagerConfig `json:"vaultSecretManager" pflag:",Vault Secret Manager config."` } @@ -89,6 +107,11 @@ type AWSSecretManagerConfig struct { Resources corev1.ResourceRequirements `json:"resources" pflag:"-,Specifies resource requirements for the init container."` } +type GCPSecretManagerConfig struct { + SidecarImage string `json:"sidecarImage" pflag:",Specifies the sidecar docker image to use"` + Resources corev1.ResourceRequirements `json:"resources" pflag:"-,Specifies resource requirements for the init container."` +} + type VaultSecretManagerConfig struct { Role string `json:"role" pflag:",Specifies the vault role to use"` KVVersion KVVersion `json:"kvVersion" pflag:"-,The KV Engine Version. Defaults to 2. Use 1 for unversioned secrets. Refer to - https://www.vaultproject.io/docs/secrets/kv#kv-secrets-engine."` diff --git a/flytepropeller/pkg/webhook/config/config_flags.go b/flytepropeller/pkg/webhook/config/config_flags.go index 7ef9575d7..089bc0064 100755 --- a/flytepropeller/pkg/webhook/config/config_flags.go +++ b/flytepropeller/pkg/webhook/config/config_flags.go @@ -58,6 +58,7 @@ func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.Int32(fmt.Sprintf("%v%v", prefix, "servicePort"), DefaultConfig.ServicePort, "The port on the service that hosting webhook.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "secretName"), DefaultConfig.SecretName, "Secret name to write generated certs to.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "awsSecretManager.sidecarImage"), DefaultConfig.AWSSecretManagerConfig.SidecarImage, "Specifies the sidecar docker image to use") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "gcpSecretManager.sidecarImage"), DefaultConfig.GCPSecretManagerConfig.SidecarImage, "Specifies the sidecar docker image to use") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "vaultSecretManager.role"), DefaultConfig.VaultSecretManagerConfig.Role, "Specifies the vault role to use") return cmdFlags } diff --git a/flytepropeller/pkg/webhook/config/config_flags_test.go b/flytepropeller/pkg/webhook/config/config_flags_test.go index e68b5af13..613a0f6a3 100755 --- a/flytepropeller/pkg/webhook/config/config_flags_test.go +++ b/flytepropeller/pkg/webhook/config/config_flags_test.go @@ -211,6 +211,20 @@ func TestConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_gcpSecretManager.sidecarImage", func(t *testing.T) { + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("gcpSecretManager.sidecarImage", testValue) + if vString, err := cmdFlags.GetString("gcpSecretManager.sidecarImage"); err == nil { + testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.GCPSecretManagerConfig.SidecarImage) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) t.Run("Test_vaultSecretManager.role", func(t *testing.T) { t.Run("Override", func(t *testing.T) { diff --git a/flytepropeller/pkg/webhook/config/secretmanagertype_enumer.go b/flytepropeller/pkg/webhook/config/secretmanagertype_enumer.go index ce33f910f..986b8b135 100644 --- a/flytepropeller/pkg/webhook/config/secretmanagertype_enumer.go +++ b/flytepropeller/pkg/webhook/config/secretmanagertype_enumer.go @@ -7,9 +7,9 @@ import ( "fmt" ) -const _SecretManagerTypeName = "GlobalK8sAWSVault" +const _SecretManagerTypeName = "GlobalK8sAWSGCPVault" -var _SecretManagerTypeIndex = [...]uint8{0, 6, 9, 12, 17} +var _SecretManagerTypeIndex = [...]uint8{0, 6, 9, 12, 15, 20} func (i SecretManagerType) String() string { if i < 0 || i >= SecretManagerType(len(_SecretManagerTypeIndex)-1) { @@ -18,13 +18,14 @@ func (i SecretManagerType) String() string { return _SecretManagerTypeName[_SecretManagerTypeIndex[i]:_SecretManagerTypeIndex[i+1]] } -var _SecretManagerTypeValues = []SecretManagerType{0, 1, 2, 3} +var _SecretManagerTypeValues = []SecretManagerType{0, 1, 2, 3, 4} var _SecretManagerTypeNameToValueMap = map[string]SecretManagerType{ _SecretManagerTypeName[0:6]: 0, _SecretManagerTypeName[6:9]: 1, _SecretManagerTypeName[9:12]: 2, - _SecretManagerTypeName[12:17]: 3, + _SecretManagerTypeName[12:15]: 3, + _SecretManagerTypeName[15:20]: 4, } // SecretManagerTypeString retrieves an enum value from the enum constants string name. diff --git a/flytepropeller/pkg/webhook/gcp_secret_manager.go b/flytepropeller/pkg/webhook/gcp_secret_manager.go new file mode 100644 index 000000000..f17c1509a --- /dev/null +++ b/flytepropeller/pkg/webhook/gcp_secret_manager.go @@ -0,0 +1,156 @@ +package webhook + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flytepropeller/pkg/webhook/config" + "github.com/flyteorg/flytestdlib/logger" + corev1 "k8s.io/api/core/v1" +) + +const ( + // GCPSecretsVolumeName defines the static name of the volume used for mounting/sharing secrets between init-container + // sidecar and the rest of the containers in the pod. + GCPSecretsVolumeName = "gcp-secret-vol" // #nosec +) + +var ( + // GCPSecretMountPath defines the default mount path for secrets + GCPSecretMountPath = filepath.Join(string(os.PathSeparator), "etc", "flyte", "secrets") +) + +// GCPSecretManagerInjector allows injecting of secrets from GCP Secret Manager as files. It uses a Google Cloud +// SDK SideCar as an init-container to download the secret and save it to a local volume shared with all other +// containers in the pod. It supports multiple secrets to be mounted but that will result into adding an init +// container for each secret. The Google serviceaccount (GSA) associated with the Pod, either via Workload +// Identity (recommended) or the underlying node's serviceacccount, must have permissions to pull the secret +// from GCP Secret Manager. Currently, the secret must also exist in the same project. Otherwise, the Pod will +// fail with an init-error. +// Files will be mounted on /etc/flyte/secrets// +type GCPSecretManagerInjector struct { + cfg config.GCPSecretManagerConfig +} + +func formatGCPSecretAccessCommand(secret *core.Secret) []string { + // `gcloud` writes this file with permission 0600. + // This will cause permission issues in the main container when using non-root + // users, so we fix the file permissions with `chmod`. + secretDir := strings.ToLower(filepath.Join(GCPSecretMountPath, secret.Group)) + secretPath := strings.ToLower(filepath.Join(secretDir, secret.GroupVersion)) + args := []string{ + "gcloud", + "secrets", + "versions", + "access", + secret.GroupVersion, + fmt.Sprintf("--secret=%s", secret.Group), + fmt.Sprintf( + "--out-file=%s", + secretPath, + ), + "&&", + "chmod", + "+rX", + secretDir, + secretPath, + } + return []string{"sh", "-c", strings.Join(args, " ")} +} + +func formatGCPInitContainerName(index int) string { + return fmt.Sprintf("gcp-pull-secret-%v", index) +} + +func (i GCPSecretManagerInjector) Type() config.SecretManagerType { + return config.SecretManagerTypeGCP +} + +func (i GCPSecretManagerInjector) Inject(ctx context.Context, secret *core.Secret, p *corev1.Pod) (newP *corev1.Pod, injected bool, err error) { + if len(secret.Group) == 0 || len(secret.GroupVersion) == 0 { + return nil, false, fmt.Errorf("GCP Secrets Webhook require both group and group version to be set. "+ + "Secret: [%v]", secret) + } + + switch secret.MountRequirement { + case core.Secret_ANY: + fallthrough + case core.Secret_FILE: + // A Volume with a static name so that if we try to inject multiple secrets, we won't mount multiple volumes. + // We use Memory as the storage medium for volume source to avoid + vol := corev1.Volume{ + Name: GCPSecretsVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + } + + p.Spec.Volumes = appendVolumeIfNotExists(p.Spec.Volumes, vol) + p.Spec.InitContainers = append(p.Spec.InitContainers, createGCPSidecarContainer(i.cfg, p, secret)) + + secretVolumeMount := corev1.VolumeMount{ + Name: GCPSecretsVolumeName, + ReadOnly: true, + MountPath: GCPSecretMountPath, + } + + p.Spec.Containers = AppendVolumeMounts(p.Spec.Containers, secretVolumeMount) + p.Spec.InitContainers = AppendVolumeMounts(p.Spec.InitContainers, secretVolumeMount) + + // Inject GCP secret-inject webhook annotations to mount the secret in a predictable location. + envVars := []corev1.EnvVar{ + // Set environment variable to let the container know where to find the mounted files. + { + Name: SecretPathDefaultDirEnvVar, + Value: GCPSecretMountPath, + }, + // Sets an empty prefix to let the containers know the file names will match the secret keys as-is. + { + Name: SecretPathFilePrefixEnvVar, + Value: "", + }, + } + + for _, envVar := range envVars { + p.Spec.InitContainers = AppendEnvVars(p.Spec.InitContainers, envVar) + p.Spec.Containers = AppendEnvVars(p.Spec.Containers, envVar) + } + case core.Secret_ENV_VAR: + fallthrough + default: + err := fmt.Errorf("unrecognized mount requirement [%v] for secret [%v]", secret.MountRequirement.String(), secret.Key) + logger.Error(ctx, err) + return p, false, err + } + + return p, true, nil +} + +func createGCPSidecarContainer(cfg config.GCPSecretManagerConfig, p *corev1.Pod, secret *core.Secret) corev1.Container { + return corev1.Container{ + Image: cfg.SidecarImage, + // Create a unique name to allow multiple secrets to be mounted. + Name: formatGCPInitContainerName(len(p.Spec.InitContainers)), + Command: formatGCPSecretAccessCommand(secret), + VolumeMounts: []corev1.VolumeMount{ + { + Name: GCPSecretsVolumeName, + MountPath: GCPSecretMountPath, + }, + }, + Resources: cfg.Resources, + } +} + +// NewGCPSecretManagerInjector creates a SecretInjector that's able to mount secrets from GCP Secret Manager. +func NewGCPSecretManagerInjector(cfg config.GCPSecretManagerConfig) GCPSecretManagerInjector { + return GCPSecretManagerInjector{ + cfg: cfg, + } +} diff --git a/flytepropeller/pkg/webhook/gcp_secret_manager_test.go b/flytepropeller/pkg/webhook/gcp_secret_manager_test.go new file mode 100644 index 000000000..26805eafc --- /dev/null +++ b/flytepropeller/pkg/webhook/gcp_secret_manager_test.go @@ -0,0 +1,79 @@ +package webhook + +import ( + "context" + "testing" + + "github.com/flyteorg/flytepropeller/pkg/webhook/config" + + "github.com/go-test/deep" + + "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" +) + +func TestGCPSecretManagerInjector_Inject(t *testing.T) { + injector := NewGCPSecretManagerInjector(config.DefaultConfig.GCPSecretManagerConfig) + inputSecret := &core.Secret{ + Group: "TestSecret", + GroupVersion: "2", + } + + expected := &corev1.Pod{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "gcp-secret-vol", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + }, + }, + }, + }, + + InitContainers: []corev1.Container{ + { + Name: "gcp-pull-secret-0", + Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine", + Command: []string{ + "sh", + "-c", + "gcloud secrets versions access 2 --secret=TestSecret --out-file=/etc/flyte/secrets/testsecret/2 && chmod +rX /etc/flyte/secrets/testsecret /etc/flyte/secrets/testsecret/2", + }, + Env: []corev1.EnvVar{ + { + Name: "FLYTE_SECRETS_DEFAULT_DIR", + Value: "/etc/flyte/secrets", + }, + { + Name: "FLYTE_SECRETS_FILE_PREFIX", + Value: "", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gcp-secret-vol", + MountPath: "/etc/flyte/secrets", + }, + }, + Resources: config.DefaultConfig.GCPSecretManagerConfig.Resources, + }, + }, + Containers: []corev1.Container{}, + }, + } + + p := &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{}, + }, + } + actualP, injected, err := injector.Inject(context.Background(), inputSecret, p) + assert.NoError(t, err) + assert.True(t, injected) + if diff := deep.Equal(actualP, expected); diff != nil { + assert.Fail(t, "actual != expected", "Diff: %v", diff) + } +} diff --git a/flytepropeller/pkg/webhook/secrets.go b/flytepropeller/pkg/webhook/secrets.go index ffffc53cd..eae878cef 100644 --- a/flytepropeller/pkg/webhook/secrets.go +++ b/flytepropeller/pkg/webhook/secrets.go @@ -74,6 +74,7 @@ func NewSecretsMutator(cfg *config.Config, _ promutils.Scope) *SecretsMutator { NewGlobalSecrets(secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig())), NewK8sSecretsInjector(), NewAWSSecretManagerInjector(cfg.AWSSecretManagerConfig), + NewGCPSecretManagerInjector(cfg.GCPSecretManagerConfig), NewVaultSecretManagerInjector(cfg.VaultSecretManagerConfig), }, }