Add support for GCP secret manager (flyteorg#547)

Signed-off-by: Jeev B <jeevb@users.noreply.github.com>

pkg/webhook/config/config.go

@@ -33,6 +33,19 @@ var (
  },
  },
  },
+ GCPSecretManagerConfig: GCPSecretManagerConfig{
+ SidecarImage: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine",
+ Resources: corev1.ResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceMemory: resource.MustParse("500Mi"),
+ corev1.ResourceCPU: resource.MustParse("200m"),
+ },
+ Limits: corev1.ResourceList{
+ corev1.ResourceMemory: resource.MustParse("500Mi"),
+ corev1.ResourceCPU: resource.MustParse("200m"),
+ },
+ },
+ },
  VaultSecretManagerConfig: VaultSecretManagerConfig{
  Role: "flyte",
  KVVersion: KVVersion2,
@@ -57,6 +70,10 @@ const (
  // Manager and mount them to a local file system (in memory) and share that mount with other containers in the pod.
  SecretManagerTypeAWS
 
+ // SecretManagerTypeGCP defines a secret manager webhook that injects a side car to pull secrets from GCP Secret
+ // Manager and mount them to a local file system (in memory) and share that mount with other containers in the pod.
+ SecretManagerTypeGCP
+
  // SecretManagerTypeVault defines a secret manager webhook that pulls secrets from Hashicorp Vault.
  SecretManagerTypeVault
 )
@@ -81,6 +98,7 @@ type Config struct {
  SecretName string `json:"secretName" pflag:",Secret name to write generated certs to."`
  SecretManagerType SecretManagerType `json:"secretManagerType" pflag:"-,Secret manager type to use if secrets are not found in global secrets."`
  AWSSecretManagerConfig AWSSecretManagerConfig `json:"awsSecretManager" pflag:",AWS Secret Manager config."`
+ GCPSecretManagerConfig GCPSecretManagerConfig `json:"gcpSecretManager" pflag:",GCP Secret Manager config."`
  VaultSecretManagerConfig VaultSecretManagerConfig `json:"vaultSecretManager" pflag:",Vault Secret Manager config."`
 }
 
@@ -89,6 +107,11 @@ type AWSSecretManagerConfig struct {
  Resources corev1.ResourceRequirements `json:"resources" pflag:"-,Specifies resource requirements for the init container."`
 }
 
+type GCPSecretManagerConfig struct {
+ SidecarImage string `json:"sidecarImage" pflag:",Specifies the sidecar docker image to use"`
+ Resources corev1.ResourceRequirements `json:"resources" pflag:"-,Specifies resource requirements for the init container."`
+}
+
 type VaultSecretManagerConfig struct {
  Role string `json:"role" pflag:",Specifies the vault role to use"`
  KVVersion KVVersion `json:"kvVersion" pflag:"-,The KV Engine Version. Defaults to 2. Use 1 for unversioned secrets. Refer to - https://www.vaultproject.io/docs/secrets/kv#kv-secrets-engine."`

pkg/webhook/gcp_secret_manager.go

@@ -0,0 +1,156 @@
+package webhook
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+ "github.com/flyteorg/flytepropeller/pkg/webhook/config"
+ "github.com/flyteorg/flytestdlib/logger"
+ corev1 "k8s.io/api/core/v1"
+)
+
+const (
+ // GCPSecretsVolumeName defines the static name of the volume used for mounting/sharing secrets between init-container
+ // sidecar and the rest of the containers in the pod.
+ GCPSecretsVolumeName = "gcp-secret-vol" // #nosec
+)
+
+var (
+ // GCPSecretMountPath defines the default mount path for secrets
+ GCPSecretMountPath = filepath.Join(string(os.PathSeparator), "etc", "flyte", "secrets")
+)
+
+// GCPSecretManagerInjector allows injecting of secrets from GCP Secret Manager as files. It uses a Google Cloud
+// SDK SideCar as an init-container to download the secret and save it to a local volume shared with all other
+// containers in the pod. It supports multiple secrets to be mounted but that will result into adding an init
+// container for each secret. The Google serviceaccount (GSA) associated with the Pod, either via Workload
+// Identity (recommended) or the underlying node's serviceacccount, must have permissions to pull the secret
+// from GCP Secret Manager. Currently, the secret must also exist in the same project. Otherwise, the Pod will
+// fail with an init-error.
+// Files will be mounted on /etc/flyte/secrets/<SecretGroup>/<SecretGroupVersion>
+type GCPSecretManagerInjector struct {
+ cfg config.GCPSecretManagerConfig
+}
+
+func formatGCPSecretAccessCommand(secret *core.Secret) []string {
+ // `gcloud` writes this file with permission 0600.
+ // This will cause permission issues in the main container when using non-root
+ // users, so we fix the file permissions with `chmod`.
+ secretDir := strings.ToLower(filepath.Join(GCPSecretMountPath, secret.Group))
+ secretPath := strings.ToLower(filepath.Join(secretDir, secret.GroupVersion))
+ args := []string{
+ "gcloud",
+ "secrets",
+ "versions",
+ "access",
+ secret.GroupVersion,
+ fmt.Sprintf("--secret=%s", secret.Group),
+ fmt.Sprintf(
+ "--out-file=%s",
+ secretPath,
+ ),
+ "&&",
+ "chmod",
+ "+rX",
+ secretDir,
+ secretPath,
+ }
+ return []string{"sh", "-c", strings.Join(args, " ")}
+}
+
+func formatGCPInitContainerName(index int) string {
+ return fmt.Sprintf("gcp-pull-secret-%v", index)
+}
+
+func (i GCPSecretManagerInjector) Type() config.SecretManagerType {
+ return config.SecretManagerTypeGCP
+}
+
+func (i GCPSecretManagerInjector) Inject(ctx context.Context, secret *core.Secret, p *corev1.Pod) (newP *corev1.Pod, injected bool, err error) {
+ if len(secret.Group) == 0 || len(secret.GroupVersion) == 0 {
+ return nil, false, fmt.Errorf("GCP Secrets Webhook require both group and group version to be set. "+
+ "Secret: [%v]", secret)
+ }
+
+ switch secret.MountRequirement {
+ case core.Secret_ANY:
+ fallthrough
+ case core.Secret_FILE:
+ // A Volume with a static name so that if we try to inject multiple secrets, we won't mount multiple volumes.
+ // We use Memory as the storage medium for volume source to avoid
+ vol := corev1.Volume{
+ Name: GCPSecretsVolumeName,
+ VolumeSource: corev1.VolumeSource{
+ EmptyDir: &corev1.EmptyDirVolumeSource{
+ Medium: corev1.StorageMediumMemory,
+ },
+ },
+ }
+
+ p.Spec.Volumes = appendVolumeIfNotExists(p.Spec.Volumes, vol)
+ p.Spec.InitContainers = append(p.Spec.InitContainers, createGCPSidecarContainer(i.cfg, p, secret))
+
+ secretVolumeMount := corev1.VolumeMount{
+ Name: GCPSecretsVolumeName,
+ ReadOnly: true,
+ MountPath: GCPSecretMountPath,
+ }
+
+ p.Spec.Containers = AppendVolumeMounts(p.Spec.Containers, secretVolumeMount)
+ p.Spec.InitContainers = AppendVolumeMounts(p.Spec.InitContainers, secretVolumeMount)
+
+ // Inject GCP secret-inject webhook annotations to mount the secret in a predictable location.
+ envVars := []corev1.EnvVar{
+ // Set environment variable to let the container know where to find the mounted files.
+ {
+ Name: SecretPathDefaultDirEnvVar,
+ Value: GCPSecretMountPath,
+ },
+ // Sets an empty prefix to let the containers know the file names will match the secret keys as-is.
+ {
+ Name: SecretPathFilePrefixEnvVar,
+ Value: "",
+ },
+ }
+
+ for _, envVar := range envVars {
+ p.Spec.InitContainers = AppendEnvVars(p.Spec.InitContainers, envVar)
+ p.Spec.Containers = AppendEnvVars(p.Spec.Containers, envVar)
+ }
+ case core.Secret_ENV_VAR:
+ fallthrough
+ default:
+ err := fmt.Errorf("unrecognized mount requirement [%v] for secret [%v]", secret.MountRequirement.String(), secret.Key)
+ logger.Error(ctx, err)
+ return p, false, err
+ }
+
+ return p, true, nil
+}
+
+func createGCPSidecarContainer(cfg config.GCPSecretManagerConfig, p *corev1.Pod, secret *core.Secret) corev1.Container {
+ return corev1.Container{
+ Image: cfg.SidecarImage,
+ // Create a unique name to allow multiple secrets to be mounted.
+ Name: formatGCPInitContainerName(len(p.Spec.InitContainers)),
+ Command: formatGCPSecretAccessCommand(secret),
+ VolumeMounts: []corev1.VolumeMount{
+ {
+ Name: GCPSecretsVolumeName,
+ MountPath: GCPSecretMountPath,
+ },
+ },
+ Resources: cfg.Resources,
+ }
+}
+
+// NewGCPSecretManagerInjector creates a SecretInjector that's able to mount secrets from GCP Secret Manager.
+func NewGCPSecretManagerInjector(cfg config.GCPSecretManagerConfig) GCPSecretManagerInjector {
+ return GCPSecretManagerInjector{
+ cfg: cfg,
+ }
+}

pkg/webhook/gcp_secret_manager_test.go

@@ -0,0 +1,79 @@
+package webhook
+
+import (
+ "context"
+ "testing"
+
+ "github.com/flyteorg/flytepropeller/pkg/webhook/config"
+
+ "github.com/go-test/deep"
+
+ "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+ "github.com/stretchr/testify/assert"
+ corev1 "k8s.io/api/core/v1"
+)
+
+func TestGCPSecretManagerInjector_Inject(t *testing.T) {
+ injector := NewGCPSecretManagerInjector(config.DefaultConfig.GCPSecretManagerConfig)
+ inputSecret := &core.Secret{
+ Group: "TestSecret",
+ GroupVersion: "2",
+ }
+
+ expected := &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Volumes: []corev1.Volume{
+ {
+ Name: "gcp-secret-vol",
+ VolumeSource: corev1.VolumeSource{
+ EmptyDir: &corev1.EmptyDirVolumeSource{
+ Medium: corev1.StorageMediumMemory,
+ },
+ },
+ },
+ },
+
+ InitContainers: []corev1.Container{
+ {
+ Name: "gcp-pull-secret-0",
+ Image: "gcr.io/google.com/cloudsdktool/cloud-sdk:alpine",
+ Command: []string{
+ "sh",
+ "-c",
+ "gcloud secrets versions access 2 --secret=TestSecret --out-file=/etc/flyte/secrets/testsecret/2 && chmod +rX /etc/flyte/secrets/testsecret /etc/flyte/secrets/testsecret/2",
+ },
+ Env: []corev1.EnvVar{
+ {
+ Name: "FLYTE_SECRETS_DEFAULT_DIR",
+ Value: "/etc/flyte/secrets",
+ },
+ {
+ Name: "FLYTE_SECRETS_FILE_PREFIX",
+ Value: "",
+ },
+ },
+ VolumeMounts: []corev1.VolumeMount{
+ {
+ Name: "gcp-secret-vol",
+ MountPath: "/etc/flyte/secrets",
+ },
+ },
+ Resources: config.DefaultConfig.GCPSecretManagerConfig.Resources,
+ },
+ },
+ Containers: []corev1.Container{},
+ },
+ }
+
+ p := &corev1.Pod{
+ Spec: corev1.PodSpec{
+ Containers: []corev1.Container{},
+ },
+ }
+ actualP, injected, err := injector.Inject(context.Background(), inputSecret, p)
+ assert.NoError(t, err)
+ assert.True(t, injected)
+ if diff := deep.Equal(actualP, expected); diff != nil {
+ assert.Fail(t, "actual != expected", "Diff: %v", diff)
+ }
+}

pkg/webhook/secrets.go

@@ -74,6 +74,7 @@ func NewSecretsMutator(cfg *config.Config, _ promutils.Scope) *SecretsMutator {
  NewGlobalSecrets(secretmanager.NewFileEnvSecretManager(secretmanager.GetConfig())),
  NewK8sSecretsInjector(),
  NewAWSSecretManagerInjector(cfg.AWSSecretManagerConfig),
+ NewGCPSecretManagerInjector(cfg.GCPSecretManagerConfig),
  NewVaultSecretManagerInjector(cfg.VaultSecretManagerConfig),
  },
  }