Standardize handling of credentials (#90)

Makefile

@@ -13,7 +13,6 @@ LD_FLAGS = " \
 ENV ?= kind
 IMG ?= etok
 TAG ?= $(VERSION)-$(RANDOM_SUFFIX)
-DEPLOY_FLAGS ?= --secret-file $(GOOGLE_APPLICATION_CREDENTIALS)
 KUBECTX=""
 
 # Override vars if ENV=gke
@@ -46,7 +45,7 @@ local: image push
 
 # Same as above - image still needs to be built and pushed/loaded
 .PHONY: deploy
-deploy: image push
+deploy: image push deploy-operator-secret
 	$(BUILD_BIN) install --context $(KUBECTX) --local --image $(IMG):$(TAG) $(DEPLOY_FLAGS) \
 		--backup-provider=gcs --gcs-bucket=$(BACKUP_BUCKET)
 
@@ -61,9 +60,30 @@ crds: build
 	$(BUILD_BIN) install --context $(KUBECTX) --local --crds-only
 
 .PHONY: undeploy
-undeploy: build
+undeploy: build delete-operator-secret
 	$(BUILD_BIN) install --local --dry-run | $(KUBECTL) delete -f - --wait --ignore-not-found=true
 
+
+# Deploy a secret containing GCP svc acc key, on kind, for the operator to use
+.PHONY: deploy-operator-secret
+deploy-operator-secret: delete-operator-secret create-operator-namespace
+ifeq ($(ENV),kind)
+	$(KUBECTL) --namespace=etok create secret generic etok --from-file=GOOGLE_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS)
+endif
+
+.PHONY: delete-operator-secret
+delete-operator-secret:
+ifeq ($(ENV),kind)
+	$(KUBECTL) --namespace=etok delete secret etok --ignore-not-found
+endif
+
+# Create operator namespace, ignore already exists errors
+.PHONY: create-operator-namespace
+create-operator-namespace:
+ifeq ($(ENV),kind)
+	$(KUBECTL) create namespace etok 2>/dev/null || true
+endif
+
 .PHONY: e2e
 e2e: image push deploy
 	go test -v ./test/e2e -failfast -context $(KUBECTX)

README.md

@@ -134,7 +134,7 @@ etok install --backup-provider=gcs --gcs-bucket=backups-bucket
 
 Note: only GCS is supported at present.
 
-Be sure to provide the appropriate credentials to the operator at install time. Either provide the path to a file containing a GCP service account key via the `--secret-file` flag, or setup workload identity (see below). The service account needs the following permissions on the bucket:
+Be sure to provide the appropriate credentials of a GCP service account to the operator at install time. Either [create a secret containing credentials](#credentials), or [setup workload identity](#workload-identity). The service account needs the following permissions on the bucket:
 
 ```
 storage.buckets.get
@@ -147,9 +147,9 @@ To opt a workspace out of automatic backup and restore, pass the `--ephemeral` f
 
 ## Credentials
 
-Etok looks for credentials in a secret named `etok`. If found, the credentials contained within are made available to terraform as environment variables.
+Etok looks for credentials in a secret named `etok` in the relevant namespace. The credentials contained within are made available as environment variables.
 
-For instance to set credentials for the [GCP provider](https://www.terraform.io/docs/providers/google/guides/provider_reference.html#full-reference):
+For instance to set credentials for the [Terraform GCP provider](https://www.terraform.io/docs/providers/google/guides/provider_reference.html#full-reference), or for making backups to GCS:
 
 ```
 kubectl create secret generic etok --from-file=GOOGLE_CREDENTIALS=[path to service account key]

cmd/install/deployment.go

@@ -121,25 +121,13 @@ func deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment
 	deployment.Spec.Template.Labels = selector
 
 	if c.withSecret {
-		deployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, corev1.Volume{
-			Name: "secrets",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: "etok",
+		deployment.Spec.Template.Spec.Containers[0].EnvFrom = append(deployment.Spec.Template.Spec.Containers[0].EnvFrom, corev1.EnvFromSource{
+			SecretRef: &corev1.SecretEnvSource{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: "etok",
 				},
 			},
 		})
-
-		deployment.Spec.Template.Spec.Containers[0].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
-			Name:      "secrets",
-			MountPath: "/secrets/secret-file.json",
-			SubPath:   "secret-file.json",
-		})
-
-		deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
-			Name:  "GOOGLE_APPLICATION_CREDENTIALS",
-			Value: "/secrets/secret-file.json",
-		})
 	}
 
 	return deployment

cmd/install/deployment_test.go

@@ -29,20 +29,10 @@ func TestDeployment(t *testing.T) {
 			namespace: "default",
 			opts:      []podTemplateOption{WithSecret(true)},
 			assertions: func(deploy *appsv1.Deployment) {
-				assert.Contains(t, deploy.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
-					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
-					Value: "/secrets/secret-file.json",
-				})
-				assert.Contains(t, deploy.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
-					Name:      "secrets",
-					MountPath: "/secrets/secret-file.json",
-					SubPath:   "secret-file.json",
-				})
-				assert.Contains(t, deploy.Spec.Template.Spec.Volumes, corev1.Volume{
-					Name: "secrets",
-					VolumeSource: corev1.VolumeSource{
-						Secret: &corev1.SecretVolumeSource{
-							SecretName: "etok",
+				assert.Contains(t, deploy.Spec.Template.Spec.Containers[0].EnvFrom, corev1.EnvFromSource{
+					SecretRef: &corev1.SecretEnvSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: "etok",
 						},
 					},
 				})

cmd/install/install.go

@@ -12,8 +12,10 @@ import (
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 
@@ -68,8 +70,6 @@ type installOptions struct {
 	image       string
 	kubeContext string
 
-	// Path on local fs containing GCP service account key
-	secretFile string
 	// Annotations to add to the service account resource
 	serviceAccountAnnotations map[string]string
 
@@ -128,7 +128,6 @@ func InstallCmd(f *cmdutil.Factory) (*cobra.Command, *installOptions) {
 	cmd.Flags().BoolVar(&o.wait, "wait", true, "Toggle waiting for deployment to be ready")
 	cmd.Flags().DurationVar(&o.timeout, "timeout", 60*time.Second, "Timeout for waiting for deployment to be ready")
 
-	cmd.Flags().StringVar(&o.secretFile, "secret-file", "", "Path on local filesystem to key file")
 	cmd.Flags().StringToStringVar(&o.serviceAccountAnnotations, "sa-annotations", map[string]string{}, "Annotations to add to the etok ServiceAccount. Add iam.gke.io/gcp-service-account=[GSA_NAME]@[PROJECT_NAME].iam.gserviceaccount.com for workload identity")
 	cmd.Flags().BoolVar(&o.crdsOnly, "crds-only", o.crdsOnly, "Only generate CRD resources. Useful for updating CRDs for an existing Etok install.")
 
@@ -180,7 +179,15 @@ func (o *installOptions) install(ctx context.Context) error {
 		resources = append(resources, namespace(o.namespace))
 		resources = append(resources, serviceAccount(o.namespace, o.serviceAccountAnnotations))
 
-		secretPresent := o.secretFile != ""
+		// Determine if a secret named 'etok' is present
+		var secretPresent bool
+		err := o.RuntimeClient.Get(ctx, types.NamespacedName{Namespace: "etok", Name: "etok"}, &corev1.Secret{})
+		if err != nil && !kerrors.IsNotFound(err) {
+			return fmt.Errorf("unable to check for secret: %w", err)
+		}
+		if err == nil {
+			secretPresent = true
+		}
 
 		// Deploy options
 		dopts := []podTemplateOption{}
@@ -192,15 +199,6 @@ func (o *installOptions) install(ctx context.Context) error {
 
 		deploy = deployment(o.namespace, dopts...)
 		resources = append(resources, deploy)
-
-		if o.secretFile != "" {
-			key, err := ioutil.ReadFile(o.secretFile)
-			if err != nil {
-				return err
-			}
-
-			resources = append(resources, secret(o.namespace, key))
-		}
 	}
 
 	// Set labels

cmd/install/install_test.go

@@ -25,6 +25,7 @@ import (
 	cmdutil "github.com/leg100/etok/cmd/util"
 	etokclient "github.com/leg100/etok/pkg/client"
 	"github.com/leg100/etok/pkg/scheme"
+	"github.com/leg100/etok/pkg/testobj"
 	"github.com/leg100/etok/pkg/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -72,17 +73,36 @@ func TestInstall(t *testing.T) {
 			name: "fresh install with custom image",
 			args: []string{"install", "--wait=false", "--image", "bugsbunny:v123"},
 			assertions: func(t *testutil.T, client runtimeclient.Client) {
-				var d = deploy()
-				client.Get(context.Background(), runtimeclient.ObjectKeyFromObject(d), d)
+				var d appsv1.Deployment
+				client.Get(context.Background(), types.NamespacedName{Namespace: defaultNamespace, Name: "etok"}, &d)
+
 				assert.Equal(t, "bugsbunny:v123", d.Spec.Template.Spec.Containers[0].Image)
 			},
 		},
+		{
+			name: "fresh install with secret found",
+			args: []string{"install", "--wait=false"},
+			objs: []runtimeclient.Object{testobj.Secret("etok", "etok")},
+			assertions: func(t *testutil.T, client runtimeclient.Client) {
+				var d appsv1.Deployment
+				client.Get(context.Background(), types.NamespacedName{Namespace: defaultNamespace, Name: "etok"}, &d)
+
+				assert.Contains(t, d.Spec.Template.Spec.Containers[0].EnvFrom, corev1.EnvFromSource{
+					SecretRef: &corev1.SecretEnvSource{
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: "etok",
+						},
+					},
+				})
+			},
+		},
 		{
 			name: "fresh install with backups enabled",
 			args: []string{"install", "--wait=false", "--backup-provider=gcs", "--gcs-bucket=backups-bucket"},
 			assertions: func(t *testutil.T, client runtimeclient.Client) {
-				var d = deploy()
-				client.Get(context.Background(), runtimeclient.ObjectKeyFromObject(d), d)
+				var d appsv1.Deployment
+				client.Get(context.Background(), types.NamespacedName{Namespace: defaultNamespace, Name: "etok"}, &d)
+
 				assert.Contains(t, d.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "ETOK_BACKUP_PROVIDER", Value: "gcs"})
 				assert.Contains(t, d.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "ETOK_GCS_BUCKET", Value: "backups-bucket"})
 			},
@@ -119,10 +139,6 @@ func TestInstall(t *testing.T) {
 			cmd.SetOut(buf)
 			cmd.SetArgs(tt.args)
 
-			// Set path to secret file
-			secretTmpDir := t.NewTempDir().Write("secret.txt", []byte("secret-sauce"))
-			opts.secretFile = secretTmpDir.Path("secret.txt")
-
 			// Mock a remote web server from which YAML files will be retrieved
 			mockWebServer(t)
 
@@ -149,8 +165,7 @@ func TestInstall(t *testing.T) {
 				assert.NoError(t, client.Get(context.Background(), runtimeclient.ObjectKeyFromObject(res), res))
 			}
 
-			// assert non-CRD resources are present unless only CRDs are
-			// requested
+			// assert non-CRD resources are present
 			if !opts.crdsOnly {
 				for _, res := range wantedResources() {
 					assert.NoError(t, client.Get(context.Background(), runtimeclient.ObjectKeyFromObject(res), res))
@@ -210,6 +225,9 @@ func TestInstallDryRun(t *testing.T) {
 
 		out := new(bytes.Buffer)
 		opts := &installOptions{
+			Client: &etokclient.Client{
+				RuntimeClient: fake.NewFakeClientWithScheme(scheme.Scheme),
+			},
 			Factory: &cmdutil.Factory{
 				IOStreams: cmdutil.IOStreams{Out: out},
 			},
@@ -241,7 +259,6 @@ func wantedCRDs() (resources []runtimeclient.Object) {
 func wantedResources() (resources []runtimeclient.Object) {
 	resources = append(resources, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "etok"}})
 	resources = append(resources, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: "etok", Name: "etok"}})
-	resources = append(resources, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "etok", Name: "etok"}})
 	resources = append(resources, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "etok"}})
 	resources = append(resources, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "etok-user"}})
 	resources = append(resources, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "etok-admin"}})

cmd/install/resources.go

@@ -89,21 +89,3 @@ func adminClusterRoleBinding() *rbacv1.ClusterRoleBinding {
 		},
 	}
 }
-
-func secret(namespace string, key []byte) *corev1.Secret {
-	secret := &corev1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Secret",
-			APIVersion: corev1.SchemeGroupVersion.String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "etok",
-			Namespace: namespace,
-		},
-		Data: map[string][]byte{
-			"secret-file.json": key,
-		},
-	}
-
-	return secret
-}

cmd/manager/manager.go

@@ -3,6 +3,8 @@ package manager
 import (
 	"flag"
 	"fmt"
+	"io/ioutil"
+	"os"
 	"runtime"
 
 	"k8s.io/klog/v2"
@@ -76,6 +78,17 @@ func ManagerCmd(f *cmdutil.Factory) *cobra.Command {
 
 			klog.V(0).Info("Runner image: " + o.Image)
 
+			// Convert GOOGLE_CREDENTIALS=<key> to
+			// GOOGLE_APPLICATION_CREDENTIALS=<file-path-containing-key>
+			if gcreds := os.Getenv("GOOGLE_CREDENTIALS"); gcreds != "" {
+				if err := ioutil.WriteFile("/google_application_credentials.json", []byte(gcreds), 0400); err != nil {
+					return fmt.Errorf("unable to write google credentials to disk: %w", err)
+				}
+				if err := os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "/google_application_credentials.json"); err != nil {
+					return fmt.Errorf("unable to create environment variable GOOGLE_APPLICATION_CREDENTIALS: %w", err)
+				}
+			}
+
 			var backupProvider backup.Provider
 			if o.backupProviderName != "" {
 				switch o.backupProviderName {