Merge pull request gocardless#225 from gocardless/vault-files

cmd/theatre-envconsul/main.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	execpkg "os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 	"syscall"
 
@@ -65,7 +66,7 @@ func main() {
 	defer cancel()
 
 	if err := mainError(ctx, command); err != nil {
-		logger.Error(err, "exiting with error", "error", err)
+		logger.Error(err, "exiting with error")
 		os.Exit(1)
 	}
 }
@@ -133,6 +134,38 @@ func mainError(ctx context.Context, command string) (err error) {
 			}
 		}
 
+		var filePaths = environment{}
+
+		// Rewrite 'vault-file:' prefixed env vars to 'vault:' prefixed env vars. Store the
+		// paths to which they should be written to in filePaths. When no path is
+		// provided, use "" as a placeholder.
+		//
+		// For reference, the expected formats are 'vault-file:tls-key/2021010100' and
+		// 'vault-file:ssh-key/2021010100:/home/user/.ssh/id_rsa'
+		for key, value := range env {
+			if strings.HasPrefix(value, "vault-file:") {
+				trimmed := strings.TrimSpace(
+					strings.TrimPrefix(value, "vault-file:"),
+				)
+				if len(trimmed) == 0 {
+					return fmt.Errorf("empty vault-file env var: %v", value)
+				}
+
+				split := strings.SplitN(trimmed, ":", 2)
+
+				// determine if we define a path at which to place the file. For SplitN,
+				// N=2 so we only have two cases
+				switch len(split) {
+				case 2: // path and key
+					filePaths[key] = split[1]
+					env[key] = fmt.Sprintf("vault:%s", split[0])
+				case 1: // just key
+					filePaths[key] = ""
+					env[key] = fmt.Sprintf("vault:%s", trimmed)
+				}
+			}
+		}
+
 		var secretEnv = environment{}
 
 		// For all the environment values that look like they should be vault references, we
@@ -184,7 +217,11 @@ func mainError(ctx context.Context, command string) (err error) {
 
 		output, err := execpkg.CommandContext(ctx, envconsulBinaryPath, envconsulArgs...).Output()
 		if err != nil {
-			return errors.Wrap(err, "failed to get envconsul environment variables")
+			if ee, ok := err.(*execpkg.ExitError); ok {
+				output = ee.Stderr
+			}
+
+			return errors.Wrapf(err, "failed to get envconsul environment variables: %s", output)
 		}
 
 		envMap := map[string]string{}
@@ -193,6 +230,40 @@ func mainError(ctx context.Context, command string) (err error) {
 			return errors.Wrap(err, "failed to decode envconsul environment variables")
 		}
 
+		// For every file reference in filePaths, write the value resolved by envconsul to
+		// the path in filePaths. Returns the path of the written file in the env var that
+		// requested it.
+		for key, path := range filePaths {
+			if path == "" {
+				// generate file path prefixed by key
+				tempFilePath, err := ioutil.TempFile("", fmt.Sprintf("%s-*", key))
+				if err != nil {
+					return errors.Wrap(err, fmt.Sprintf("failed to write temporary file for key %s", key))
+				}
+
+				path = tempFilePath.Name()
+			}
+			// ensure the path structure is available
+			err := os.MkdirAll(filepath.Dir(path), 0600)
+			if err != nil {
+				return fmt.Errorf("failed to ensure path structure is available: %s", err.Error())
+			}
+
+			logger.Info(
+				"creating vault secret file",
+				"event", "envconsul_secret_file.create",
+				"path", path,
+			)
+			// write file with value of envMap[key]
+			if err := ioutil.WriteFile(path, []byte(envMap[key]), 0600); err != nil {
+				return errors.Wrap(err,
+					fmt.Sprintf("failed to write file with key %s to path %s", key, path))
+			}
+
+			// update the env with the location of the file we've written
+			envMap[key] = path
+		}
+
 		// Update the environment variables based on updated environment variables
 		for key, value := range envMap {
 			os.Setenv(key, value)

cmd/vault-manager/acceptance/acceptance.go

@@ -273,6 +273,36 @@ spec:
         - env
 `
 
+const annotatedNonRootPodYAMLFiles = `
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: read-a-secret-
+  namespace: staging # provisioned by the acceptance kustomize overlay
+  annotations:
+    envconsul-injector.vault.crd.gocardless.com/configs: app
+spec:
+  serviceAccountName: secret-reader
+  restartPolicy: Never
+  containers:
+    - name: app
+      image: theatre:latest
+      imagePullPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1001
+      env:
+        - name: VAULT_FILE_RESOLVED_KEY
+          value: vault-file:jimmy:/tmp/jimmy
+        - name: VAULT_TMP_FILE_RESOLVED_KEY
+          value: vault-file:jimmy
+      command:
+        - bash
+        - -c
+        - 'echo -n "file:" && cat $(echo $VAULT_FILE_RESOLVED_KEY) && echo -n " tmp:" && cat $(echo $VAULT_TMP_FILE_RESOLVED_KEY)'
+        `
+
 func (r *Runner) Run(logger kitlog.Logger, config *rest.Config) {
 	var (
 		clientset      *kubernetes.Clientset
@@ -287,7 +317,7 @@ func (r *Runner) Run(logger kitlog.Logger, config *rest.Config) {
 
 	// Create pod from fixture, verify that pod runs successfully and resolves the secret
 	// environment variable
-	expectResolvesEnvVariables := func() {
+	expectResolvesEnvVariables := func(expects func(buffer bytes.Buffer)) {
 		ctx := context.Background()
 		decoder := scheme.Codecs.UniversalDeserializer()
 		obj, _, err := decoder.Decode([]byte(podFixtureYAML), nil, nil)
@@ -324,25 +354,51 @@ func (r *Runner) Run(logger kitlog.Logger, config *rest.Config) {
 		_, err = io.Copy(&buffer, logs)
 
 		Expect(err).NotTo(HaveOccurred())
+		expects(buffer)
+	}
+
+	expectsFunc := func(buffer bytes.Buffer) {
 		Expect(buffer.String()).To(
 			ContainSubstring(fmt.Sprintf("VAULT_RESOLVED_KEY=%s", SentinelSecretValue)),
 		)
+		return
+	}
+
+	expectsFuncFiles := func(buffer bytes.Buffer) {
+		Expect(buffer.String()).To(
+			ContainSubstring(fmt.Sprintf("file:%s", SentinelSecretValue)),
+		)
+		Expect(buffer.String()).To(
+			ContainSubstring(fmt.Sprintf("tmp:%s", SentinelSecretValue)),
+		)
+		return
 	}
 
 	Describe("theatre-envconsul", func() {
 		BeforeEach(func() { podFixtureYAML = rawPodYAML })
 
-		It("Resolves env variables into the pod command", expectResolvesEnvVariables)
+		It("Resolves env variables into the pod command", func() { expectResolvesEnvVariables(expectsFunc) })
 
 		Context("As configured by the vault envconsul-injector webhook", func() {
 			BeforeEach(func() { podFixtureYAML = annotatedPodYAML })
 
-			It("Resolves env variables into the pod command", expectResolvesEnvVariables)
+			It("Resolves env variables into the pod command", func() { expectResolvesEnvVariables(expectsFunc) })
 
 			Context("With a non-root user", func() {
 				BeforeEach(func() { podFixtureYAML = annotatedNonRootPodYAML })
 
-				It("Resolves env variables into the pod command", expectResolvesEnvVariables)
+				It("Resolves env variables into the pod command", func() { expectResolvesEnvVariables(expectsFunc) })
+			})
+		})
+	})
+
+	Describe("theatre-envconsul files", func() {
+		Context("As configured by the vault envconsul-injector webhook", func() {
+
+			Context("With a non-root user", func() {
+				BeforeEach(func() { podFixtureYAML = annotatedNonRootPodYAMLFiles })
+
+				It("Resolves env variables into the pod command", func() { expectResolvesEnvVariables(expectsFuncFiles) })
 			})
 		})
 	})