SOPS: Add support for HashiCorp Vault token-based authentication

.github/workflows/e2e.yaml

@@ -51,6 +51,8 @@ jobs:
         uses: fluxcd/pkg/actions/kubectl@main
         with:
           version: 1.21.2
+      - name: Setup SOPS
+        uses: fluxcd/pkg/actions/sops@main
       - name: Run controller tests
         run: make test
       - name: Check if working tree is dirty

controllers/kustomization_decryptor.go

@@ -45,13 +45,19 @@ import (
 	intkeyservice "github.com/fluxcd/kustomize-controller/internal/sops/keyservice"
 )
 
-const DecryptionProviderSOPS = "sops"
+const (
+	// DecryptionProviderSOPS is the SOPS provider name
+	DecryptionProviderSOPS = "sops"
+	// DecryptionVaultTokenFileName is the name of the file containing the Vault token
+	DecryptionVaultTokenFileName = "sops.vault-token"
+)
 
 type KustomizeDecryptor struct {
 	client.Client
 	kustomization kustomizev1.Kustomization
 	homeDir       string
 	ageIdentities []string
+	vaultToken    string
 }
 
 func NewDecryptor(kubeClient client.Client,
@@ -147,24 +153,34 @@ func (kd *KustomizeDecryptor) ImportKeys(ctx context.Context) error {
 		defer os.RemoveAll(tmpDir)
 
 		var ageIdentities []string
-		for name, file := range secret.Data {
+		var vaultToken string
+		for name, value := range secret.Data {
 			switch filepath.Ext(name) {
 			case ".asc":
 				keyPath, err := securejoin.SecureJoin(tmpDir, name)
 				if err != nil {
 					return err
 				}
-				if err := os.WriteFile(keyPath, file, os.ModePerm); err != nil {
+				if err := os.WriteFile(keyPath, value, os.ModePerm); err != nil {
 					return fmt.Errorf("unable to write key to storage: %w", err)
 				}
 				if err := kd.gpgImport(keyPath); err != nil {
 					return err
 				}
 			case ".agekey":
-				ageIdentities = append(ageIdentities, string(file))
+				ageIdentities = append(ageIdentities, string(value))
+			case ".vault-token":
+				// Make sure we have the absolute file name
+				if name == DecryptionVaultTokenFileName {
+					token := string(value)
+					token = strings.Trim(strings.TrimSpace(token), "\n")
+					vaultToken = token
+				}
 			}
 		}
+
 		kd.ageIdentities = ageIdentities
+		kd.vaultToken = vaultToken
 	}
 
 	return nil
@@ -256,7 +272,7 @@ func (kd KustomizeDecryptor) DataWithFormat(data []byte, inputFormat, outputForm
 
 	metadataKey, err := tree.Metadata.GetDataKeyWithKeyServices(
 		[]keyservice.KeyServiceClient{
-			intkeyservice.NewLocalClient(intkeyservice.NewServer(false, kd.homeDir, kd.ageIdentities)),
+			intkeyservice.NewLocalClient(intkeyservice.NewServer(false, kd.homeDir, kd.vaultToken, kd.ageIdentities)),
 		},
 	)
 	if err != nil {

controllers/kustomization_decryptor_test.go

@@ -20,12 +20,14 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"testing"
 	"time"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
 	"github.com/fluxcd/pkg/apis/meta"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+	"github.com/hashicorp/vault/api"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,9 +37,29 @@ import (
 
 func TestKustomizationReconciler_Decryptor(t *testing.T) {
 	g := NewWithT(t)
+
+	cli, err := api.NewClient(api.DefaultConfig())
+	g.Expect(err).NotTo(HaveOccurred(), "failed to create vault client")
+
+	// create a master key on the vault transit engine
+	path, data := "sops/keys/firstkey", map[string]interface{}{"type": "rsa-4096"}
+	_, err = cli.Logical().Write(path, data)
+	g.Expect(err).NotTo(HaveOccurred(), "failed to write key")
+
+	// encrypt the testdata vault secret
+	cmd := exec.Command("sops", "--hc-vault-transit", cli.Address()+"/v1/sops/keys/firstkey", "--encrypt", "--encrypted-regex", "^(data|stringData)$", "--in-place", "./testdata/sops/secret.vault.yaml")
+	err = cmd.Run()
+	g.Expect(err).NotTo(HaveOccurred(), "failed to encrypt file")
+
+	// defer the testdata vault secret decryption, to leave a clean testdata vault secret
+	defer func() {
+		cmd := exec.Command("sops", "--hc-vault-transit", cli.Address()+"/v1/sops/keys/firstkey", "--decrypt", "--encrypted-regex", "^(data|stringData)$", "--in-place", "./testdata/sops/secret.vault.yaml")
+		err = cmd.Run()
+	}()
+
 	id := "sops-" + randStringRunes(5)
 
-	err := createNamespace(id)
+	err = createNamespace(id)
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
 
 	err = createKubeConfigSecret(id)
@@ -83,8 +105,9 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 			Namespace: sopsSecretKey.Namespace,
 		},
 		StringData: map[string]string{
-			"pgp.asc":    string(pgpKey),
-			"age.agekey": string(ageKey),
+			"pgp.asc":          string(pgpKey),
+			"age.agekey":       string(ageKey),
+			"sops.vault-token": "secret",
 		},
 	}
 
@@ -171,6 +194,10 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 		var encodedSecret corev1.Secret
 		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-month", Namespace: id}, &encodedSecret)).To(Succeed())
 		g.Expect(string(encodedSecret.Data["month.yaml"])).To(Equal("month: May\n"))
+
+		var hcvaultSecret corev1.Secret
+		g.Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: "sops-hcvault", Namespace: id}, &hcvaultSecret)).To(Succeed())
+		g.Expect(string(hcvaultSecret.Data["secret"])).To(Equal("my-sops-vault-secret\n"))
 	})
 
 	t.Run("does not emit change events for identical secrets", func(t *testing.T) {

controllers/suite_test.go

@@ -37,6 +37,8 @@ import (
 	"github.com/fluxcd/pkg/runtime/testenv"
 	"github.com/fluxcd/pkg/testserver"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+	"github.com/hashicorp/vault/api"
+	"github.com/ory/dockertest"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -58,6 +60,8 @@ const (
 	reconciliationInterval = time.Second * 5
 )
 
+const vaultVersion = "1.2.2"
+
 var (
 	k8sClient    client.Client
 	testEnv      *testenv.Environment
@@ -116,6 +120,12 @@ func runInContext(registerControllers func(*testenv.Environment), run func() err
 		panic(fmt.Sprintf("Failed to create k8s client: %v", err))
 	}
 
+	// Create a vault test instance
+	pool, resource, err := createVaultTestInstance()
+	defer func() {
+		pool.Purge(resource)
+	}()
+
 	runErr := run()
 
 	if debugMode {
@@ -361,3 +371,45 @@ func createArtifact(artifactServer *testserver.ArtifactServer, fixture, path str
 
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
+
+func createVaultTestInstance() (*dockertest.Pool, *dockertest.Resource, error) {
+	// uses a sensible default on windows (tcp/http) and linux/osx (socket)
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		return nil, nil, fmt.Errorf("Could not connect to docker: %s", err)
+	}
+
+	// pulls an image, creates a container based on it and runs it
+	resource, err := pool.Run("vault", vaultVersion, []string{"VAULT_DEV_ROOT_TOKEN_ID=secret"})
+	if err != nil {
+		return nil, nil, fmt.Errorf("Could not start resource: %s", err)
+	}
+
+	os.Setenv("VAULT_ADDR", fmt.Sprintf("http://127.0.0.1:%v", resource.GetPort("8200/tcp")))
+	os.Setenv("VAULT_TOKEN", "secret")
+	// exponential backoff-retry, because the application in the container might not be ready to accept connections yet
+	if err := pool.Retry(func() error {
+		cli, err := api.NewClient(api.DefaultConfig())
+		if err != nil {
+			return fmt.Errorf("Cannot create Vault Client: %w", err)
+		}
+		status, err := cli.Sys().InitStatus()
+		if err != nil {
+			return err
+		}
+		if status != true {
+			return fmt.Errorf("Vault not ready yet")
+		}
+		if err := cli.Sys().Mount("sops", &api.MountInput{
+			Type: "transit",
+		}); err != nil {
+			return fmt.Errorf("Cannot create Vault Transit Engine: %w", err)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, nil, fmt.Errorf("Could not connect to docker: %w", err)
+	}
+
+	return pool, resource, nil
+}

controllers/testdata/sops/secret.vault.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+    secret: bXktc29wcy12YXVsdC1zZWNyZXQK
+kind: Secret
+metadata:
+    name: sops-hcvault
+    namespace: default
+type: Opaque

docs/spec/v1beta2/kustomization.md

@@ -1001,6 +1001,54 @@ spec:
       name: sops-age
 ```
 
+### HashiCorp Vault
+
+Export the `VAULT_ADDR`  and `VAULT_TOKEN` enviromnet variables to your shell,
+then use `sops` to encrypt a kubernetes secret (see [HashiCorp Vault](https://www.vaultproject.io/docs/secrets/transit)
+for more details on enabling the transit backend and [sops](https://github.com/mozilla/sops#encrypting-using-hashicorp-vault)).
+
+Then use `sops` to encrypt a kubernetes secret:
+
+```console
+$ export VAULT_ADDR=https://vault.example.com:8200
+$ export VAULT_TOKEN=my-token
+$ sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/my-encryption-key --encrypt \
+--encrypted-regex '^(data|stringData)$' --in-place my-secret.yaml
+```
+
+Commit and push the encrypted file to Git.
+
+> **Note** that you should encrypt only the `data` section, encrypting the Kubernetes
+> secret metadata, kind or apiVersion is not supported by kustomize-controller.
+
+Create a secret in the `default` namespace with the vault token,
+the key name must be `sops.vault-token` to be detected as a vault token:
+
+```sh
+echo $VAULT_TOKEN |
+kubectl -n default create secret generic sops-hcvault \
+--from-file=sops.vault-token=/dev/stdin
+```
+
+Configure decryption by referring the private key secret:
+
+```yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: my-secrets
+  namespace: default
+spec:
+  interval: 5m
+  path: "./"
+  sourceRef:
+    kind: GitRepository
+    name: my-secrets
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-hcvault
+```
 ### Kustomize secretGenerator
 
 SOPS encrypted data can be stored as a base64 encoded Secret,