Support storing bootstrap token in Vault (#1061)

acceptance/tests/vault/helpers.go

@@ -18,9 +18,15 @@ const (
 path "consul/data/secret/gossip" {
   capabilities = ["read"]
 }`
+
+	bootstrapTokenPolicy = `
+path "consul/data/secret/bootstrap" {
+  capabilities = ["read"]
+}`
+
 	replicationTokenPolicy = `
 path "consul/data/secret/replication" {
-  capabilities = ["read", "update"]
+  capabilities = ["read"]
 }`
 
 	enterpriseLicensePolicy = `
@@ -118,7 +124,13 @@ func configureEnterpriseLicenseVaultSecret(t *testing.T, vaultClient *vapi.Clien
 
 // configureKubernetesAuthRoles configures roles for the Kubernetes auth method
 // that will be used by the test Helm chart installation.
-func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns, authPath, datacenter string, cfg *config.TestConfig) {
+func configureKubernetesAuthRoles(
+	t *testing.T,
+	vaultClient *vapi.Client,
+	consulReleaseName, ns, authPath, datacenter string,
+	cfg *config.TestConfig,
+	isPrimary bool,
+) {
 	consulClientServiceAccountName := fmt.Sprintf("%s-consul-client", consulReleaseName)
 	consulServerServiceAccountName := fmt.Sprintf("%s-consul-server", consulReleaseName)
 	sharedPolicies := "consul-gossip"
@@ -142,10 +154,16 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	_, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-client", authPath), params)
 	require.NoError(t, err)
 
+	// Both primary and secondary datacenters need access to the replication token, but
+	// only the primary needs to be able to read the bootstrap token.
+	policies := fmt.Sprintf(sharedPolicies+",connect-ca-%s,consul-server-%s,consul-replication-token", datacenter, datacenter)
+	if isPrimary {
+		policies += ",consul-bootstrap-token"
+	}
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
 		"bound_service_account_namespaces": ns,
-		"policies":                         fmt.Sprintf(sharedPolicies+",connect-ca-%s,consul-server-%s,consul-replication-token", datacenter, datacenter),
+		"policies":                         policies,
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-server", authPath), params)
@@ -160,6 +178,22 @@ func configureKubernetesAuthRoles(t *testing.T, vaultClient *vapi.Client, consul
 	}
 	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/consul-ca", authPath), params)
 	require.NoError(t, err)
+
+	logger.Log(t, "Creating kubernetes auth role for the server-acl-init job")
+	policies = "consul-replication-token"
+	if isPrimary {
+		policies += ",consul-bootstrap-token"
+	}
+	serverACLInitSAName := fmt.Sprintf("%s-consul-server-acl-init", consulReleaseName)
+	params = map[string]interface{}{
+		"bound_service_account_names":      serverACLInitSAName,
+		"bound_service_account_namespaces": ns,
+		"policies":                         policies,
+		"ttl":                              "24h",
+	}
+
+	_, err = vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/server-acl-init", authPath), params)
+	require.NoError(t, err)
 }
 
 // configurePKICA generates a CA in Vault.
@@ -213,7 +247,7 @@ path %q {
 
 // configureReplicationTokenVaultSecret generates a replication token secret ID,
 // stores it in vault as a secret and configures a policy to access it.
-func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client, consulReleaseName, ns string, authMethodPaths ...string) string {
+func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client) string {
 	// Create the Vault Policy for the replication token.
 	logger.Log(t, "Creating replication token policy")
 	err := vaultClient.Sys().PutPolicy("consul-replication-token", replicationTokenPolicy)
@@ -227,25 +261,36 @@ func configureReplicationTokenVaultSecret(t *testing.T, vaultClient *vapi.Client
 	logger.Log(t, "Creating the replication token secret")
 	params := map[string]interface{}{
 		"data": map[string]interface{}{
-			"replication": token,
+			"token": token,
 		},
 	}
 	_, err = vaultClient.Logical().Write("consul/data/secret/replication", params)
 	require.NoError(t, err)
 
-	logger.Log(t, "Creating kubernetes auth role for the server-acl-init job")
-	serverACLInitSAName := fmt.Sprintf("%s-consul-server-acl-init", consulReleaseName)
-	params = map[string]interface{}{
-		"bound_service_account_names":      serverACLInitSAName,
-		"bound_service_account_namespaces": ns,
-		"policies":                         "consul-replication-token",
-		"ttl":                              "24h",
-	}
+	return token
+}
+
+// configureBootstrapTokenVaultSecret generates the bootstrap token secret ID,
+// stores it in vault as a secret and configures a policy to access it.
+func configureBootstrapTokenVaultSecret(t *testing.T, vaultClient *vapi.Client) string {
+	// Create the Vault Policy for the bootstrap token.
+	logger.Log(t, "Creating bootstrap token policy")
+	err := vaultClient.Sys().PutPolicy("consul-bootstrap-token", bootstrapTokenPolicy)
+	require.NoError(t, err)
 
-	for _, authMethodPath := range authMethodPaths {
-		_, err := vaultClient.Logical().Write(fmt.Sprintf("auth/%s/role/server-acl-init", authMethodPath), params)
-		require.NoError(t, err)
+	// Generate the token secret.
+	token, err := uuid.GenerateUUID()
+	require.NoError(t, err)
+
+	// Create the replication token secret.
+	logger.Log(t, "Creating the bootstrap token secret")
+	params := map[string]interface{}{
+		"data": map[string]interface{}{
+			"token": token,
+		},
 	}
+	_, err = vaultClient.Logical().Write("consul/data/secret/bootstrap", params)
+	require.NoError(t, err)
 
 	return token
 }

acceptance/tests/vault/vault_test.go

@@ -41,7 +41,9 @@ func TestVault(t *testing.T) {
 		configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg)
 	}
 
-	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg)
+	bootstrapToken := configureBootstrapTokenVaultSecret(t, vaultClient)
+
+	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg, true)
 
 	configurePKICA(t, vaultClient)
 	certPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
@@ -57,10 +59,11 @@ func TestVault(t *testing.T) {
 		"connectInject.replicas": "1",
 		"controller.enabled":     "true",
 
-		"global.secretsBackend.vault.enabled":          "true",
-		"global.secretsBackend.vault.consulServerRole": "consul-server",
-		"global.secretsBackend.vault.consulClientRole": "consul-client",
-		"global.secretsBackend.vault.consulCARole":     "consul-ca",
+		"global.secretsBackend.vault.enabled":              "true",
+		"global.secretsBackend.vault.consulServerRole":     "consul-server",
+		"global.secretsBackend.vault.consulClientRole":     "consul-client",
+		"global.secretsBackend.vault.consulCARole":         "consul-ca",
+		"global.secretsBackend.vault.manageSystemACLsRole": "server-acl-init",
 
 		"global.secretsBackend.vault.ca.secretName": vaultCASecret,
 		"global.secretsBackend.vault.ca.secretKey":  "tls.crt",
@@ -69,10 +72,12 @@ func TestVault(t *testing.T) {
 		"global.secretsBackend.vault.connectCA.rootPKIPath":         "connect_root",
 		"global.secretsBackend.vault.connectCA.intermediatePKIPath": "dc1/connect_inter",
 
-		"global.acls.manageSystemACLs":       "true",
-		"global.tls.enabled":                 "true",
-		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
-		"global.gossipEncryption.secretKey":  "gossip",
+		"global.acls.manageSystemACLs":          "true",
+		"global.acls.bootstrapToken.secretName": "consul/data/secret/bootstrap",
+		"global.acls.bootstrapToken.secretKey":  "token",
+		"global.tls.enabled":                    "true",
+		"global.gossipEncryption.secretName":    "consul/data/secret/gossip",
+		"global.gossipEncryption.secretKey":     "gossip",
 
 		"ingressGateways.enabled":               "true",
 		"ingressGateways.defaults.replicas":     "1",
@@ -103,6 +108,7 @@ func TestVault(t *testing.T) {
 
 	// Validate that the gossip encryption key is set correctly.
 	logger.Log(t, "Validating the gossip key has been set correctly.")
+	consulCluster.ACLToken = bootstrapToken
 	consulClient := consulCluster.SetupConsulClient(t, true)
 	keys, err := consulClient.Operator().KeyringList(nil)
 	require.NoError(t, err)

acceptance/tests/vault/vault_wan_fed_test.go

@@ -72,7 +72,7 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		configureEnterpriseLicenseVaultSecret(t, vaultClient, cfg)
 	}
 
-	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg)
+	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes", "dc1", cfg, true)
 
 	// Configure Vault Kubernetes auth method for the secondary datacenter.
 	{
@@ -113,14 +113,15 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		secondaryVaultCluster.ConfigureAuthMethod(t, vaultClient, "kubernetes-dc2", k8sAuthMethodHost, authMethodRBACName, ns)
 	}
 
-	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "dc2", cfg)
+	configureKubernetesAuthRoles(t, vaultClient, consulReleaseName, ns, "kubernetes-dc2", "dc2", cfg, false)
 
 	// Generate a CA and create PKI roles for the primary and secondary Consul servers.
 	configurePKICA(t, vaultClient)
 	primaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc1")
 	secondaryCertPath := configurePKICertificates(t, vaultClient, consulReleaseName, ns, "dc2")
 
-	replicationToken := configureReplicationTokenVaultSecret(t, vaultClient, consulReleaseName, ns, "kubernetes", "kubernetes-dc2")
+	bootstrapToken := configureBootstrapTokenVaultSecret(t, vaultClient)
+	replicationToken := configureReplicationTokenVaultSecret(t, vaultClient)
 
 	// Create the Vault Policy for the Connect CA in both datacenters.
 	createConnectCAPolicy(t, vaultClient, "dc1")
@@ -156,9 +157,11 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 		// ACL config.
 		"global.acls.manageSystemACLs":            "true",
+		"global.acls.bootstrapToken.secretName":   "consul/data/secret/bootstrap",
+		"global.acls.bootstrapToken.secretKey":    "token",
 		"global.acls.createReplicationToken":      "true",
 		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
-		"global.acls.replicationToken.secretKey":  "replication",
+		"global.acls.replicationToken.secretKey":  "token",
 
 		// Mesh config.
 		"connectInject.enabled": "true",
@@ -219,7 +222,7 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 		// ACL config.
 		"global.acls.manageSystemACLs":            "true",
 		"global.acls.replicationToken.secretName": "consul/data/secret/replication",
-		"global.acls.replicationToken.secretKey":  "replication",
+		"global.acls.replicationToken.secretKey":  "token",
 
 		// Mesh config.
 		"connectInject.enabled": "true",
@@ -263,6 +266,7 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 
 	// Verify federation between servers.
 	logger.Log(t, "verifying federation was successful")
+	primaryConsulCluster.ACLToken = bootstrapToken
 	primaryClient := primaryConsulCluster.SetupConsulClient(t, true)
 	secondaryConsulCluster.ACLToken = replicationToken
 	secondaryClient := secondaryConsulCluster.SetupConsulClient(t, true)

charts/consul/templates/_helpers.tpl

@@ -73,6 +73,13 @@ as well as the global.name setting.
           {{ "{{" }}- end -{{ "}}" }}
 {{- end -}}
 
+{{- define "consul.vaultBootstrapTokenConfigTemplate" -}}
+|
+          {{ "{{" }}- with secret "{{ .Values.global.acls.bootstrapToken.secretName }}" -{{ "}}" }}
+          acl { tokens { initial_management = "{{ "{{" }}- {{ printf ".Data.data.%s" .Values.global.acls.bootstrapToken.secretKey }} -{{ "}}" }}" }}
+          {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file

charts/consul/templates/server-acl-init-job.yaml

@@ -4,8 +4,10 @@
 {{- if and .Values.global.acls.createReplicationToken (not .Values.global.acls.manageSystemACLs) }}{{ fail "if global.acls.createReplicationToken is true, global.acls.manageSystemACLs must be true" }}{{ end -}}
 {{- if .Values.global.bootstrapACLs }}{{ fail "global.bootstrapACLs was removed, use global.acls.manageSystemACLs instead" }}{{ end -}}
 {{- if .Values.global.acls.manageSystemACLs }}
-{{- if and .Values.global.secretsBackend.vault.enabled .Values.global.acls.replicationToken.secretName (not .Values.global.secretsBackend.vault.manageSystemACLsRole) }}{{ fail "global.secretsBackend.vault.manageSystemACLsRole must be set if global.secretsBackend.vault.enabled is true and global.acls.replicationToken is provided" }}{{ end -}}
+{{- if or (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.acls.bootstrapToken.secretKey))  (and .Values.global.acls.bootstrapToken.secretKey (not .Values.global.acls.bootstrapToken.secretName))}}{{ fail "both global.acls.bootstrapToken.secretKey and global.acls.bootstrapToken.secretName must be set if one of them is provided" }}{{ end -}}
 {{- if or (and .Values.global.acls.replicationToken.secretName (not .Values.global.acls.replicationToken.secretKey))  (and .Values.global.acls.replicationToken.secretKey (not .Values.global.acls.replicationToken.secretName))}}{{ fail "both global.acls.replicationToken.secretKey and global.acls.replicationToken.secretName must be set if one of them is provided" }}{{ end -}}
+{{- if (and .Values.global.secretsBackend.vault.enabled (and (not .Values.global.acls.bootstrapToken.secretName) (not .Values.global.acls.replicationToken.secretName ))) }}{{fail "global.acls.bootstrapToken or global.acls.replicationToken must be provided when global.secretsBackend.vault.enabled and global.acls.manageSystemACLs are true" }}{{ end -}}
+{{- if (and .Values.global.secretsBackend.vault.enabled (not .Values.global.secretsBackend.vault.manageSystemACLsRole)) }}{{fail "global.secretsBackend.vault.manageSystemACLsRole is required when global.secretsBackend.vault.enabled and global.acls.manageSystemACLs are true" }}{{ end -}}
   {{- /* We don't render this job when server.updatePartition > 0 because that
     means a server rollout is in progress and this job won't complete unless
     the rollout is finished (which won't happen until the partition is 0).
@@ -36,11 +38,19 @@ spec:
         component: server-acl-init
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
-        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        {{- if .Values.global.secretsBackend.vault.enabled }}
         "vault.hashicorp.com/agent-pre-populate-only": "true"
         "vault.hashicorp.com/agent-inject": "true"
+        {{- if .Values.global.acls.bootstrapToken.secretName }}
+        {{- with .Values.global.acls.bootstrapToken }}
+        "vault.hashicorp.com/agent-inject-secret-bootstrap-token": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-template-bootstrap-token": {{ template "consul.vaultSecretTemplate" . }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.global.tls.enabled }}
         "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
         "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- end }}
         {{- if .Values.global.secretsBackend.vault.manageSystemACLsRole }}
         "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.manageSystemACLsRole }}
         {{- else if .Values.global.tls.enabled }}
@@ -61,7 +71,7 @@ spec:
     spec:
       restartPolicy: Never
       serviceAccountName: {{ template "consul.fullname" . }}-server-acl-init
-      {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey)) }}
+      {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName .Values.global.acls.bootstrapToken.secretName) }}
       volumes:
         {{- if and .Values.global.tls.enabled (not .Values.global.secretsBackend.vault.enabled) }}
         - name: consul-ca-cert
@@ -75,7 +85,7 @@ spec:
               - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }}
                 path: tls.crt
         {{- end }}
-        {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+        {{- if (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.secretsBackend.vault.enabled)) }}
         - name: bootstrap-token
           secret:
             secretName: {{ .Values.global.acls.bootstrapToken.secretName }}
@@ -99,14 +109,14 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-          {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey)) }}
+          {{- if (or .Values.global.tls.enabled .Values.global.acls.replicationToken.secretName .Values.global.acls.bootstrapToken.secretName) }}
           volumeMounts:
             {{- if and .Values.global.tls.enabled (not .Values.global.secretsBackend.vault.enabled) }}
             - name: consul-ca-cert
               mountPath: /consul/tls/ca
               readOnly: true
             {{- end }}
-            {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+            {{- if (and .Values.global.acls.bootstrapToken.secretName (not .Values.global.secretsBackend.vault.enabled)) }}
             - name: bootstrap-token
               mountPath: /consul/acl/tokens
               readOnly: true
@@ -127,6 +137,7 @@ spec:
                 -log-json={{ .Values.global.logJSON }} \
                 -resource-prefix=${CONSUL_FULLNAME} \
                 -k8s-namespace={{ .Release.Namespace }} \
+                -set-server-tokens={{ $serverEnabled }} \
 
                 {{- if .Values.externalServers.enabled }}
                 {{- if and .Values.externalServers.enabled (not .Values.externalServers.hosts) }}{{ fail "externalServers.hosts must be set if externalServers.enabled is true" }}{{ end -}}
@@ -240,9 +251,14 @@ spec:
                 -federation=true \
                 {{- end }}
 
-                {{- if (and .Values.global.acls.bootstrapToken.secretName .Values.global.acls.bootstrapToken.secretKey) }}
+                {{- if .Values.global.acls.bootstrapToken.secretName }}
+                {{- if .Values.global.secretsBackend.vault.enabled }}
+                -bootstrap-token-file=/vault/secrets/bootstrap-token \
+                {{- else }}
                 -bootstrap-token-file=/consul/acl/tokens/bootstrap-token \
-                {{- else if .Values.global.acls.replicationToken.secretName }}
+                {{- end }}
+                {{- end }}
+                {{- if .Values.global.acls.replicationToken.secretName }}
                 {{- if .Values.global.secretsBackend.vault.enabled }}
                 -acl-replication-token-file=/vault/secrets/replication-token \
                 {{- else }}