Server TLS bootstrapping (#881)

* Support Vault server running with TLS (#874)
* Change vault cluster in acceptance tests to only run with TLS. All tests will run against vault with TLS because that is the use case we think will be the most valuable for users to test
* Support adding Vault CA as a secret to pods that will be using vault agent. We need to add two annotations to pods:
      * vault.hashicorp.com/agent-extra-secret with the value of the vault CA secret name. The secret will be mounted to vault agent at /vault/custom path. See docs here
      * vault.hashicorp.com/ca-cert - with the path of the ca file inside the vault agent container. This should be /vault/custom/<secret key>
* Most pods will only need those annotations. The server pods also need the Vault CA secret to be mounted as a volume because it needs the CA to be on the file system for the vault connect CA provider.

* add terminating and ingress gateways TLS support (#894)
* Support TLS with vault for the server-acl-init job (#889)
* Support TLS with Vault for the sync catalog deployment (#890)
* Support server TLS with vault for the client snapshot agent deployment (#891)

Co-authored-by: Iryna Shustava <iryna@hashicorp.com>
Co-authored-by: Luke Kysow <1034429+lkysow@users.noreply.github.com>

acceptance/framework/vault/vault_cluster.go

@@ -131,18 +131,20 @@ func (v *VaultCluster) bootstrap(t *testing.T, ctx environment.TestContext) {
 		Type:   "kv-v2",
 		Config: vapi.MountConfigInput{},
 	})
-	if err != nil {
-		t.Fatal("unable to mount kv-v2 secrets engine", "err", err)
-	}
-	// TODO: add the PKI Secrets Engine when we have a need for it.
+	require.NoError(t, err)
+
+	// Enable the PKI Secrets engine.
+	err = v.vaultClient.Sys().Mount("pki", &vapi.MountInput{
+		Type:   "pki",
+		Config: vapi.MountConfigInput{},
+	})
+	require.NoError(t, err)
 
 	// Enable Kube Auth.
 	err = v.vaultClient.Sys().EnableAuthWithOptions("kubernetes", &vapi.EnableAuthOptions{
 		Type: "kubernetes",
 	})
-	if err != nil {
-		t.Fatal("unable to enable kube auth", "err", err)
-	}
+	require.NoError(t, err)
 
 	v.logger.Logf(t, "updating vault kube auth config")
 
@@ -230,6 +232,7 @@ func defaultHelmValues(releaseName string) map[string]string {
 		"server.standalone.enabled":                "true",
 		"server.standalone.config":                 serverConfig,
 		"injector.enabled":                         "true",
+		"ui.enabled":                               "true",
 	}
 }
 

acceptance/tests/vault/vault_test.go

@@ -43,13 +43,22 @@ path "/connect_inter/*" {
   capabilities = [ "create", "read", "update", "delete", "list" ]
 }
 `
+	serverTLSPolicy = `
+path "pki/issue/consul-server" {
+  capabilities = ["create", "update"]
+}`
+	caPolicy = `
+path "pki/cert/ca" {
+  capabilities = ["read"]
+}`
 )
 
 // TestVault installs Vault, bootstraps it with secrets, policies, and Kube Auth Method.
 // It then configures Consul to use vault as the backend and checks that it works.
 func TestVault(t *testing.T) {
 	cfg := suite.Config()
 	ctx := suite.Environment().DefaultContext(t)
+	ns := ctx.KubectlOptions(t).Namespace
 
 	consulReleaseName := helpers.RandomName()
 	vaultReleaseName := helpers.RandomName()
@@ -68,6 +77,7 @@ func TestVault(t *testing.T) {
 	err := vaultClient.Sys().PutPolicy("consul-gossip", gossipPolicy)
 	require.NoError(t, err)
 
+	// Create the Vault Policy for the connect-ca.
 	err = vaultClient.Sys().PutPolicy("connect-ca", connectCAPolicy)
 	require.NoError(t, err)
 
@@ -80,7 +90,7 @@ func TestVault(t *testing.T) {
 	logger.Log(t, "Creating the consul-server and consul-client roles")
 	params := map[string]interface{}{
 		"bound_service_account_names":      consulClientServiceAccountName,
-		"bound_service_account_namespaces": "default",
+		"bound_service_account_namespaces": ns,
 		"policies":                         "consul-gossip",
 		"ttl":                              "24h",
 	}
@@ -89,13 +99,24 @@ func TestVault(t *testing.T) {
 
 	params = map[string]interface{}{
 		"bound_service_account_names":      consulServerServiceAccountName,
-		"bound_service_account_namespaces": "default",
-		"policies":                         "consul-gossip,connect-ca",
+		"bound_service_account_namespaces": ns,
+		"policies":                         "consul-gossip,connect-ca,consul-server",
 		"ttl":                              "24h",
 	}
 	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-server", params)
 	require.NoError(t, err)
 
+	// Create the CA role that all components will use to fetch the Server CA certs.
+	params = map[string]interface{}{
+		"bound_service_account_names":      "*",
+		"bound_service_account_namespaces": ns,
+		"policies":                         "consul-ca",
+		"ttl":                              "24h",
+	}
+	_, err = vaultClient.Logical().Write("auth/kubernetes/role/consul-ca", params)
+	require.NoError(t, err)
+
+	// Generate the gossip secret.
 	gossipKey, err := generateGossipSecret()
 	require.NoError(t, err)
 
@@ -110,21 +131,56 @@ func TestVault(t *testing.T) {
 	require.NoError(t, err)
 
 	vaultCASecret := vault.CASecretName(vaultReleaseName)
+
+	// Bootstrap TLS by creating the CA infrastructure required for Consul server TLS and also create the `consul-server` PKI role.
+	// Using https://learn.hashicorp.com/tutorials/consul/vault-pki-consul-secure-tls.
+	// Generate the root CA.
+	params = map[string]interface{}{
+		"common_name": "dc1.consul",
+		"ttl":         "24h",
+	}
+	_, err = vaultClient.Logical().Write("pki/root/generate/internal", params)
+	require.NoError(t, err)
+
+	// Create the Vault PKI Role.
+	name := consulReleaseName + "-consul"
+	allowedDomains := fmt.Sprintf("dc1.consul,%s-server,%s-server.%s,%s-server.%s.svc", name, name, ns, name, ns)
+	params = map[string]interface{}{
+		"allowed_domains":    allowedDomains,
+		"allow_bare_domains": "true",
+		"allow_localhost":    "true",
+		"allow_subdomains":   "true",
+		"generate_lease":     "true",
+		"max_ttl":            "1h",
+	}
+	_, err = vaultClient.Logical().Write("pki/roles/consul-server", params)
+	require.NoError(t, err)
+
+	// Create the server and ca policies
+	err = vaultClient.Sys().PutPolicy("consul-server", serverTLSPolicy)
+	require.NoError(t, err)
+	err = vaultClient.Sys().PutPolicy("consul-ca", caPolicy)
+	require.NoError(t, err)
+
 	consulHelmValues := map[string]string{
+		// TODO: Update the global image once 1.11 is GA.
 		"global.image": "docker.mirror.hashicorp.services/hashicorpdev/consul:latest",
 
 		"server.enabled":              "true",
 		"server.replicas":             "1",
 		"server.extraVolumes[0].type": "secret",
 		"server.extraVolumes[0].name": vaultCASecret,
 		"server.extraVolumes[0].load": "false",
+		"global.datacenter":           "dc1",
 
-		"connectInject.enabled": "true",
-		"controller.enabled":    "true",
+		"connectInject.enabled":  "true",
+		"connectInject.replicas": "1",
+		"controller.enabled":     "true",
 
 		"global.secretsBackend.vault.enabled":          "true",
 		"global.secretsBackend.vault.consulServerRole": "consul-server",
 		"global.secretsBackend.vault.consulClientRole": "consul-client",
+		"global.secretsBackend.vault.consulCARole":     "consul-ca",
 
 		"global.secretsBackend.vault.ca.secretName": vaultCASecret,
 		"global.secretsBackend.vault.ca.secretKey":  "tls.crt",
@@ -137,6 +193,24 @@ func TestVault(t *testing.T) {
 		"global.tls.enabled":                 "true",
 		"global.gossipEncryption.secretName": "consul/data/secret/gossip",
 		"global.gossipEncryption.secretKey":  "gossip",
+
+		"ingressGateways.enabled":               "true",
+		"ingressGateways.defaults.replicas":     "1",
+		"terminatingGateways.enabled":           "true",
+		"terminatingGateways.defaults.replicas": "1",
+
+		"server.serverCert.secretName": "pki/issue/consul-server",
+		"global.tls.caCert.secretName": "pki/cert/ca",
+		"global.tls.httpsOnly":         "false",
+		"global.tls.enableAutoEncrypt": "true",
+
+		// For sync catalog, it is sufficient to check that the deployment is running and ready
+		// because we only care that get-auto-encrypt-client-ca init container was able
+		// to talk to the Consul server using the CA from Vault. For this reason,
+		// we don't need any services to be synced in either direction.
+		"syncCatalog.enabled":  "true",
+		"syncCatalog.toConsul": "false",
+		"syncCatalog.toK8S":    "false",
 	}
 	logger.Log(t, "Installing Consul")
 	consulCluster := consul.NewHelmCluster(t, consulHelmValues, ctx, cfg, consulReleaseName)

charts/consul/templates/_helpers.tpl

@@ -21,6 +21,36 @@ as well as the global.name setting.
             {{ "{{" }}- {{ printf ".Data.data.%s" .secretKey }} -{{ "}}" }}
             {{ "{{" }}- end -{{ "}}" }}
 {{- end -}}
+
+{{- define "consul.serverTLSCATemplate" -}}
+ |
+            {{ "{{" }}- with secret "{{ .Values.global.tls.caCert.secretName }}" -{{ "}}" }}
+            {{ "{{" }}- .Data.certificate -{{ "}}" }}
+            {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
+{{- define "consul.serverTLSCertTemplate" -}}
+ |
+            {{ "{{" }}- with secret "{{ .Values.server.serverCert.secretName }}" "{{ printf "common_name=server.%s.%s" .Values.global.datacenter .Values.global.domain }}"
+            "ttl=1h" "alt_names={{ include "consul.serverTLSAltNames" . }}" "ip_sans=127.0.0.1" -{{ "}}" }}
+            {{ "{{" }}- .Data.certificate -{{ "}}" }}
+            {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
+{{- define "consul.serverTLSKeyTemplate" -}}
+ |
+            {{ "{{" }}- with secret "{{ .Values.server.serverCert.secretName }}" "{{ printf "common_name=server.%s.%s" .Values.global.datacenter .Values.global.domain }}"
+            "ttl=1h" "alt_names={{ include "consul.serverTLSAltNames" . }}" "ip_sans=127.0.0.1" -{{ "}}" }}
+            {{ "{{" }}- .Data.private_key -{{ "}}" }}
+            {{ "{{" }}- end -{{ "}}" }}
+{{- end -}}
+
+{{- define "consul.serverTLSAltNames" -}}
+{{- $name := include "consul.fullname" . -}}
+{{- $ns := .Release.Namespace -}}
+{{ printf "localhost,%s-server,*.%s-server,*.%s-server.%s,*.%s-server.%s.svc,*.server.%s.%s" $name $name $name $ns $name $ns (.Values.global.datacenter ) (.Values.global.domain) }}
+{{- end -}}
+
 {{/*
 Sets up the extra-from-values config file passed to consul and then uses sed to do any necessary
 substitution for HOST_IP/POD_IP/HOSTNAME. Useful for dogstats telemetry. The output file
@@ -108,13 +138,19 @@ This template is for an init container.
         {{- else }}
         -server-addr={{ template "consul.fullname" . }}-server \
         -server-port=8501 \
+        {{- if .Values.global.secretsBackend.vault.enabled }}
+        -ca-file=/vault/secrets/serverca.crt
+        {{- else }}
         -ca-file=/consul/tls/ca/tls.crt
         {{- end }}
+        {{- end }}
   volumeMounts:
     {{- if not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) }}
+    {{- if not .Values.global.secretsBackend.vault.enabled }}
     - name: consul-ca-cert
       mountPath: /consul/tls/ca
     {{- end }}
+    {{- end }}
     - name: consul-auto-encrypt-ca-cert
       mountPath: /consul/tls/client/ca
   resources:

charts/consul/templates/client-daemonset.yaml

@@ -3,6 +3,9 @@
 {{- $serverEnabled := (or (and (ne (.Values.server.enabled | toString) "-") .Values.server.enabled) (and (eq (.Values.server.enabled | toString) "-") .Values.global.enabled)) -}}
 {{- if (and .Values.global.adminPartitions.enabled $serverEnabled (ne .Values.global.adminPartitions.name "default"))}}{{ fail "global.adminPartitions.name has to be \"default\" in the server cluster" }}{{ end -}}
 {{- if (and (not .Values.global.secretsBackend.vault.consulClientRole) .Values.global.secretsBackend.vault.enabled) }}{{ fail "global.secretsBackend.vault.consulClientRole must be provided if global.secretsBackend.vault.enabled=true." }}{{ end -}}
+{{- if (and (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) (not .Values.global.tls.caCert.secretName)) }}{{ fail "global.tls.caCert.secretName must be provided if global.tls.enabled=true and global.secretsBackend.vault.enabled=true." }}{{ end -}}
+{{- if (and (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) (not .Values.global.tls.enableAutoEncrypt)) }}{{ fail "global.tls.enableAutoEncrypt must be true if global.secretsBackend.vault.enabled=true and global.tls.enabled=true" }}{{ end -}}
+{{- if (and (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) (not .Values.global.secretsBackend.vault.consulCARole)) }}{{ fail "global.secretsBackend.vault.consulCARole must be provided if global.secretsBackend.vault.enabled=true and global.tls.enabled=true" }}{{ end -}}
 # DaemonSet to run the Consul clients on every node.
 apiVersion: apps/v1
 kind: DaemonSet
@@ -47,10 +50,14 @@ spec:
         {{- end }}
         {{- if .Values.global.gossipEncryption.secretName }}
         {{- with .Values.global.gossipEncryption }}
-        "vault.hashicorp.com/agent-inject-secret-gossip.txt": "{{ .secretName }}"
+        "vault.hashicorp.com/agent-inject-secret-gossip.txt": {{ .secretName }}
         "vault.hashicorp.com/agent-inject-template-gossip.txt": {{ template "consul.vaultGossipTemplate" . }}
         {{- end }}
         {{- end }}
+        {{- if .Values.global.tls.enabled }}
+        "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
+        "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- end }}
         {{- end }}
         "consul.hashicorp.com/connect-inject": "false"
         "consul.hashicorp.com/config-checksum": {{ include (print $.Template.BasePath "/client-config-configmap.yaml") . | sha256sum }}
@@ -104,6 +111,7 @@ spec:
           configMap:
             name: {{ template "consul.fullname" . }}-client-config
         {{- if .Values.global.tls.enabled }}
+        {{- if not .Values.global.secretsBackend.vault.enabled }}
         - name: consul-ca-cert
           secret:
             {{- if .Values.global.tls.caCert.secretName }}
@@ -132,6 +140,7 @@ spec:
             medium: "Memory"
         {{- end }}
         {{- end }}
+        {{- end }}
         {{- range .Values.client.extraVolumes }}
         - name: userconfig-{{ .name }}
           {{ .type }}:
@@ -235,7 +244,11 @@ spec:
                 {{- end }}
                 -hcl='leave_on_terminate = true' \
                 {{- if .Values.global.tls.enabled }}
+                {{- if .Values.global.secretsBackend.vault.enabled }}
+                -hcl='ca_file = "/vault/secrets/serverca.crt"' \
+                {{- else }}
                 -hcl='ca_file = "/consul/tls/ca/tls.crt"' \
+                {{- end }}
                 {{- if .Values.global.tls.enableAutoEncrypt }}
                 -hcl='auto_encrypt = {tls = true}' \
                 -hcl="auto_encrypt = {ip_san = [\"$HOST_IP\",\"$POD_IP\"]}" \
@@ -304,6 +317,7 @@ spec:
             - name: config
               mountPath: /consul/config
             {{- if .Values.global.tls.enabled }}
+            {{- if not .Values.global.secretsBackend.vault.enabled }}
             - name: consul-ca-cert
               mountPath: /consul/tls/ca
               readOnly: true
@@ -313,6 +327,7 @@ spec:
               readOnly: true
             {{- end }}
             {{- end }}
+            {{- end }}
             {{- range .Values.client.extraVolumes }}
             - name: userconfig-{{ .name }}
               readOnly: true
@@ -446,6 +461,7 @@ spec:
             mv {{ .Values.global.datacenter }}-client-{{ .Values.global.domain }}-0.pem tls.crt
             mv {{ .Values.global.datacenter }}-client-{{ .Values.global.domain }}-0-key.pem tls.key
         volumeMounts:
+          {{- if not .Values.global.secretsBackend.vault.enabled }}
           - name: consul-client-cert
             mountPath: /consul/tls/client
           - name: consul-ca-cert
@@ -454,6 +470,7 @@ spec:
           - name: consul-ca-key
             mountPath: /consul/tls/ca/key
             readOnly: true
+          {{- end }}
         resources:
           requests:
             memory: "50Mi"

charts/consul/templates/client-snapshot-agent-deployment.yaml

@@ -27,6 +27,17 @@ spec:
         component: client-snapshot-agent
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
+        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
+        "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
+        "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
+        "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
+        "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
+        {{- end }}
+        {{- end }}
     spec:
       {{- if .Values.client.tolerations }}
       tolerations:

charts/consul/templates/connect-inject-deployment.yaml

@@ -36,6 +36,17 @@ spec:
         component: connect-injector
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
+        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
+        "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
+        "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
+        "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
+        "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
+        {{- end }}
+        {{- end }}
     spec:
       serviceAccountName: {{ template "consul.fullname" . }}-connect-injector-webhook-svc-account
       containers:

charts/consul/templates/controller-deployment.yaml

@@ -30,6 +30,17 @@ spec:
         component: controller
       annotations:
         "consul.hashicorp.com/connect-inject": "false"
+        {{- if (and .Values.global.secretsBackend.vault.enabled .Values.global.tls.enabled) }}
+        "vault.hashicorp.com/agent-init-first": "true"
+        "vault.hashicorp.com/agent-inject": "true"
+        "vault.hashicorp.com/role": {{ .Values.global.secretsBackend.vault.consulCARole }}
+        "vault.hashicorp.com/agent-inject-secret-serverca.crt": {{ .Values.global.tls.caCert.secretName }}
+        "vault.hashicorp.com/agent-inject-template-serverca.crt": {{ template "consul.serverTLSCATemplate" . }}
+        {{- if and .Values.global.secretsBackend.vault.ca.secretName .Values.global.secretsBackend.vault.ca.secretKey }}
+        "vault.hashicorp.com/agent-extra-secret": "{{ .Values.global.secretsBackend.vault.ca.secretName }}"
+        "vault.hashicorp.com/ca-cert": "/vault/custom/{{ .Values.global.secretsBackend.vault.ca.secretKey }}"
+        {{- end }}
+        {{- end }}
     spec:
       {{- if or .Values.global.acls.manageSystemACLs (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }}
       initContainers: