Refactor mesh-gateway ACL flow (#1085)

* Refactor mesh-gateway ACL flow
hashicorp · Mar 11, 2022 · 4ddea76 · 4ddea76
1 parent eaf7165
commit 4ddea76
Show file tree

Hide file tree

Showing 14 changed files with 355 additions and 144 deletions.
diff --git a/acceptance/tests/vault/vault_wan_fed_test.go b/acceptance/tests/vault/vault_wan_fed_test.go
@@ -200,12 +200,26 @@ func TestVault_WANFederationViaGateways(t *testing.T) {
 	primaryConsulCluster := consul.NewHelmCluster(t, primaryConsulHelmValues, primaryCtx, cfg, consulReleaseName)
 	primaryConsulCluster.Create(t)
 
+	var k8sAuthMethodHost string
+	// When running on kind, the kube API address in kubeconfig will have a localhost address
+	// which will not work from inside the container. That's why we need to use the endpoints address instead
+	// which will point the node IP.
+	if cfg.UseKind {
+		// The Kubernetes AuthMethod host is read from the endpoints for the Kubernetes service.
+		kubernetesEndpoint, err := secondaryCtx.KubernetesClient(t).CoreV1().Endpoints("default").Get(context.Background(), "kubernetes", metav1.GetOptions{})
+		require.NoError(t, err)
+		k8sAuthMethodHost = fmt.Sprintf("%s:%d", kubernetesEndpoint.Subsets[0].Addresses[0].IP, kubernetesEndpoint.Subsets[0].Ports[0].Port)
+	} else {
+		k8sAuthMethodHost = k8s.KubernetesAPIServerHostFromOptions(t, secondaryCtx.KubectlOptions(t))
+	}
+
 	// Get the address of the mesh gateway.
 	primaryMeshGWAddress := meshGatewayAddress(t, cfg, primaryCtx, consulReleaseName)
 	secondaryConsulHelmValues := map[string]string{
 		"global.datacenter": "dc2",
 
 		"global.federation.enabled":            "true",
+		"global.federation.k8sAuthMethodHost":  k8sAuthMethodHost,
 		"global.federation.primaryDatacenter":  "dc1",
 		"global.federation.primaryGateways[0]": primaryMeshGWAddress,
 

diff --git a/charts/consul/templates/api-gateway-controller-deployment.yaml b/charts/consul/templates/api-gateway-controller-deployment.yaml
@@ -205,7 +205,7 @@ spec:
             {{- if .Values.global.adminPartitions.enabled }}
             -partition={{ .Values.global.adminPartitions.name }} \
             {{- end }}
-            -log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
+            -log-level={{ default .Values.global.logLevel .Values.apiGateway.logLevel }} \
             -log-json={{ .Values.global.logJSON }}
         resources:
           requests:

diff --git a/charts/consul/templates/client-snapshot-agent-deployment.yaml b/charts/consul/templates/client-snapshot-agent-deployment.yaml
@@ -229,7 +229,7 @@ spec:
               {{- if .Values.global.adminPartitions.enabled }}
               -partition={{ .Values.global.adminPartitions.name }} \
               {{- end }}
-              -log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
+              -log-level={{ default .Values.global.logLevel }} \
               -log-json={{ .Values.global.logJSON }}
         resources:
           requests:

diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml
@@ -348,7 +348,7 @@ spec:
               {{- if .Values.global.adminPartitions.enabled }}
               -partition={{ .Values.global.adminPartitions.name }} \
               {{- end }}
-              -log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
+              -log-level={{ default .Values.global.logLevel .Values.connectInject.logLevel }} \
               -log-json={{ .Values.global.logJSON }}
         resources:
           requests:

diff --git a/charts/consul/templates/mesh-gateway-clusterrole.yaml b/charts/consul/templates/mesh-gateway-clusterrole.yaml
@@ -19,15 +19,6 @@ rules:
     verbs:
       - use
 {{- end }}
-{{- if .Values.global.acls.manageSystemACLs }}
-  - apiGroups: [""]
-    resources:
-      - secrets
-    resourceNames:
-      - {{ template "consul.fullname" . }}-mesh-gateway-acl-token
-    verbs:
-      - get
-{{- end }}
 {{- if eq .Values.meshGateway.wanAddress.source "Service" }}
   - apiGroups: [""]
     resources:

diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml
@@ -121,36 +121,46 @@ spec:
         {{- if (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }}
         {{- include "consul.getAutoEncryptClientCA" . | nindent 8 }}
         {{- end }}
-        # service-init registers the mesh gateway service.
-        - name: service-init
+        - name: mesh-gateway-init
           image: {{ .Values.global.imageK8S }}
           env:
-            - name: HOST_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.hostIP
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.hostIP
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          {{- if .Values.global.tls.enabled }}
+          - name: CONSUL_CACERT
+            value: /consul/tls/ca/tls.crt
+          {{- end }}
+          - name: CONSUL_HTTP_ADDR
             {{- if .Values.global.tls.enabled }}
-            - name: CONSUL_HTTP_ADDR
-              value: https://$(HOST_IP):8501
-            - name: CONSUL_CACERT
-              value: /consul/tls/ca/tls.crt
+            value: https://$(HOST_IP):8501
             {{- else }}
-            - name: CONSUL_HTTP_ADDR
-              value: http://$(HOST_IP):8500
+            value: http://$(HOST_IP):8500
             {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
             - |
                 {{- if .Values.global.acls.manageSystemACLs }}
                 consul-k8s-control-plane acl-init \
-                  -secret-name="{{ template "consul.fullname" . }}-mesh-gateway-acl-token" \
-                  -k8s-namespace={{ .Release.Namespace }} \
-                  -token-sink-file=/consul/service/acl-token
+                  -component-name=mesh-gateway \
+                  -token-sink-file=/consul/service/acl-token \
+                  {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
+                  -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \
+                  -primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \
+                  {{- else }}
+                  -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \
+                  {{- end }}
+                  {{- if .Values.global.adminPartitions.enabled }}
+                  -partition={{ .Values.global.adminPartitions.name }} \
+                  {{- end }}
+                  -log-level={{ default .Values.global.logLevel }} \
+                  -log-json={{ .Values.global.logJSON }}
                 {{ end }}
 
                 {{- $source := .Values.meshGateway.wanAddress.source }}
@@ -258,6 +268,9 @@ spec:
             {{- end }}
           {{- end }}
           volumeMounts:
+          - mountPath: /consul/service
+            name: consul-service
+            readOnly: true
           - name: consul-bin
             mountPath: /consul-bin
           {{- if .Values.global.tls.enabled }}
@@ -285,12 +298,9 @@ spec:
                   fieldPath: spec.nodeName
             {{- end }}
             {{- if .Values.global.acls.manageSystemACLs }}
-            - name: CONSUL_HTTP_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: "{{ template "consul.fullname" . }}-mesh-gateway-acl-token"
-                  key: "token"
-            {{- end}}
+            - name: CONSUL_HTTP_TOKEN_FILE
+              value: /consul/service/acl-token
+            {{- end }}
             {{- if .Values.global.tls.enabled }}
             - name: CONSUL_HTTP_ADDR
               value: https://$(HOST_IP):8501
@@ -337,7 +347,7 @@ spec:
           lifecycle:
             preStop:
               exec:
-                command: ["/bin/sh", "-ec", "/consul-bin/consul services deregister -id=\"{{ .Values.meshGateway.consulServiceName }}\""]
+                command: ["/bin/sh", "-ec", "/consul-bin/consul services deregister -id=\"{{ .Values.meshGateway.consulServiceName }}\"", "&&", "/bin/sh", "-ec", "/consul-bin/consul logout"]
 
         # consul-sidecar ensures the mesh gateway is always registered with
         # the local Consul agent, even if it loses the initial registration.

diff --git a/charts/consul/templates/server-acl-init-job.yaml b/charts/consul/templates/server-acl-init-job.yaml
@@ -194,7 +194,7 @@ spec:
                 {{- end }}
 
                 {{- if .Values.meshGateway.enabled }}
-                -create-mesh-gateway-token=true \
+                -mesh-gateway=true \
                 {{- end }}
 
                 {{- if .Values.ingressGateways.enabled }}

diff --git a/charts/consul/templates/sync-catalog-deployment.yaml b/charts/consul/templates/sync-catalog-deployment.yaml
@@ -276,7 +276,7 @@ spec:
               {{- if .Values.global.adminPartitions.enabled }}
               -partition={{ .Values.global.adminPartitions.name }} \
               {{- end }}
-              -log-level={{ default .Values.global.logLevel .Values.controller.logLevel }} \
+              -log-level={{ default .Values.global.logLevel .Values.syncCatalog.logLevel }} \
               -log-json={{ .Values.global.logJSON }}
         resources:
           requests:

diff --git a/charts/consul/test/unit/mesh-gateway-clusterrole.bats b/charts/consul/test/unit/mesh-gateway-clusterrole.bats
@@ -32,18 +32,6 @@ load _helpers
   [ "${actual}" = "podsecuritypolicies" ]
 }
 
-@test "meshGateway/ClusterRole: rules for global.acls.manageSystemACLs=true" {
-  cd `chart_dir`
-  local actual=$(helm template \
-      -s templates/mesh-gateway-clusterrole.yaml  \
-      --set 'meshGateway.enabled=true' \
-      --set 'connectInject.enabled=true' \
-      --set 'global.acls.manageSystemACLs=true' \
-      . | tee /dev/stderr |
-      yq -r '.rules[0].resources[0]' | tee /dev/stderr)
-  [ "${actual}" = "secrets" ]
-}
-
 @test "meshGateway/ClusterRole: rules for meshGateway.wanAddress.source=Service" {
   cd `chart_dir`
   local actual=$(helm template \
@@ -83,5 +71,5 @@ load _helpers
       --set 'meshGateway.wanAddress.source=Service' \
       . | tee /dev/stderr |
       yq -r '.rules | length' | tee /dev/stderr)
-  [ "${actual}" = "3" ]
+  [ "${actual}" = "2" ]
 }