Switch to distroless consul-dataplane image with Envoy 1.24 (#1676)

* Do not run consul-dataplane in a shell * Unquote arguments to consul-dataplane * Avoid substituting environment variables into consul-dataplane command/args * Bump consul-server-connection-manager to 0.1.0 * fix: Cleanup consul servers in server-acl-init test * Default to hashicorppreview/consul-dataplane:1.0-dev until consul-dataplane beta4 is out Co-authored-by: Paul Glass <pglass@hashicorp.com>
hashicorp · Nov 10, 2022 · aa5a8dc · aa5a8dc
1 parent 612277d
commit aa5a8dc
Show file tree

Hide file tree

Showing 14 changed files with 415 additions and 367 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## UNRELEASED
 
-BREAKING_CHANGES:
+BREAKING CHANGES:
 * CLI:
   * Change default behavior of `consul-k8s install` to perform the installation when no answer is provided to the prompt. [[GH-1673](https://github.com/hashicorp/consul-k8s/pull/1673)]
 * Helm:
@@ -21,7 +21,7 @@ BREAKING_CHANGES:
   * Require `meshGateway.enabled` when peering is enabled. [[GH-1683](https://github.com/hashicorp/consul-k8s/pull/1683)]
 
 FEATURES:
-* Consul-dataplane:
+* Consul Dataplane:
   * Support merged metrics with consul-dataplane. [[GH-1635](https://github.com/hashicorp/consul-k8s/pull/1635)]
   * Support transparent proxying when using consul-dataplane. [[GH-1625](https://github.com/hashicorp/consul-k8s/pull/1478),[GH-1632](https://github.com/hashicorp/consul-k8s/pull/1632)]
   * Enable sync-catalog to only talk to Consul servers. [[GH-1659](https://github.com/hashicorp/consul-k8s/pull/1659)]
@@ -44,6 +44,8 @@ IMPROVEMENTS:
   * API Gateway: Create PodSecurityPolicy for controller when `global.enablePodSecurityPolicies=true`. [[GH-1656](https://github.com/hashicorp/consul-k8s/pull/1656)]
   * API Gateway: Create PodSecurityPolicy and allow controller to bind it to ServiceAccounts that it creates for Gateway Deployments when `global.enablePodSecurityPolicies=true`. [[GH-1672](https://github.com/hashicorp/consul-k8s/pull/1672)]
   * Deploy `expose-servers` service only when Admin Partitions(ENT) is enabled. [[GH-1683](https://github.com/hashicorp/consul-k8s/pull/1683)]
+  * Use a distroless image for `consul-dataplane`. [[GH-1676](https://github.com/hashicorp/consul-k8s/pull/1676)]
+  * The Envoy version is now 1.24.0 for `consul-dataplane`. [[GH-1676](https://github.com/hashicorp/consul-k8s/pull/1676)]
 
 BUG FIXES:
 * Peering

diff --git a/acceptance/tests/metrics/metrics_test.go b/acceptance/tests/metrics/metrics_test.go
@@ -100,9 +100,8 @@ func TestAppMetrics(t *testing.T) {
 	ns := ctx.KubectlOptions(t).Namespace
 
 	helmValues := map[string]string{
-		"global.datacenter":      "dc1",
-		"global.metrics.enabled": "true",
-
+		"global.datacenter":                          "dc1",
+		"global.metrics.enabled":                     "true",
 		"connectInject.enabled":                      "true",
 		"connectInject.metrics.defaultEnableMerging": "true",
 	}

diff --git a/charts/consul/templates/ingress-gateways-deployment.yaml b/charts/consul/templates/ingress-gateways-deployment.yaml
@@ -227,6 +227,9 @@ spec:
         resources: {{ toYaml (default $defaults.resources .resources) | nindent 10 }}
         {{- end }}
         volumeMounts:
+        - name: consul-service
+          mountPath: /consul/service
+          readOnly: true
         {{- if and $root.Values.global.tls.enabled (not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots)) }}
         - name: consul-ca-cert
           mountPath: /consul/tls/ca
@@ -245,65 +248,67 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
+        - name: DP_ENVOY_READY_BIND_ADDRESS
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: DP_CREDENTIAL_LOGIN_META1
+          value: pod=$(NAMESPACE)/$(POD_NAME)
+        - name: DP_CREDENTIAL_LOGIN_META2
+          value: component=ingress-gateway
         - name: DP_SERVICE_NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
         command:
-        - /bin/sh
-        - -ec
-        - |
-          consul-dataplane \
-            -envoy-ready-bind-address=$POD_IP \
-            -envoy-ready-bind-port=21000 \
-            {{- if $root.Values.externalServers.enabled }}
-            -addresses={{ $root.Values.externalServers.hosts | first | quote }} \
-            {{- else }}
-            -addresses="{{ template "consul.fullname" $root }}-server.{{ $root.Release.Namespace }}.svc" \
-            {{- end }}
-            {{- if $root.Values.externalServers.enabled }}
-            -grpc-port={{ $root.Values.externalServers.grpcPort }} \
-            {{- else }}
-            -grpc-port=8502 \
-            {{- end }}
-            -proxy-service-id=$POD_NAME \
-            -service-node-name=$DP_SERVICE_NODE_NAME \
-            {{- if $root.Values.global.enableConsulNamespaces }}
-            -service-namespace={{ (default $defaults.consulNamespace .consulNamespace) }} \
-            {{- end }}
-            {{- if and $root.Values.global.tls.enabled }}
-            {{- if (not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots)) }}
-            -ca-certs=/consul/tls/ca/tls.crt \
-            {{- end }}
-            {{- if and $root.Values.externalServers.enabled $root.Values.externalServers.tlsServerName }}
-            -tls-server-name={{ $root.Values.externalServers.tlsServerName }} \
-            {{- else if $root.Values.global.cloud.enabled }}
-            -tls-server-name=server.{{ $root.Values.global.datacenter}}.{{ $root.Values.global.domain}} \
-            {{- end }}
-            {{- else }}
-            -tls-disabled \
-            {{- end }}
-            {{- if $root.Values.global.acls.manageSystemACLs }}
-            -credential-type=login \
-            -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token \
-            -login-meta=component=ingress-gateway \
-            -login-meta=pod=${NAMESPACE}/${POD_NAME} \
-            -login-auth-method={{ template "consul.fullname" $root }}-k8s-component-auth-method \
-            {{- if $root.Values.global.adminPartitions.enabled }}
-            -login-partition={{ $root.Values.global.adminPartitions.name }} \
-            {{- end }}
-            {{- end }}
-            {{- if $root.Values.global.adminPartitions.enabled }}
-            -service-partition={{ $root.Values.global.adminPartitions.name }} \
-            {{- end }}
-            -log-level={{ default $root.Values.global.logLevel }} \
-            -log-json={{ $root.Values.global.logJSON }} \
-            {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
-            -telemetry-prom-scrape-path="/metrics"
-            {{- end }}
-            {{- if and $root.Values.externalServers.enabled $root.Values.externalServers.skipServerWatch }}
-            -server-watch-disabled=true
-            {{- end }}
+        - consul-dataplane
+        args:
+        - -envoy-ready-bind-port=21000
+        {{- if $root.Values.externalServers.enabled }}
+        - -addresses={{ $root.Values.externalServers.hosts | first }}
+        {{- else }}
+        - -addresses={{ template "consul.fullname" $root }}-server.{{ $root.Release.Namespace }}.svc
+        {{- end }}
+        {{- if $root.Values.externalServers.enabled }}
+        - -grpc-port={{ $root.Values.externalServers.grpcPort }}
+        {{- else }}
+        - -grpc-port=8502
+        {{- end }}
+        - -proxy-service-id-path=/consul/service/proxy-id
+        {{- if $root.Values.global.enableConsulNamespaces }}
+        - -service-namespace={{ (default $defaults.consulNamespace .consulNamespace) }}
+        {{- end }}
+        {{- if and $root.Values.global.tls.enabled }}
+        {{- if (not (and $root.Values.externalServers.enabled $root.Values.externalServers.useSystemRoots)) }}
+        - -ca-certs=/consul/tls/ca/tls.crt
+        {{- end }}
+        {{- if and $root.Values.externalServers.enabled $root.Values.externalServers.tlsServerName }}
+        - -tls-server-name={{ $root.Values.externalServers.tlsServerName }}
+        {{- else if $root.Values.global.cloud.enabled }}
+        - -tls-server-name=server.{{ $root.Values.global.datacenter}}.{{ $root.Values.global.domain}}
+        {{- end }}
+        {{- else }}
+        - -tls-disabled
+        {{- end }}
+        {{- if $root.Values.global.acls.manageSystemACLs }}
+        - -credential-type=login
+        - -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token
+        - -login-auth-method={{ template "consul.fullname" $root }}-k8s-component-auth-method
+        {{- if $root.Values.global.adminPartitions.enabled }}
+        - -login-partition={{ $root.Values.global.adminPartitions.name }}
+        {{- end }}
+        {{- end }}
+        {{- if $root.Values.global.adminPartitions.enabled }}
+        - -service-partition={{ $root.Values.global.adminPartitions.name }}
+        {{- end }}
+        - -log-level={{ default $root.Values.global.logLevel }}
+        - -log-json={{ $root.Values.global.logJSON }}
+        {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+        - -telemetry-prom-scrape-path=/metrics
+        {{- end }}
+        {{- if and $root.Values.externalServers.enabled $root.Values.externalServers.skipServerWatch }}
+        - -server-watch-disabled=true
+        {{- end }}
         livenessProbe:
           tcpSocket:
             port: 21000

diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml
@@ -195,65 +195,64 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+        - name: DP_CREDENTIAL_LOGIN_META1
+          value: pod=$(NAMESPACE)/$(POD_NAME)
+        - name: DP_CREDENTIAL_LOGIN_META2
+          value: component=mesh-gateway
         - name: DP_SERVICE_NODE_NAME
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
         command:
-        - /bin/sh
-        - -ec
-        - |
-          consul-dataplane \
-            {{- if .Values.externalServers.enabled }}
-            -addresses={{ .Values.externalServers.hosts | first | quote }} \
-            {{- else }}
-            -addresses="{{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc" \
-            {{- end }}
-            {{- if .Values.externalServers.enabled }}
-            -grpc-port={{ .Values.externalServers.grpcPort }} \
-            {{- else }}
-            -grpc-port=8502 \
-            {{- end }}
-            -proxy-service-id=$POD_NAME \
-            -service-node-name=$DP_SERVICE_NODE_NAME \
-            {{- if .Values.global.tls.enabled }}
-            {{- if (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) }}
-            -ca-certs=/consul/tls/ca/tls.crt \
-            {{- end }}
-            {{- if and .Values.externalServers.enabled .Values.externalServers.tlsServerName }}
-            -tls-server-name={{.Values.externalServers.tlsServerName }} \
-            {{- else if .Values.global.cloud.enabled }}
-            -tls-server-name=server.{{ .Values.global.datacenter}}.{{ .Values.global.domain}} \
-            {{- end }}
-            {{- else }}
-            -tls-disabled \
-            {{- end }}
-            {{- if .Values.global.acls.manageSystemACLs }}
-            -credential-type=login \
-            -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token \
-            -login-meta=component=mesh-gateway \
-            -login-meta=pod=${NAMESPACE}/${POD_NAME} \
-            {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
-            -login-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \
-            -login-datacenter={{ .Values.global.federation.primaryDatacenter }} \
-            {{- else }}
-            -login-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \
-            {{- end }}
-            {{- if .Values.global.adminPartitions.enabled }}
-            -login-partition={{ .Values.global.adminPartitions.name }} \
-            {{- end }}
-            {{- end }}
-            {{- if .Values.global.adminPartitions.enabled }}
-            -service-partition={{ .Values.global.adminPartitions.name }} \
-            {{- end }}
-            -log-level={{ default .Values.global.logLevel }} \
-            -log-json={{ .Values.global.logJSON }} \
-            {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }}
-            -telemetry-prom-scrape-path="/metrics"
-            {{- end }}
-            {{- if and .Values.externalServers.enabled .Values.externalServers.skipServerWatch }}
-            -server-watch-disabled=true
-            {{- end }}
+        - consul-dataplane
+        args:
+        {{- if .Values.externalServers.enabled }}
+        - -addresses={{ .Values.externalServers.hosts | first }}
+        {{- else }}
+        - -addresses={{ template "consul.fullname" . }}-server.{{ .Release.Namespace }}.svc
+        {{- end }}
+        {{- if .Values.externalServers.enabled }}
+        - -grpc-port={{ .Values.externalServers.grpcPort }}
+        {{- else }}
+        - -grpc-port=8502
+        {{- end }}
+        - -proxy-service-id-path=/consul/service/proxy-id
+        {{- if .Values.global.tls.enabled }}
+        {{- if (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots)) }}
+        - -ca-certs=/consul/tls/ca/tls.crt
+        {{- end }}
+        {{- if and .Values.externalServers.enabled .Values.externalServers.tlsServerName }}
+        - -tls-server-name={{.Values.externalServers.tlsServerName }}
+        {{- else if .Values.global.cloud.enabled }}
+        - -tls-server-name=server.{{ .Values.global.datacenter}}.{{ .Values.global.domain}}
+        {{- end }}
+        {{- else }}
+        - -tls-disabled
+        {{- end }}
+        {{- if .Values.global.acls.manageSystemACLs }}
+        - -credential-type=login
+        - -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token
+        {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
+        - -login-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }}
+        - -login-datacenter={{ .Values.global.federation.primaryDatacenter }}
+        {{- else }}
+        - -login-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method
+        {{- end }}
+        {{- if .Values.global.adminPartitions.enabled }}
+        - -login-partition={{ .Values.global.adminPartitions.name }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.global.adminPartitions.enabled }}
+        - -service-partition={{ .Values.global.adminPartitions.name }}
+        {{- end }}
+        - -log-level={{ default .Values.global.logLevel }}
+        - -log-json={{ .Values.global.logJSON }}
+        {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }}
+        - -telemetry-prom-scrape-path=/metrics
+        {{- end }}
+        {{- if and .Values.externalServers.enabled .Values.externalServers.skipServerWatch }}
+        - -server-watch-disabled=true
+        {{- end }}
         livenessProbe:
           tcpSocket:
             port: {{ .Values.meshGateway.containerPort }}