From 38ccc4c8708e6fc3e16a5284b2ecd3cb4d3094f5 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Tue, 30 Aug 2022 19:25:25 -0400 Subject: [PATCH] # This is a combination of 14 commits. # This is the 1st commit message: Add service for terminating-gateways # This is the commit message #2: Add gateway-kind:terminating to deployment # This is the commit message #3: Add registration path for terminating gateways # This is the commit message #4: Add BATS tests # This is the commit message #5: Remove registration from terminating gateways deployment # This is the commit message #6: Set ports AFAIK in service # This is the commit message #7: Begin setting values for endpoints controller # This is the commit message #8: Copy values from deployment to endpoints controller (as comment) # This is the commit message #9: Use connect-init instead of acl-init # This is the commit message #10: Remove guards from term gw service (they will get hit by the deployment) # This is the commit message #11: Range over gateways to produce a service for each deployment # This is the commit message #12: Add test for multiple gateways # This is the commit message #13: Remove the format script # This is the commit message #14: Note which parts of the config have been set --- .../terminating-gateways-deployment.yaml | 56 +++------- .../terminating-gateways-service.yaml | 32 ++++++ .../unit/terminating-gateways-service.bats | 49 ++++++++ .../connect-inject/endpoints_controller.go | 105 +++++++++++++++++- 4 files changed, 199 insertions(+), 43 deletions(-) create mode 100644 charts/consul/templates/terminating-gateways-service.yaml create mode 100644 charts/consul/test/unit/terminating-gateways-service.bats diff --git a/charts/consul/templates/terminating-gateways-deployment.yaml b/charts/consul/templates/terminating-gateways-deployment.yaml index a89750a3cf..51bcd3f9be 100644 --- a/charts/consul/templates/terminating-gateways-deployment.yaml +++ b/charts/consul/templates/terminating-gateways-deployment.yaml @@ -67,6 +67,7 @@ spec: component: terminating-gateway terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }} annotations: + "consul.hashicorp.com/gateway-kind": "terminating" {{- if (and $root.Values.global.secretsBackend.vault.enabled $root.Values.global.tls.enabled) }} "vault.hashicorp.com/agent-init-first": "true" "vault.hashicorp.com/agent-inject": "true" @@ -202,50 +203,21 @@ spec: - "-ec" - | {{- if $root.Values.global.acls.manageSystemACLs }} - consul-k8s-control-plane acl-init \ - -component-name=terminating-gateway/{{ template "consul.fullname" $root }}-{{ .name }} \ - -acl-auth-method={{ template "consul.fullname" $root }}-k8s-component-auth-method \ - {{- if $root.Values.global.adminPartitions.enabled }} - -partition={{ $root.Values.global.adminPartitions.name }} \ + consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -consul-api-timeout={{ .Values.global.consulAPITimeout }} \ + -gateway -gateway-name="terminating-gateway" \ + -consul-node-name=${POD_NAMESPACE} \ + {{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }} + -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \ + -primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \ + {{- else }} + -acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \ {{- end }} - -token-sink-file=/consul/service/acl-token \ - -consul-api-timeout={{ $root.Values.global.consulAPITimeout }} \ - -log-level={{ default $root.Values.global.logLevel }} \ - -log-json={{ $root.Values.global.logJSON }} + -service-name={{ .name }} \ + -acl-token-sink=/consul/service/acl-token \ + -log-level={{ default .Values.global.logLevel }} \ + -log-json={{ .Values.global.logJSON }} {{- end }} - - cat > /consul/service/service.hcl << EOF - service { - kind = "terminating-gateway" - name = "{{ .name }}" - id = "${POD_NAME}" - {{- if $root.Values.global.enableConsulNamespaces }} - namespace = "{{ (default $defaults.consulNamespace .consulNamespace) }}" - {{- end }} - {{- if $root.Values.global.adminPartitions.enabled }} - partition = "{{ $root.Values.global.adminPartitions.name }}" - {{- end }} - address = "${POD_IP}" - port = 8443 - {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} - proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } } - {{- end }} - checks = [ - { - name = "Terminating Gateway Listening" - interval = "10s" - tcp = "${POD_IP}:8443" - deregister_critical_service_after = "6h" - } - ] - } - EOF - - /consul-bin/consul services register \ - {{- if $root.Values.global.acls.manageSystemACLs }} - -token-file=/consul/service/acl-token \ - {{- end }} - /consul/service/service.hcl volumeMounts: - name: consul-service mountPath: /consul/service diff --git a/charts/consul/templates/terminating-gateways-service.yaml b/charts/consul/templates/terminating-gateways-service.yaml new file mode 100644 index 0000000000..d70a40d3d9 --- /dev/null +++ b/charts/consul/templates/terminating-gateways-service.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.terminatingGateways.enabled }} + +{{- $root := . }} +{{- $defaults := .Values.terminatingGateways.defaults }} + +{{- range .Values.terminatingGateways.gateways }} + +{{- $service := .service }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "consul.fullname" $root }}-{{ .name }} + namespace: {{ $root.Release.Namespace }} + labels: + app: {{ template "consul.name" $root }} + chart: {{ template "consul.chart" $root }} + heritage: {{ $root.Release.Service }} + release: {{ $root.Release.Name }} + component: terminating-gateways +spec: + selector: + app: {{ template "consul.name" $root }} + release: "{{ $root.Release.Name }}" + component: terminating-gateways + ports: + - name: gateway + port: 80 # TODO what should this be set to? + targetPort: 8443 + type: ClusterIP +--- +{{- end }} +{{- end }} diff --git a/charts/consul/test/unit/terminating-gateways-service.bats b/charts/consul/test/unit/terminating-gateways-service.bats new file mode 100644 index 0000000000..d831e512e6 --- /dev/null +++ b/charts/consul/test/unit/terminating-gateways-service.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats + +load _helpers + +@test "terminatingGateways/Service: disabled by default" { + cd `chart_dir` + assert_empty helm template \ + -s templates/terminating-gateways-service.yaml \ + . +} + +@test "terminatingGateways/Service: enabled with terminatingGateways and connectInject enabled" { + cd `chart_dir` + local object=$(helm template \ + -s templates/terminating-gateways-service.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq -s '.[0]' | tee /dev/stderr) + + local actual=$(echo $object | yq '. | length > 0' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +#-------------------------------------------------------------------- +# multiple gateways + +@test "terminatingGateways/Service: multiple gateways" { + cd `chart_dir` + local object=$(helm template \ + -s templates/terminating-gateways-service.yaml \ + --set 'terminatingGateways.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'terminatingGateways.gateways[0].name=gateway1' \ + --set 'terminatingGateways.gateways[1].name=gateway2' \ + . | tee /dev/stderr | + yq -s -r '.' | tee /dev/stderr) + + local actual=$(echo $object | yq -r '.[0].metadata.name' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-gateway1" ] + + local actual=$(echo $object | yq -r '.[1].metadata.name' | tee /dev/stderr) + [ "${actual}" = "release-name-consul-gateway2" ] + + local actual=$(echo "$object" | + yq -r '.[2] | length > 0' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + diff --git a/control-plane/connect-inject/endpoints_controller.go b/control-plane/connect-inject/endpoints_controller.go index 1e0dc85692..54cf28ffee 100644 --- a/control-plane/connect-inject/endpoints_controller.go +++ b/control-plane/connect-inject/endpoints_controller.go @@ -284,8 +284,20 @@ func (r *EndpointsController) registerGateway(pod corev1.Pod, serviceEndpoints c } // For pods managed by this controller, create and register the service instance. if managedByEndpointsController { + // Existence of this annotation has already been checked. + gatewayKind := pod.Labels[annotationGatewayKind] + + // Select the correct registration method. + var createGatewayRegistrations func(corev1.Pod, corev1.Endpoints, string) (*api.CatalogRegistration, error) + switch gatewayKind { + case "mesh-gateway": + createGatewayRegistrations = r.createGatewayRegistrations + case "terminating-gateways": + createGatewayRegistrations = r.createTerminatingGatewayRegistrations + } + // Get information from the pod to create service instance registrations. - serviceRegistration, err := r.createGatewayRegistrations(pod, serviceEndpoints, healthStatus) + serviceRegistration, err := createGatewayRegistrations(pod, serviceEndpoints, healthStatus) if err != nil { r.Log.Error(err, "failed to create service registrations for endpoints", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) return err @@ -591,6 +603,7 @@ func (r *EndpointsController) createServiceRegistrations(pod corev1.Pod, service return serviceRegistration, proxyServiceRegistration, nil } +// TODO rename this to `createMeshGatewayRegistrations`. // createGatewayRegistrations creates the gateway service registrations with the information from the Pod. func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, serviceEndpoints corev1.Endpoints, healthStatus string) (*api.CatalogRegistration, error) { wanAddr, wanPort, err := r.getWanData(pod, serviceEndpoints) @@ -660,6 +673,96 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service return serviceRegistration, nil } +// createTerminatingGatewayRegistrations creates the gateway service registrations with the information from the Pod. +func (r *EndpointsController) createTerminatingGatewayRegistrations(pod corev1.Pod, serviceEndpoints corev1.Endpoints, healthStatus string) (*api.CatalogRegistration, error) { + svcID := serviceID(pod, serviceEndpoints) + + meta := map[string]string{ + MetaKeyPodName: pod.Name, + MetaKeyKubeServiceName: serviceEndpoints.Name, + MetaKeyKubeNS: serviceEndpoints.Namespace, + MetaKeyManagedBy: managedByValue, + } + for k, v := range pod.Annotations { + if strings.HasPrefix(k, annotationMeta) && strings.TrimPrefix(k, annotationMeta) != "" { + if v == "$POD_NAME" { + meta[strings.TrimPrefix(k, annotationMeta)] = pod.Name + } else { + meta[strings.TrimPrefix(k, annotationMeta)] = v + } + } + } + + tags := consulTags(pod) + + /* + service { + ✅ kind = "terminating-gateway" + name = "{{ .name }}" + ✅ id = "${POD_NAME}" + {{- if $root.Values.global.enableConsulNamespaces }} + namespace = "{{ (default $defaults.consulNamespace .consulNamespace) }}" + {{- end }} + {{- if $root.Values.global.adminPartitions.enabled }} + partition = "{{ $root.Values.global.adminPartitions.name }}" + {{- end }} + ✅ address = "${POD_IP}" + ✅ port = 8443 + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } } + {{- end }} + checks = [ + { + ✅ name = "Terminating Gateway Listening" + ✅ interval = "10s" + ✅ tcp = "${POD_IP}:8443" + ✅ deregister_critical_service_after = "6h" + } + ] + } + EOF + */ + + consulNS := r.consulNamespace(pod.Namespace) + service := &api.AgentService{ + Kind: api.ServiceKindTerminatingGateway, + ID: svcID, + Service: "terminating-gateway", + Port: 8443, + Address: pod.Status.PodIP, + Meta: meta, + Namespace: consulNS, + Tags: tags, + } + serviceRegistration := &api.CatalogRegistration{ + Node: ConsulNodeName, + Address: ConsulNodeAddress, + Service: service, + Check: &api.AgentCheck{ + CheckID: consulHealthCheckID(pod.Namespace, svcID), + Name: ConsulKubernetesCheckName, + Type: ConsulKubernetesCheckType, + Status: healthStatus, + ServiceID: svcID, + Output: getHealthCheckStatusReason(healthStatus, pod.Name, pod.Namespace), + Namespace: consulNS, + }, + Checks: api.HealthChecks{ + &api.HealthCheck{ + Name: "Terminating Gateway Listening", + Definition: api.HealthCheckDefinition{ + TCP: fmt.Sprintf("%s:8443", pod.Status.PodIP), + IntervalDuration: time.Duration(10 * time.Second), + DeregisterCriticalServiceAfterDuration: time.Duration(6 * time.Hour), + }, + }, + }, + SkipNodeUpdate: true, + } + + return serviceRegistration, nil +} + func (r *EndpointsController) getWanData(pod corev1.Pod, endpoints corev1.Endpoints) (string, int, error) { var wanAddr string var wanPort int