Skip to content

Commit

Permalink
# This is a combination of 14 commits.
Browse files Browse the repository at this point in the history 
# This is the 1st commit message:

Add service for terminating-gateways

# This is the commit message #2:

Add gateway-kind:terminating to deployment

# This is the commit message #3:

Add registration path for terminating gateways

# This is the commit message #4:

Add BATS tests

# This is the commit message #5:

Remove registration from terminating gateways deployment

# This is the commit message #6:

Set ports AFAIK in service

# This is the commit message #7:

Begin setting values for endpoints controller

# This is the commit message #8:

Copy values from deployment to endpoints controller (as comment)

# This is the commit message #9:

Use connect-init instead of acl-init

# This is the commit message #10:

Remove guards from term gw service (they will get hit by the deployment)

# This is the commit message #11:

Range over gateways to produce a service for each deployment

# This is the commit message #12:

Add test for multiple gateways

# This is the commit message #13:

Remove the format script

# This is the commit message #14:

Note which parts of the config have been set
  • Loading branch information
Thomas Eckert committed Sep 8, 2022
1 parent 8ad1b3d commit 38ccc4c
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 43 deletions.
56 changes: 14 additions & 42 deletions charts/consul/templates/terminating-gateways-deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ spec:
component: terminating-gateway
terminating-gateway-name: {{ template "consul.fullname" $root }}-{{ .name }}
annotations:
"consul.hashicorp.com/gateway-kind": "terminating"
{{- if (and $root.Values.global.secretsBackend.vault.enabled $root.Values.global.tls.enabled) }}
"vault.hashicorp.com/agent-init-first": "true"
"vault.hashicorp.com/agent-inject": "true"
Expand Down Expand Up @@ -202,50 +203,21 @@ spec:
- "-ec"
- |
{{- if $root.Values.global.acls.manageSystemACLs }}
consul-k8s-control-plane acl-init \
-component-name=terminating-gateway/{{ template "consul.fullname" $root }}-{{ .name }} \
-acl-auth-method={{ template "consul.fullname" $root }}-k8s-component-auth-method \
{{- if $root.Values.global.adminPartitions.enabled }}
-partition={{ $root.Values.global.adminPartitions.name }} \
consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \
-consul-api-timeout={{ .Values.global.consulAPITimeout }} \
-gateway -gateway-name="terminating-gateway" \
-consul-node-name=${POD_NAMESPACE} \
{{- if and .Values.global.federation.enabled .Values.global.federation.primaryDatacenter }}
-acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method-{{ .Values.global.datacenter }} \
-primary-datacenter={{ .Values.global.federation.primaryDatacenter }} \
{{- else }}
-acl-auth-method={{ template "consul.fullname" . }}-k8s-component-auth-method \
{{- end }}
-token-sink-file=/consul/service/acl-token \
-consul-api-timeout={{ $root.Values.global.consulAPITimeout }} \
-log-level={{ default $root.Values.global.logLevel }} \
-log-json={{ $root.Values.global.logJSON }}
-service-name={{ .name }} \
-acl-token-sink=/consul/service/acl-token \
-log-level={{ default .Values.global.logLevel }} \
-log-json={{ .Values.global.logJSON }}
{{- end }}
cat > /consul/service/service.hcl << EOF
service {
kind = "terminating-gateway"
name = "{{ .name }}"
id = "${POD_NAME}"
{{- if $root.Values.global.enableConsulNamespaces }}
namespace = "{{ (default $defaults.consulNamespace .consulNamespace) }}"
{{- end }}
{{- if $root.Values.global.adminPartitions.enabled }}
partition = "{{ $root.Values.global.adminPartitions.name }}"
{{- end }}
address = "${POD_IP}"
port = 8443
{{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } }
{{- end }}
checks = [
{
name = "Terminating Gateway Listening"
interval = "10s"
tcp = "${POD_IP}:8443"
deregister_critical_service_after = "6h"
}
]
}
EOF
/consul-bin/consul services register \
{{- if $root.Values.global.acls.manageSystemACLs }}
-token-file=/consul/service/acl-token \
{{- end }}
/consul/service/service.hcl
volumeMounts:
- name: consul-service
mountPath: /consul/service
Expand Down
32 changes: 32 additions & 0 deletions charts/consul/templates/terminating-gateways-service.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
{{- if and .Values.terminatingGateways.enabled }}

{{- $root := . }}
{{- $defaults := .Values.terminatingGateways.defaults }}

{{- range .Values.terminatingGateways.gateways }}

{{- $service := .service }}
apiVersion: v1
kind: Service
metadata:
name: {{ template "consul.fullname" $root }}-{{ .name }}
namespace: {{ $root.Release.Namespace }}
labels:
app: {{ template "consul.name" $root }}
chart: {{ template "consul.chart" $root }}
heritage: {{ $root.Release.Service }}
release: {{ $root.Release.Name }}
component: terminating-gateways
spec:
selector:
app: {{ template "consul.name" $root }}
release: "{{ $root.Release.Name }}"
component: terminating-gateways
ports:
- name: gateway
port: 80 # TODO what should this be set to?
targetPort: 8443
type: ClusterIP
---
{{- end }}
{{- end }}
49 changes: 49 additions & 0 deletions charts/consul/test/unit/terminating-gateways-service.bats
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env bats

load _helpers

@test "terminatingGateways/Service: disabled by default" {
cd `chart_dir`
assert_empty helm template \
-s templates/terminating-gateways-service.yaml \
.
}

@test "terminatingGateways/Service: enabled with terminatingGateways and connectInject enabled" {
cd `chart_dir`
local object=$(helm template \
-s templates/terminating-gateways-service.yaml \
--set 'terminatingGateways.enabled=true' \
--set 'connectInject.enabled=true' \
. | tee /dev/stderr |
yq -s '.[0]' | tee /dev/stderr)

local actual=$(echo $object | yq '. | length > 0' | tee /dev/stderr)
[ "${actual}" = "true" ]
}

#--------------------------------------------------------------------
# multiple gateways

@test "terminatingGateways/Service: multiple gateways" {
cd `chart_dir`
local object=$(helm template \
-s templates/terminating-gateways-service.yaml \
--set 'terminatingGateways.enabled=true' \
--set 'connectInject.enabled=true' \
--set 'terminatingGateways.gateways[0].name=gateway1' \
--set 'terminatingGateways.gateways[1].name=gateway2' \
. | tee /dev/stderr |
yq -s -r '.' | tee /dev/stderr)

local actual=$(echo $object | yq -r '.[0].metadata.name' | tee /dev/stderr)
[ "${actual}" = "release-name-consul-gateway1" ]

local actual=$(echo $object | yq -r '.[1].metadata.name' | tee /dev/stderr)
[ "${actual}" = "release-name-consul-gateway2" ]

local actual=$(echo "$object" |
yq -r '.[2] | length > 0' | tee /dev/stderr)
[ "${actual}" = "false" ]
}

105 changes: 104 additions & 1 deletion control-plane/connect-inject/endpoints_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,8 +284,20 @@ func (r *EndpointsController) registerGateway(pod corev1.Pod, serviceEndpoints c
}
// For pods managed by this controller, create and register the service instance.
if managedByEndpointsController {
// Existence of this annotation has already been checked.
gatewayKind := pod.Labels[annotationGatewayKind]

// Select the correct registration method.
var createGatewayRegistrations func(corev1.Pod, corev1.Endpoints, string) (*api.CatalogRegistration, error)
switch gatewayKind {
case "mesh-gateway":
createGatewayRegistrations = r.createGatewayRegistrations
case "terminating-gateways":
createGatewayRegistrations = r.createTerminatingGatewayRegistrations
}

// Get information from the pod to create service instance registrations.
serviceRegistration, err := r.createGatewayRegistrations(pod, serviceEndpoints, healthStatus)
serviceRegistration, err := createGatewayRegistrations(pod, serviceEndpoints, healthStatus)
if err != nil {
r.Log.Error(err, "failed to create service registrations for endpoints", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace)
return err
Expand Down Expand Up @@ -591,6 +603,7 @@ func (r *EndpointsController) createServiceRegistrations(pod corev1.Pod, service
return serviceRegistration, proxyServiceRegistration, nil
}

// TODO rename this to `createMeshGatewayRegistrations`.
// createGatewayRegistrations creates the gateway service registrations with the information from the Pod.
func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, serviceEndpoints corev1.Endpoints, healthStatus string) (*api.CatalogRegistration, error) {
wanAddr, wanPort, err := r.getWanData(pod, serviceEndpoints)
Expand Down Expand Up @@ -660,6 +673,96 @@ func (r *EndpointsController) createGatewayRegistrations(pod corev1.Pod, service
return serviceRegistration, nil
}

// createTerminatingGatewayRegistrations creates the gateway service registrations with the information from the Pod.
func (r *EndpointsController) createTerminatingGatewayRegistrations(pod corev1.Pod, serviceEndpoints corev1.Endpoints, healthStatus string) (*api.CatalogRegistration, error) {
svcID := serviceID(pod, serviceEndpoints)

meta := map[string]string{
MetaKeyPodName: pod.Name,
MetaKeyKubeServiceName: serviceEndpoints.Name,
MetaKeyKubeNS: serviceEndpoints.Namespace,
MetaKeyManagedBy: managedByValue,
}
for k, v := range pod.Annotations {
if strings.HasPrefix(k, annotationMeta) && strings.TrimPrefix(k, annotationMeta) != "" {
if v == "$POD_NAME" {
meta[strings.TrimPrefix(k, annotationMeta)] = pod.Name
} else {
meta[strings.TrimPrefix(k, annotationMeta)] = v
}
}
}

tags := consulTags(pod)

/*
service {
✅ kind = "terminating-gateway"
name = "{{ .name }}"
✅ id = "${POD_NAME}"
{{- if $root.Values.global.enableConsulNamespaces }}
namespace = "{{ (default $defaults.consulNamespace .consulNamespace) }}"
{{- end }}
{{- if $root.Values.global.adminPartitions.enabled }}
partition = "{{ $root.Values.global.adminPartitions.name }}"
{{- end }}
✅ address = "${POD_IP}"
✅ port = 8443
{{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
proxy { config { envoy_prometheus_bind_addr = "${POD_IP}:20200" } }
{{- end }}
checks = [
{
✅ name = "Terminating Gateway Listening"
✅ interval = "10s"
✅ tcp = "${POD_IP}:8443"
✅ deregister_critical_service_after = "6h"
}
]
}
EOF
*/

consulNS := r.consulNamespace(pod.Namespace)
service := &api.AgentService{
Kind: api.ServiceKindTerminatingGateway,
ID: svcID,
Service: "terminating-gateway",
Port: 8443,
Address: pod.Status.PodIP,
Meta: meta,
Namespace: consulNS,
Tags: tags,
}
serviceRegistration := &api.CatalogRegistration{
Node: ConsulNodeName,
Address: ConsulNodeAddress,
Service: service,
Check: &api.AgentCheck{
CheckID: consulHealthCheckID(pod.Namespace, svcID),
Name: ConsulKubernetesCheckName,
Type: ConsulKubernetesCheckType,
Status: healthStatus,
ServiceID: svcID,
Output: getHealthCheckStatusReason(healthStatus, pod.Name, pod.Namespace),
Namespace: consulNS,
},
Checks: api.HealthChecks{
&api.HealthCheck{
Name: "Terminating Gateway Listening",
Definition: api.HealthCheckDefinition{
TCP: fmt.Sprintf("%s:8443", pod.Status.PodIP),
IntervalDuration: time.Duration(10 * time.Second),
DeregisterCriticalServiceAfterDuration: time.Duration(6 * time.Hour),
},
},
},
SkipNodeUpdate: true,
}

return serviceRegistration, nil
}

func (r *EndpointsController) getWanData(pod corev1.Pod, endpoints corev1.Endpoints) (string, int, error) {
var wanAddr string
var wanPort int
Expand Down

0 comments on commit 38ccc4c

Please sign in to comment.