From 8f8cd9d623411a3ba0c8b1964ef3136c6dc1de55 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Mon, 7 Nov 2022 17:08:43 -0700 Subject: [PATCH 1/5] refactor: split connect-inject into multiple packages --- .../connect-inject/common/annotations.go | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 control-plane/connect-inject/common/annotations.go diff --git a/control-plane/connect-inject/common/annotations.go b/control-plane/connect-inject/common/annotations.go new file mode 100644 index 0000000000..3830fe9e0c --- /dev/null +++ b/control-plane/connect-inject/common/annotations.go @@ -0,0 +1,192 @@ +package common + +const ( + // KeyInjectStatus is the key of the annotation that is added to + // a pod after an injection is done. + KeyInjectStatus = "consul.hashicorp.com/connect-inject-status" + + // KeyTransparentProxyStatus is the key of the annotation that is added to + // a pod when transparent proxy is done. + KeyTransparentProxyStatus = "consul.hashicorp.com/transparent-proxy-status" + + // KeyManagedBy is the key of the label that is added to pods managed + // by the Endpoints controller. This is to support upgrading from consul-k8s + // without Endpoints controller to consul-k8s with Endpoints controller + // without disrupting services managed the old way. + KeyManagedBy = "consul.hashicorp.com/connect-inject-managed-by" + + // AnnotationInject is the key of the annotation that controls whether + // injection is explicitly enabled or disabled for a pod. This should + // be set to a truthy or falsy value, as parseable by strconv.ParseBool. + AnnotationInject = "consul.hashicorp.com/connect-inject" + + // AnnotationGatewayKind is the key of the annotation that indicates pods + // that represent Consul Connect Gateways. This should be set to a + // value that is either "mesh", "ingress" or "terminating". + AnnotationGatewayKind = "consul.hashicorp.com/gateway-kind" + + // AnnotationGatewayConsulServiceName is the key of the annotation whose value + // is the service name with which the mesh gateway is registered. + AnnotationGatewayConsulServiceName = "consul.hashicorp.com/gateway-consul-service-name" + + // AnnotationMeshGatewayContainerPort is the key of the annotation whose value is + // used as the port and also registered as the LAN port when the mesh-gateway + // service is registered. + AnnotationMeshGatewayContainerPort = "consul.hashicorp.com/mesh-gateway-container-port" + + // AnnotationGatewayWANSource is the key of the annotation that determines which + // source to use to determine the wan address and wan port for the mesh-gateway + // service registration. + AnnotationGatewayWANSource = "consul.hashicorp.com/gateway-wan-address-source" + + // AnnotationGatewayWANAddress is the key of the annotation that when the source + // of the mesh-gateway is 'Static', is the value of the WAN address for the gateway. + AnnotationGatewayWANAddress = "consul.hashicorp.com/gateway-wan-address-static" + + // AnnotationGatewayWANPort is the key of the annotation whose value is the + // WAN port for the mesh-gateway service registration. + AnnotationGatewayWANPort = "consul.hashicorp.com/gateway-wan-port" + + // AnnotationGatewayNamespace is the key of the annotation that indicates the + // Consul namespace where a Terminating or Ingress Gateway pod is deployed. + AnnotationGatewayNamespace = "consul.hashicorp.com/gateway-namespace" + + // AnnotationInjectMountVolumes is the key of the annotation that controls whether + // the data volume that connect inject uses to store data including the Consul ACL token + // is mounted to other containers in the pod. It is a comma-separated list of container names + // to mount the volume on. It will be mounted at the path `/consul/connect-inject`. + AnnotationInjectMountVolumes = "consul.hashicorp.com/connect-inject-mount-volume" + + // AnnotationService is the name of the service to proxy. + // This defaults to the name of the Kubernetes service associated with the pod. + AnnotationService = "consul.hashicorp.com/connect-service" + + // AnnotationKubernetesService is the name of the Kubernetes service to register. + // This allows a pod to specify what Kubernetes service should trigger a Consul + // service registration in the case of multiple services referencing a deployment. + AnnotationKubernetesService = "consul.hashicorp.com/kubernetes-service" + + // AnnotationPort is the name or value of the port to proxy incoming + // connections to. + AnnotationPort = "consul.hashicorp.com/connect-service-port" + + // AnnotationUpstreams is a list of upstreams to register with the + // proxy in the format of `:,...`. The + // service name should map to a Consul service namd and the local port + // is the local port in the pod that the listener will bind to. It can + // be a named port. + AnnotationUpstreams = "consul.hashicorp.com/connect-service-upstreams" + + // AnnotationTags is a list of tags to register with the service + // this is specified as a comma separated list e.g. abc,123. + AnnotationTags = "consul.hashicorp.com/service-tags" + + // AnnotationMeta is a list of metadata key/value pairs to add to the service + // registration. This is specified in the format `:` + // e.g. consul.hashicorp.com/service-meta-foo:bar. + AnnotationMeta = "consul.hashicorp.com/service-meta-" + + // annotations for sidecar proxy resource limits. + AnnotationSidecarProxyCPULimit = "consul.hashicorp.com/sidecar-proxy-cpu-limit" + AnnotationSidecarProxyCPURequest = "consul.hashicorp.com/sidecar-proxy-cpu-request" + AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit" + AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request" + + // annotations for sidecar volumes. + AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" + AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" + + // annotations for sidecar concurrency. + AnnotationEnvoyProxyConcurrency = "consul.hashicorp.com/consul-envoy-proxy-concurrency" + + // annotations for metrics to configure where Prometheus scrapes + // metrics from, whether to run a merged metrics endpoint on the consul + // sidecar, and configure the connect service metrics. + AnnotationEnableMetrics = "consul.hashicorp.com/enable-metrics" + AnnotationEnableMetricsMerging = "consul.hashicorp.com/enable-metrics-merging" + AnnotationMergedMetricsPort = "consul.hashicorp.com/merged-metrics-port" + AnnotationPrometheusScrapePort = "consul.hashicorp.com/prometheus-scrape-port" + AnnotationPrometheusScrapePath = "consul.hashicorp.com/prometheus-scrape-path" + AnnotationServiceMetricsPort = "consul.hashicorp.com/service-metrics-port" + AnnotationServiceMetricsPath = "consul.hashicorp.com/service-metrics-path" + + // annotations for configuring TLS for Prometheus. + AnnotationPrometheusCAFile = "consul.hashicorp.com/prometheus-ca-file" + AnnotationPrometheusCAPath = "consul.hashicorp.com/prometheus-ca-path" + AnnotationPrometheusCertFile = "consul.hashicorp.com/prometheus-cert-file" + AnnotationPrometheusKeyFile = "consul.hashicorp.com/prometheus-key-file" + + // AnnotationEnvoyExtraArgs is a space-separated list of arguments to be passed to the + // envoy binary. See list of args here: https://www.envoyproxy.io/docs/envoy/latest/operations/cli + // e.g. consul.hashicorp.com/envoy-extra-args: "--log-level debug --disable-hot-restart" + // The arguments passed in via this annotation will take precendence over arguments + // passed via the -envoy-extra-args flag. + AnnotationEnvoyExtraArgs = "consul.hashicorp.com/envoy-extra-args" + + // AnnotationConsulNamespace is the Consul namespace the service is registered into. + AnnotationConsulNamespace = "consul.hashicorp.com/consul-namespace" + + // KeyConsulDNS enables or disables Consul DNS for a given pod. It can also be set as a label + // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting + // with their own annotation. + // This annotation/label takes a boolean value (true/false). + KeyConsulDNS = "consul.hashicorp.com/consul-dns" + + // KeyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label + // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting + // with their own annotation. + // This annotation/label takes a boolean value (true/false). + KeyTransparentProxy = "consul.hashicorp.com/transparent-proxy" + + // AnnotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection. + AnnotationTProxyExcludeInboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-inbound-ports" + + // AnnotationTProxyExcludeOutboundPorts is a comma-separated list of outbound ports to exclude from traffic redirection. + AnnotationTProxyExcludeOutboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-outbound-ports" + + // AnnotationTProxyExcludeOutboundCIDRs is a comma-separated list of outbound CIDRs to exclude from traffic redirection. + AnnotationTProxyExcludeOutboundCIDRs = "consul.hashicorp.com/transparent-proxy-exclude-outbound-cidrs" + + // AnnotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection. + AnnotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids" + + // AnnotationTransparentProxyOverwriteProbes controls whether the Kubernetes probes should be overwritten + // to point to the Envoy proxy when running in Transparent Proxy mode. + AnnotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes" + + // AnnotationRedirectTraffic stores iptables.Config information so that the CNI plugin can use it to apply + // iptables rules. + AnnotationRedirectTraffic = "consul.hashicorp.com/redirect-traffic-config" + + // AnnotationOriginalPod is the value of the pod before being overwritten by the consul + // webhook/meshWebhook. + AnnotationOriginalPod = "consul.hashicorp.com/original-pod" + + // AnnotationPeeringVersion is the version of the peering resource and can be utilized + // to explicitly perform the peering operation again. + AnnotationPeeringVersion = "consul.hashicorp.com/peering-version" + + // LabelServiceIgnore is a label that can be added to a service to prevent it from being + // registered with Consul. + LabelServiceIgnore = "consul.hashicorp.com/service-ignore" + + // LabelPeeringToken is a label that can be added to a secret to allow it to be watched + // by the peering controllers. + LabelPeeringToken = "consul.hashicorp.com/peering-token" + + // Injected is used as the annotation value for keyInjectStatus and annotationInjected. + Injected = "injected" + + // Enabled is used as the annotation value for keyTransparentProxyStatus. + Enabled = "enabled" + + // ManagedByValue is the value for keyManagedBy. + ManagedByValue = "consul-k8s-endpoints-controller" +) + +// Annotations used by Prometheus. +const ( + AnnotationPrometheusScrape = "prometheus.io/scrape" + AnnotationPrometheusPath = "prometheus.io/path" + AnnotationPrometheusPort = "prometheus.io/port" +) From 0368d97e7507653208695bb2f27f9a9716056aef Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Wed, 9 Nov 2022 13:19:07 -0700 Subject: [PATCH 2/5] create a constants package and move annotations to it --- .../connect-inject/common/annotations.go | 192 ------------------ 1 file changed, 192 deletions(-) delete mode 100644 control-plane/connect-inject/common/annotations.go diff --git a/control-plane/connect-inject/common/annotations.go b/control-plane/connect-inject/common/annotations.go deleted file mode 100644 index 3830fe9e0c..0000000000 --- a/control-plane/connect-inject/common/annotations.go +++ /dev/null @@ -1,192 +0,0 @@ -package common - -const ( - // KeyInjectStatus is the key of the annotation that is added to - // a pod after an injection is done. - KeyInjectStatus = "consul.hashicorp.com/connect-inject-status" - - // KeyTransparentProxyStatus is the key of the annotation that is added to - // a pod when transparent proxy is done. - KeyTransparentProxyStatus = "consul.hashicorp.com/transparent-proxy-status" - - // KeyManagedBy is the key of the label that is added to pods managed - // by the Endpoints controller. This is to support upgrading from consul-k8s - // without Endpoints controller to consul-k8s with Endpoints controller - // without disrupting services managed the old way. - KeyManagedBy = "consul.hashicorp.com/connect-inject-managed-by" - - // AnnotationInject is the key of the annotation that controls whether - // injection is explicitly enabled or disabled for a pod. This should - // be set to a truthy or falsy value, as parseable by strconv.ParseBool. - AnnotationInject = "consul.hashicorp.com/connect-inject" - - // AnnotationGatewayKind is the key of the annotation that indicates pods - // that represent Consul Connect Gateways. This should be set to a - // value that is either "mesh", "ingress" or "terminating". - AnnotationGatewayKind = "consul.hashicorp.com/gateway-kind" - - // AnnotationGatewayConsulServiceName is the key of the annotation whose value - // is the service name with which the mesh gateway is registered. - AnnotationGatewayConsulServiceName = "consul.hashicorp.com/gateway-consul-service-name" - - // AnnotationMeshGatewayContainerPort is the key of the annotation whose value is - // used as the port and also registered as the LAN port when the mesh-gateway - // service is registered. - AnnotationMeshGatewayContainerPort = "consul.hashicorp.com/mesh-gateway-container-port" - - // AnnotationGatewayWANSource is the key of the annotation that determines which - // source to use to determine the wan address and wan port for the mesh-gateway - // service registration. - AnnotationGatewayWANSource = "consul.hashicorp.com/gateway-wan-address-source" - - // AnnotationGatewayWANAddress is the key of the annotation that when the source - // of the mesh-gateway is 'Static', is the value of the WAN address for the gateway. - AnnotationGatewayWANAddress = "consul.hashicorp.com/gateway-wan-address-static" - - // AnnotationGatewayWANPort is the key of the annotation whose value is the - // WAN port for the mesh-gateway service registration. - AnnotationGatewayWANPort = "consul.hashicorp.com/gateway-wan-port" - - // AnnotationGatewayNamespace is the key of the annotation that indicates the - // Consul namespace where a Terminating or Ingress Gateway pod is deployed. - AnnotationGatewayNamespace = "consul.hashicorp.com/gateway-namespace" - - // AnnotationInjectMountVolumes is the key of the annotation that controls whether - // the data volume that connect inject uses to store data including the Consul ACL token - // is mounted to other containers in the pod. It is a comma-separated list of container names - // to mount the volume on. It will be mounted at the path `/consul/connect-inject`. - AnnotationInjectMountVolumes = "consul.hashicorp.com/connect-inject-mount-volume" - - // AnnotationService is the name of the service to proxy. - // This defaults to the name of the Kubernetes service associated with the pod. - AnnotationService = "consul.hashicorp.com/connect-service" - - // AnnotationKubernetesService is the name of the Kubernetes service to register. - // This allows a pod to specify what Kubernetes service should trigger a Consul - // service registration in the case of multiple services referencing a deployment. - AnnotationKubernetesService = "consul.hashicorp.com/kubernetes-service" - - // AnnotationPort is the name or value of the port to proxy incoming - // connections to. - AnnotationPort = "consul.hashicorp.com/connect-service-port" - - // AnnotationUpstreams is a list of upstreams to register with the - // proxy in the format of `:,...`. The - // service name should map to a Consul service namd and the local port - // is the local port in the pod that the listener will bind to. It can - // be a named port. - AnnotationUpstreams = "consul.hashicorp.com/connect-service-upstreams" - - // AnnotationTags is a list of tags to register with the service - // this is specified as a comma separated list e.g. abc,123. - AnnotationTags = "consul.hashicorp.com/service-tags" - - // AnnotationMeta is a list of metadata key/value pairs to add to the service - // registration. This is specified in the format `:` - // e.g. consul.hashicorp.com/service-meta-foo:bar. - AnnotationMeta = "consul.hashicorp.com/service-meta-" - - // annotations for sidecar proxy resource limits. - AnnotationSidecarProxyCPULimit = "consul.hashicorp.com/sidecar-proxy-cpu-limit" - AnnotationSidecarProxyCPURequest = "consul.hashicorp.com/sidecar-proxy-cpu-request" - AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit" - AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request" - - // annotations for sidecar volumes. - AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" - AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" - - // annotations for sidecar concurrency. - AnnotationEnvoyProxyConcurrency = "consul.hashicorp.com/consul-envoy-proxy-concurrency" - - // annotations for metrics to configure where Prometheus scrapes - // metrics from, whether to run a merged metrics endpoint on the consul - // sidecar, and configure the connect service metrics. - AnnotationEnableMetrics = "consul.hashicorp.com/enable-metrics" - AnnotationEnableMetricsMerging = "consul.hashicorp.com/enable-metrics-merging" - AnnotationMergedMetricsPort = "consul.hashicorp.com/merged-metrics-port" - AnnotationPrometheusScrapePort = "consul.hashicorp.com/prometheus-scrape-port" - AnnotationPrometheusScrapePath = "consul.hashicorp.com/prometheus-scrape-path" - AnnotationServiceMetricsPort = "consul.hashicorp.com/service-metrics-port" - AnnotationServiceMetricsPath = "consul.hashicorp.com/service-metrics-path" - - // annotations for configuring TLS for Prometheus. - AnnotationPrometheusCAFile = "consul.hashicorp.com/prometheus-ca-file" - AnnotationPrometheusCAPath = "consul.hashicorp.com/prometheus-ca-path" - AnnotationPrometheusCertFile = "consul.hashicorp.com/prometheus-cert-file" - AnnotationPrometheusKeyFile = "consul.hashicorp.com/prometheus-key-file" - - // AnnotationEnvoyExtraArgs is a space-separated list of arguments to be passed to the - // envoy binary. See list of args here: https://www.envoyproxy.io/docs/envoy/latest/operations/cli - // e.g. consul.hashicorp.com/envoy-extra-args: "--log-level debug --disable-hot-restart" - // The arguments passed in via this annotation will take precendence over arguments - // passed via the -envoy-extra-args flag. - AnnotationEnvoyExtraArgs = "consul.hashicorp.com/envoy-extra-args" - - // AnnotationConsulNamespace is the Consul namespace the service is registered into. - AnnotationConsulNamespace = "consul.hashicorp.com/consul-namespace" - - // KeyConsulDNS enables or disables Consul DNS for a given pod. It can also be set as a label - // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting - // with their own annotation. - // This annotation/label takes a boolean value (true/false). - KeyConsulDNS = "consul.hashicorp.com/consul-dns" - - // KeyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label - // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting - // with their own annotation. - // This annotation/label takes a boolean value (true/false). - KeyTransparentProxy = "consul.hashicorp.com/transparent-proxy" - - // AnnotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection. - AnnotationTProxyExcludeInboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-inbound-ports" - - // AnnotationTProxyExcludeOutboundPorts is a comma-separated list of outbound ports to exclude from traffic redirection. - AnnotationTProxyExcludeOutboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-outbound-ports" - - // AnnotationTProxyExcludeOutboundCIDRs is a comma-separated list of outbound CIDRs to exclude from traffic redirection. - AnnotationTProxyExcludeOutboundCIDRs = "consul.hashicorp.com/transparent-proxy-exclude-outbound-cidrs" - - // AnnotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection. - AnnotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids" - - // AnnotationTransparentProxyOverwriteProbes controls whether the Kubernetes probes should be overwritten - // to point to the Envoy proxy when running in Transparent Proxy mode. - AnnotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes" - - // AnnotationRedirectTraffic stores iptables.Config information so that the CNI plugin can use it to apply - // iptables rules. - AnnotationRedirectTraffic = "consul.hashicorp.com/redirect-traffic-config" - - // AnnotationOriginalPod is the value of the pod before being overwritten by the consul - // webhook/meshWebhook. - AnnotationOriginalPod = "consul.hashicorp.com/original-pod" - - // AnnotationPeeringVersion is the version of the peering resource and can be utilized - // to explicitly perform the peering operation again. - AnnotationPeeringVersion = "consul.hashicorp.com/peering-version" - - // LabelServiceIgnore is a label that can be added to a service to prevent it from being - // registered with Consul. - LabelServiceIgnore = "consul.hashicorp.com/service-ignore" - - // LabelPeeringToken is a label that can be added to a secret to allow it to be watched - // by the peering controllers. - LabelPeeringToken = "consul.hashicorp.com/peering-token" - - // Injected is used as the annotation value for keyInjectStatus and annotationInjected. - Injected = "injected" - - // Enabled is used as the annotation value for keyTransparentProxyStatus. - Enabled = "enabled" - - // ManagedByValue is the value for keyManagedBy. - ManagedByValue = "consul-k8s-endpoints-controller" -) - -// Annotations used by Prometheus. -const ( - AnnotationPrometheusScrape = "prometheus.io/scrape" - AnnotationPrometheusPath = "prometheus.io/path" - AnnotationPrometheusPort = "prometheus.io/port" -) From 62ab7e2a544a1bb8539211dbff3823f247d5a993 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Mon, 7 Nov 2022 17:08:43 -0700 Subject: [PATCH 3/5] refactor: split connect-inject into multiple packages --- .../connect-inject/common/annotations.go | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 control-plane/connect-inject/common/annotations.go diff --git a/control-plane/connect-inject/common/annotations.go b/control-plane/connect-inject/common/annotations.go new file mode 100644 index 0000000000..3830fe9e0c --- /dev/null +++ b/control-plane/connect-inject/common/annotations.go @@ -0,0 +1,192 @@ +package common + +const ( + // KeyInjectStatus is the key of the annotation that is added to + // a pod after an injection is done. + KeyInjectStatus = "consul.hashicorp.com/connect-inject-status" + + // KeyTransparentProxyStatus is the key of the annotation that is added to + // a pod when transparent proxy is done. + KeyTransparentProxyStatus = "consul.hashicorp.com/transparent-proxy-status" + + // KeyManagedBy is the key of the label that is added to pods managed + // by the Endpoints controller. This is to support upgrading from consul-k8s + // without Endpoints controller to consul-k8s with Endpoints controller + // without disrupting services managed the old way. + KeyManagedBy = "consul.hashicorp.com/connect-inject-managed-by" + + // AnnotationInject is the key of the annotation that controls whether + // injection is explicitly enabled or disabled for a pod. This should + // be set to a truthy or falsy value, as parseable by strconv.ParseBool. + AnnotationInject = "consul.hashicorp.com/connect-inject" + + // AnnotationGatewayKind is the key of the annotation that indicates pods + // that represent Consul Connect Gateways. This should be set to a + // value that is either "mesh", "ingress" or "terminating". + AnnotationGatewayKind = "consul.hashicorp.com/gateway-kind" + + // AnnotationGatewayConsulServiceName is the key of the annotation whose value + // is the service name with which the mesh gateway is registered. + AnnotationGatewayConsulServiceName = "consul.hashicorp.com/gateway-consul-service-name" + + // AnnotationMeshGatewayContainerPort is the key of the annotation whose value is + // used as the port and also registered as the LAN port when the mesh-gateway + // service is registered. + AnnotationMeshGatewayContainerPort = "consul.hashicorp.com/mesh-gateway-container-port" + + // AnnotationGatewayWANSource is the key of the annotation that determines which + // source to use to determine the wan address and wan port for the mesh-gateway + // service registration. + AnnotationGatewayWANSource = "consul.hashicorp.com/gateway-wan-address-source" + + // AnnotationGatewayWANAddress is the key of the annotation that when the source + // of the mesh-gateway is 'Static', is the value of the WAN address for the gateway. + AnnotationGatewayWANAddress = "consul.hashicorp.com/gateway-wan-address-static" + + // AnnotationGatewayWANPort is the key of the annotation whose value is the + // WAN port for the mesh-gateway service registration. + AnnotationGatewayWANPort = "consul.hashicorp.com/gateway-wan-port" + + // AnnotationGatewayNamespace is the key of the annotation that indicates the + // Consul namespace where a Terminating or Ingress Gateway pod is deployed. + AnnotationGatewayNamespace = "consul.hashicorp.com/gateway-namespace" + + // AnnotationInjectMountVolumes is the key of the annotation that controls whether + // the data volume that connect inject uses to store data including the Consul ACL token + // is mounted to other containers in the pod. It is a comma-separated list of container names + // to mount the volume on. It will be mounted at the path `/consul/connect-inject`. + AnnotationInjectMountVolumes = "consul.hashicorp.com/connect-inject-mount-volume" + + // AnnotationService is the name of the service to proxy. + // This defaults to the name of the Kubernetes service associated with the pod. + AnnotationService = "consul.hashicorp.com/connect-service" + + // AnnotationKubernetesService is the name of the Kubernetes service to register. + // This allows a pod to specify what Kubernetes service should trigger a Consul + // service registration in the case of multiple services referencing a deployment. + AnnotationKubernetesService = "consul.hashicorp.com/kubernetes-service" + + // AnnotationPort is the name or value of the port to proxy incoming + // connections to. + AnnotationPort = "consul.hashicorp.com/connect-service-port" + + // AnnotationUpstreams is a list of upstreams to register with the + // proxy in the format of `:,...`. The + // service name should map to a Consul service namd and the local port + // is the local port in the pod that the listener will bind to. It can + // be a named port. + AnnotationUpstreams = "consul.hashicorp.com/connect-service-upstreams" + + // AnnotationTags is a list of tags to register with the service + // this is specified as a comma separated list e.g. abc,123. + AnnotationTags = "consul.hashicorp.com/service-tags" + + // AnnotationMeta is a list of metadata key/value pairs to add to the service + // registration. This is specified in the format `:` + // e.g. consul.hashicorp.com/service-meta-foo:bar. + AnnotationMeta = "consul.hashicorp.com/service-meta-" + + // annotations for sidecar proxy resource limits. + AnnotationSidecarProxyCPULimit = "consul.hashicorp.com/sidecar-proxy-cpu-limit" + AnnotationSidecarProxyCPURequest = "consul.hashicorp.com/sidecar-proxy-cpu-request" + AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit" + AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request" + + // annotations for sidecar volumes. + AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" + AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" + + // annotations for sidecar concurrency. + AnnotationEnvoyProxyConcurrency = "consul.hashicorp.com/consul-envoy-proxy-concurrency" + + // annotations for metrics to configure where Prometheus scrapes + // metrics from, whether to run a merged metrics endpoint on the consul + // sidecar, and configure the connect service metrics. + AnnotationEnableMetrics = "consul.hashicorp.com/enable-metrics" + AnnotationEnableMetricsMerging = "consul.hashicorp.com/enable-metrics-merging" + AnnotationMergedMetricsPort = "consul.hashicorp.com/merged-metrics-port" + AnnotationPrometheusScrapePort = "consul.hashicorp.com/prometheus-scrape-port" + AnnotationPrometheusScrapePath = "consul.hashicorp.com/prometheus-scrape-path" + AnnotationServiceMetricsPort = "consul.hashicorp.com/service-metrics-port" + AnnotationServiceMetricsPath = "consul.hashicorp.com/service-metrics-path" + + // annotations for configuring TLS for Prometheus. + AnnotationPrometheusCAFile = "consul.hashicorp.com/prometheus-ca-file" + AnnotationPrometheusCAPath = "consul.hashicorp.com/prometheus-ca-path" + AnnotationPrometheusCertFile = "consul.hashicorp.com/prometheus-cert-file" + AnnotationPrometheusKeyFile = "consul.hashicorp.com/prometheus-key-file" + + // AnnotationEnvoyExtraArgs is a space-separated list of arguments to be passed to the + // envoy binary. See list of args here: https://www.envoyproxy.io/docs/envoy/latest/operations/cli + // e.g. consul.hashicorp.com/envoy-extra-args: "--log-level debug --disable-hot-restart" + // The arguments passed in via this annotation will take precendence over arguments + // passed via the -envoy-extra-args flag. + AnnotationEnvoyExtraArgs = "consul.hashicorp.com/envoy-extra-args" + + // AnnotationConsulNamespace is the Consul namespace the service is registered into. + AnnotationConsulNamespace = "consul.hashicorp.com/consul-namespace" + + // KeyConsulDNS enables or disables Consul DNS for a given pod. It can also be set as a label + // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting + // with their own annotation. + // This annotation/label takes a boolean value (true/false). + KeyConsulDNS = "consul.hashicorp.com/consul-dns" + + // KeyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label + // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting + // with their own annotation. + // This annotation/label takes a boolean value (true/false). + KeyTransparentProxy = "consul.hashicorp.com/transparent-proxy" + + // AnnotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection. + AnnotationTProxyExcludeInboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-inbound-ports" + + // AnnotationTProxyExcludeOutboundPorts is a comma-separated list of outbound ports to exclude from traffic redirection. + AnnotationTProxyExcludeOutboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-outbound-ports" + + // AnnotationTProxyExcludeOutboundCIDRs is a comma-separated list of outbound CIDRs to exclude from traffic redirection. + AnnotationTProxyExcludeOutboundCIDRs = "consul.hashicorp.com/transparent-proxy-exclude-outbound-cidrs" + + // AnnotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection. + AnnotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids" + + // AnnotationTransparentProxyOverwriteProbes controls whether the Kubernetes probes should be overwritten + // to point to the Envoy proxy when running in Transparent Proxy mode. + AnnotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes" + + // AnnotationRedirectTraffic stores iptables.Config information so that the CNI plugin can use it to apply + // iptables rules. + AnnotationRedirectTraffic = "consul.hashicorp.com/redirect-traffic-config" + + // AnnotationOriginalPod is the value of the pod before being overwritten by the consul + // webhook/meshWebhook. + AnnotationOriginalPod = "consul.hashicorp.com/original-pod" + + // AnnotationPeeringVersion is the version of the peering resource and can be utilized + // to explicitly perform the peering operation again. + AnnotationPeeringVersion = "consul.hashicorp.com/peering-version" + + // LabelServiceIgnore is a label that can be added to a service to prevent it from being + // registered with Consul. + LabelServiceIgnore = "consul.hashicorp.com/service-ignore" + + // LabelPeeringToken is a label that can be added to a secret to allow it to be watched + // by the peering controllers. + LabelPeeringToken = "consul.hashicorp.com/peering-token" + + // Injected is used as the annotation value for keyInjectStatus and annotationInjected. + Injected = "injected" + + // Enabled is used as the annotation value for keyTransparentProxyStatus. + Enabled = "enabled" + + // ManagedByValue is the value for keyManagedBy. + ManagedByValue = "consul-k8s-endpoints-controller" +) + +// Annotations used by Prometheus. +const ( + AnnotationPrometheusScrape = "prometheus.io/scrape" + AnnotationPrometheusPath = "prometheus.io/path" + AnnotationPrometheusPort = "prometheus.io/port" +) From dd486be05b988c9097a37bf189bf1ccab164faf4 Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Mon, 7 Nov 2022 22:54:09 -0700 Subject: [PATCH 4/5] agentless: support updating health checks on consul clients during an upgrade to agentless --- .../templates/connect-inject-deployment.yaml | 58 ++-- .../connect-inject/common/annotations.go | 192 ------------- .../constants/annotations_and_labels.go | 3 + .../endpoints/consul_client_health_checks.go | 110 ++++++++ .../consul_client_health_checks_test.go | 260 ++++++++++++++++++ .../endpoints/endpoints_controller.go | 45 ++- .../endpoints_controller_ent_test.go | 10 +- .../endpoints/endpoints_controller_test.go | 215 ++++++++++++++- .../connect-inject/webhook/mesh_webhook.go | 2 + .../webhook/mesh_webhook_test.go | 62 ++++- control-plane/go.mod | 1 + control-plane/go.sum | 2 + .../subcommand/connect-init/command_test.go | 2 +- .../subcommand/inject-connect/command.go | 16 +- 14 files changed, 710 insertions(+), 268 deletions(-) delete mode 100644 control-plane/connect-inject/common/annotations.go create mode 100644 control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go create mode 100644 control-plane/connect-inject/controllers/endpoints/consul_client_health_checks_test.go diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index e4a63095cf..fa5e21bb7b 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -82,9 +82,9 @@ spec: - name: sidecar-injector image: "{{ default .Values.global.imageK8S .Values.connectInject.image }}" ports: - - containerPort: 8080 - name: webhook-server - protocol: TCP + - containerPort: 8080 + name: webhook-server + protocol: TCP env: - name: NAMESPACE valueFrom: @@ -239,24 +239,12 @@ spec: {{- end }} {{- end }} - {{- if .Values.global.consulSidecarContainer }} - {{- $consulSidecarResources := .Values.global.consulSidecarContainer.resources }} - {{- if not (kindIs "invalid" $consulSidecarResources.limits.memory) }} - -default-consul-sidecar-memory-limit={{ $consulSidecarResources.limits.memory }} \ - {{- end }} - {{- if not (kindIs "invalid" $consulSidecarResources.requests.memory) }} - -default-consul-sidecar-memory-request={{ $consulSidecarResources.requests.memory }} \ - {{- end }} - {{- if not (kindIs "invalid" $consulSidecarResources.limits.cpu) }} - -default-consul-sidecar-cpu-limit={{ $consulSidecarResources.limits.cpu }} \ - {{- end }} - {{- if not (kindIs "invalid" $consulSidecarResources.requests.cpu) }} - -default-consul-sidecar-cpu-request={{ $consulSidecarResources.requests.cpu }} \ - {{- end }} - {{- end }} {{- if .Values.global.cloud.enabled }} -tls-server-name=server.{{ .Values.global.datacenter}}.{{ .Values.global.domain}} \ {{- end }} + {{- if and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt }} + -enable-auto-encrypt \ + {{- end }} startupProbe: httpGet: path: /readyz/ready @@ -286,14 +274,14 @@ spec: timeoutSeconds: 5 volumeMounts: {{- if not (and .Values.global.secretsBackend.vault.enabled .Values.global.secretsBackend.vault.connectInject.tlsCert.secretName) }} - - name: certs - mountPath: /etc/connect-injector/certs - readOnly: true + - name: certs + mountPath: /etc/connect-injector/certs + readOnly: true {{- end }} {{- if and .Values.global.tls.enabled (not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots))}} - - name: consul-ca-cert - mountPath: /consul/tls/ca - readOnly: true + - name: consul-ca-cert + mountPath: /consul/tls/ca + readOnly: true {{- end }} {{- with .Values.connectInject.resources }} resources: @@ -301,23 +289,23 @@ spec: {{- end }} volumes: {{- if not (and .Values.global.secretsBackend.vault.enabled .Values.global.secretsBackend.vault.connectInject.tlsCert.secretName) }} - - name: certs - secret: - defaultMode: 420 - secretName: {{ template "consul.fullname" . }}-connect-inject-webhook-cert + - name: certs + secret: + defaultMode: 420 + secretName: {{ template "consul.fullname" . }}-connect-inject-webhook-cert {{- end }} {{- if .Values.global.tls.enabled }} {{- if not (and .Values.externalServers.enabled .Values.externalServers.useSystemRoots) }} - - name: consul-ca-cert - secret: + - name: consul-ca-cert + secret: {{- if .Values.global.tls.caCert.secretName }} - secretName: {{ .Values.global.tls.caCert.secretName }} + secretName: {{ .Values.global.tls.caCert.secretName }} {{- else }} - secretName: {{ template "consul.fullname" . }}-ca-cert + secretName: {{ template "consul.fullname" . }}-ca-cert {{- end }} - items: - - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }} - path: tls.crt + items: + - key: {{ default "tls.crt" .Values.global.tls.caCert.secretKey }} + path: tls.crt {{- end }} {{- end }} {{- if .Values.connectInject.priorityClassName }} diff --git a/control-plane/connect-inject/common/annotations.go b/control-plane/connect-inject/common/annotations.go deleted file mode 100644 index 3830fe9e0c..0000000000 --- a/control-plane/connect-inject/common/annotations.go +++ /dev/null @@ -1,192 +0,0 @@ -package common - -const ( - // KeyInjectStatus is the key of the annotation that is added to - // a pod after an injection is done. - KeyInjectStatus = "consul.hashicorp.com/connect-inject-status" - - // KeyTransparentProxyStatus is the key of the annotation that is added to - // a pod when transparent proxy is done. - KeyTransparentProxyStatus = "consul.hashicorp.com/transparent-proxy-status" - - // KeyManagedBy is the key of the label that is added to pods managed - // by the Endpoints controller. This is to support upgrading from consul-k8s - // without Endpoints controller to consul-k8s with Endpoints controller - // without disrupting services managed the old way. - KeyManagedBy = "consul.hashicorp.com/connect-inject-managed-by" - - // AnnotationInject is the key of the annotation that controls whether - // injection is explicitly enabled or disabled for a pod. This should - // be set to a truthy or falsy value, as parseable by strconv.ParseBool. - AnnotationInject = "consul.hashicorp.com/connect-inject" - - // AnnotationGatewayKind is the key of the annotation that indicates pods - // that represent Consul Connect Gateways. This should be set to a - // value that is either "mesh", "ingress" or "terminating". - AnnotationGatewayKind = "consul.hashicorp.com/gateway-kind" - - // AnnotationGatewayConsulServiceName is the key of the annotation whose value - // is the service name with which the mesh gateway is registered. - AnnotationGatewayConsulServiceName = "consul.hashicorp.com/gateway-consul-service-name" - - // AnnotationMeshGatewayContainerPort is the key of the annotation whose value is - // used as the port and also registered as the LAN port when the mesh-gateway - // service is registered. - AnnotationMeshGatewayContainerPort = "consul.hashicorp.com/mesh-gateway-container-port" - - // AnnotationGatewayWANSource is the key of the annotation that determines which - // source to use to determine the wan address and wan port for the mesh-gateway - // service registration. - AnnotationGatewayWANSource = "consul.hashicorp.com/gateway-wan-address-source" - - // AnnotationGatewayWANAddress is the key of the annotation that when the source - // of the mesh-gateway is 'Static', is the value of the WAN address for the gateway. - AnnotationGatewayWANAddress = "consul.hashicorp.com/gateway-wan-address-static" - - // AnnotationGatewayWANPort is the key of the annotation whose value is the - // WAN port for the mesh-gateway service registration. - AnnotationGatewayWANPort = "consul.hashicorp.com/gateway-wan-port" - - // AnnotationGatewayNamespace is the key of the annotation that indicates the - // Consul namespace where a Terminating or Ingress Gateway pod is deployed. - AnnotationGatewayNamespace = "consul.hashicorp.com/gateway-namespace" - - // AnnotationInjectMountVolumes is the key of the annotation that controls whether - // the data volume that connect inject uses to store data including the Consul ACL token - // is mounted to other containers in the pod. It is a comma-separated list of container names - // to mount the volume on. It will be mounted at the path `/consul/connect-inject`. - AnnotationInjectMountVolumes = "consul.hashicorp.com/connect-inject-mount-volume" - - // AnnotationService is the name of the service to proxy. - // This defaults to the name of the Kubernetes service associated with the pod. - AnnotationService = "consul.hashicorp.com/connect-service" - - // AnnotationKubernetesService is the name of the Kubernetes service to register. - // This allows a pod to specify what Kubernetes service should trigger a Consul - // service registration in the case of multiple services referencing a deployment. - AnnotationKubernetesService = "consul.hashicorp.com/kubernetes-service" - - // AnnotationPort is the name or value of the port to proxy incoming - // connections to. - AnnotationPort = "consul.hashicorp.com/connect-service-port" - - // AnnotationUpstreams is a list of upstreams to register with the - // proxy in the format of `:,...`. The - // service name should map to a Consul service namd and the local port - // is the local port in the pod that the listener will bind to. It can - // be a named port. - AnnotationUpstreams = "consul.hashicorp.com/connect-service-upstreams" - - // AnnotationTags is a list of tags to register with the service - // this is specified as a comma separated list e.g. abc,123. - AnnotationTags = "consul.hashicorp.com/service-tags" - - // AnnotationMeta is a list of metadata key/value pairs to add to the service - // registration. This is specified in the format `:` - // e.g. consul.hashicorp.com/service-meta-foo:bar. - AnnotationMeta = "consul.hashicorp.com/service-meta-" - - // annotations for sidecar proxy resource limits. - AnnotationSidecarProxyCPULimit = "consul.hashicorp.com/sidecar-proxy-cpu-limit" - AnnotationSidecarProxyCPURequest = "consul.hashicorp.com/sidecar-proxy-cpu-request" - AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit" - AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request" - - // annotations for sidecar volumes. - AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" - AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" - - // annotations for sidecar concurrency. - AnnotationEnvoyProxyConcurrency = "consul.hashicorp.com/consul-envoy-proxy-concurrency" - - // annotations for metrics to configure where Prometheus scrapes - // metrics from, whether to run a merged metrics endpoint on the consul - // sidecar, and configure the connect service metrics. - AnnotationEnableMetrics = "consul.hashicorp.com/enable-metrics" - AnnotationEnableMetricsMerging = "consul.hashicorp.com/enable-metrics-merging" - AnnotationMergedMetricsPort = "consul.hashicorp.com/merged-metrics-port" - AnnotationPrometheusScrapePort = "consul.hashicorp.com/prometheus-scrape-port" - AnnotationPrometheusScrapePath = "consul.hashicorp.com/prometheus-scrape-path" - AnnotationServiceMetricsPort = "consul.hashicorp.com/service-metrics-port" - AnnotationServiceMetricsPath = "consul.hashicorp.com/service-metrics-path" - - // annotations for configuring TLS for Prometheus. - AnnotationPrometheusCAFile = "consul.hashicorp.com/prometheus-ca-file" - AnnotationPrometheusCAPath = "consul.hashicorp.com/prometheus-ca-path" - AnnotationPrometheusCertFile = "consul.hashicorp.com/prometheus-cert-file" - AnnotationPrometheusKeyFile = "consul.hashicorp.com/prometheus-key-file" - - // AnnotationEnvoyExtraArgs is a space-separated list of arguments to be passed to the - // envoy binary. See list of args here: https://www.envoyproxy.io/docs/envoy/latest/operations/cli - // e.g. consul.hashicorp.com/envoy-extra-args: "--log-level debug --disable-hot-restart" - // The arguments passed in via this annotation will take precendence over arguments - // passed via the -envoy-extra-args flag. - AnnotationEnvoyExtraArgs = "consul.hashicorp.com/envoy-extra-args" - - // AnnotationConsulNamespace is the Consul namespace the service is registered into. - AnnotationConsulNamespace = "consul.hashicorp.com/consul-namespace" - - // KeyConsulDNS enables or disables Consul DNS for a given pod. It can also be set as a label - // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting - // with their own annotation. - // This annotation/label takes a boolean value (true/false). - KeyConsulDNS = "consul.hashicorp.com/consul-dns" - - // KeyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label - // on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting - // with their own annotation. - // This annotation/label takes a boolean value (true/false). - KeyTransparentProxy = "consul.hashicorp.com/transparent-proxy" - - // AnnotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection. - AnnotationTProxyExcludeInboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-inbound-ports" - - // AnnotationTProxyExcludeOutboundPorts is a comma-separated list of outbound ports to exclude from traffic redirection. - AnnotationTProxyExcludeOutboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-outbound-ports" - - // AnnotationTProxyExcludeOutboundCIDRs is a comma-separated list of outbound CIDRs to exclude from traffic redirection. - AnnotationTProxyExcludeOutboundCIDRs = "consul.hashicorp.com/transparent-proxy-exclude-outbound-cidrs" - - // AnnotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection. - AnnotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids" - - // AnnotationTransparentProxyOverwriteProbes controls whether the Kubernetes probes should be overwritten - // to point to the Envoy proxy when running in Transparent Proxy mode. - AnnotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes" - - // AnnotationRedirectTraffic stores iptables.Config information so that the CNI plugin can use it to apply - // iptables rules. - AnnotationRedirectTraffic = "consul.hashicorp.com/redirect-traffic-config" - - // AnnotationOriginalPod is the value of the pod before being overwritten by the consul - // webhook/meshWebhook. - AnnotationOriginalPod = "consul.hashicorp.com/original-pod" - - // AnnotationPeeringVersion is the version of the peering resource and can be utilized - // to explicitly perform the peering operation again. - AnnotationPeeringVersion = "consul.hashicorp.com/peering-version" - - // LabelServiceIgnore is a label that can be added to a service to prevent it from being - // registered with Consul. - LabelServiceIgnore = "consul.hashicorp.com/service-ignore" - - // LabelPeeringToken is a label that can be added to a secret to allow it to be watched - // by the peering controllers. - LabelPeeringToken = "consul.hashicorp.com/peering-token" - - // Injected is used as the annotation value for keyInjectStatus and annotationInjected. - Injected = "injected" - - // Enabled is used as the annotation value for keyTransparentProxyStatus. - Enabled = "enabled" - - // ManagedByValue is the value for keyManagedBy. - ManagedByValue = "consul-k8s-endpoints-controller" -) - -// Annotations used by Prometheus. -const ( - AnnotationPrometheusScrape = "prometheus.io/scrape" - AnnotationPrometheusPath = "prometheus.io/path" - AnnotationPrometheusPort = "prometheus.io/port" -) diff --git a/control-plane/connect-inject/constants/annotations_and_labels.go b/control-plane/connect-inject/constants/annotations_and_labels.go index 9bda073ca2..c3ba29ace2 100644 --- a/control-plane/connect-inject/constants/annotations_and_labels.go +++ b/control-plane/connect-inject/constants/annotations_and_labels.go @@ -166,6 +166,9 @@ const ( // to explicitly perform the peering operation again. AnnotationPeeringVersion = "consul.hashicorp.com/peering-version" + // AnnotationConsulK8sVersion is the current version of this binary. + AnnotationConsulK8sVersion = "consul.hashicorp.com/connect-k8s-version" + // LabelServiceIgnore is a label that can be added to a service to prevent it from being // registered with Consul. LabelServiceIgnore = "consul.hashicorp.com/service-ignore" diff --git a/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go b/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go new file mode 100644 index 0000000000..3496ee2212 --- /dev/null +++ b/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go @@ -0,0 +1,110 @@ +package endpoints + +import ( + "fmt" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-server-connection-manager/discovery" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/go-version" + corev1 "k8s.io/api/core/v1" +) + +// isConsulDataplaneSupported returns true if the consul-k8s version on the pod supports +// consul-dataplane architecture of Consul. +func isConsulDataplaneSupported(pod corev1.Pod) bool { + if anno, ok := pod.Annotations[constants.AnnotationConsulK8sVersion]; ok { + consulK8sVersion, err := version.NewVersion(anno) + if err != nil { + return false + } + consulDPSupportedVersion, err := version.NewVersion("v1.0.0-beta1") + if err != nil { + return false + } + if !consulK8sVersion.LessThan(consulDPSupportedVersion) { + return true + } + } + return false +} + +func (r *Controller) consulClientCfgForNodeAgent(serverClient *api.Client, pod corev1.Pod, state discovery.State) (*api.Config, error) { + ccCfg := &api.Config{ + Scheme: r.ConsulClientConfig.APIClientConfig.Scheme, + } + + consulClientHttpPort := 8500 + if ccCfg.Scheme == "https" { + consulClientHttpPort = 8501 + ccCfg.TLSConfig.CAFile = r.ConsulClientConfig.APIClientConfig.TLSConfig.CAFile + } + if r.consulClientHttpPort != 0 { + consulClientHttpPort = r.consulClientHttpPort + } + ccCfg.Address = fmt.Sprintf("%s:%d", pod.Status.HostIP, consulClientHttpPort) + + ccCfg.Token = state.Token + + // Check if auto-encrypt is enabled. If it is, we need to retrieve and set a different CA for the Consul client. + if r.EnableAutoEncrypt { + // Get Connect CA. + caRoots, _, err := serverClient.Agent().ConnectCARoots(nil) + if err != nil { + return nil, err + } + if caRoots == nil { + return nil, fmt.Errorf("ca root list is nil") + } + if caRoots.Roots == nil { + return nil, fmt.Errorf("ca roots is nil") + } + if len(caRoots.Roots) == 0 { + return nil, fmt.Errorf("the list of root CAs is empty") + } + + for _, root := range caRoots.Roots { + if root.Active { + ccCfg.TLSConfig.CAFile = "" + ccCfg.TLSConfig.CAPem = []byte(root.RootCertPEM) + break + } + } + } + if r.EnableConsulNamespaces { + ccCfg.Namespace = r.consulNamespace(pod.Namespace) + } + return ccCfg, nil +} + +func (r *Controller) updateHealthCheckOnConsulClient(consulClientCfg *api.Config, pod corev1.Pod, endpoints corev1.Endpoints, status string) error { + consulClient, err := consul.NewClient(consulClientCfg, r.ConsulClientConfig.APITimeout) + if err != nil { + return err + } + filter := fmt.Sprintf(`Name == "Kubernetes Health Check" and ServiceID == %q`, serviceID(pod, endpoints)) + checks, err := consulClient.Agent().ChecksWithFilter(filter) + if err != nil { + return err + } + if len(checks) > 1 { + return fmt.Errorf("more than one Kubernetes health check found") + } + if len(checks) == 0 { + r.Log.Info("detected no health checks to update", "name", endpoints.Name, "ns", endpoints.Namespace, "service-id", serviceID(pod, endpoints)) + return nil + } + for checkID := range checks { + output := "Kubernetes health checks passing" + if status == api.HealthCritical { + output = fmt.Sprintf(`Pod "%s/%s" is not ready`, pod.Namespace, pod.Name) + } + r.Log.Info("updating health check status", "name", endpoints.Name, "ns", endpoints.Namespace, "status", status) + err = consulClient.Agent().UpdateTTL(checkID, output, status) + if err != nil { + return err + } + } + return nil +} diff --git a/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks_test.go b/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks_test.go new file mode 100644 index 0000000000..da42a2fa4c --- /dev/null +++ b/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks_test.go @@ -0,0 +1,260 @@ +package endpoints + +import ( + "testing" + + logrtest "github.com/go-logr/logr/testing" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" + "github.com/hashicorp/consul-server-connection-manager/discovery" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIsConsulDataplaneSupported(t *testing.T) { + versions := map[string]struct { + expIsConsulDataplaneSupported bool + }{ + "": {false}, + "v1.0.0": {true}, + "1.0.0": {true}, + "v0.49.0": {false}, + "0.49.0-beta2": {false}, + "0.49.2": {false}, + "v1.0.0-beta1": {true}, + "v1.0.0-beta3": {true}, + "v1.1.0-beta1": {true}, + "v1.0.0-dev": {true}, + "v1.0.0-dev+abcdef": {true}, + "invalid": {false}, + } + + for version, c := range versions { + t.Run(version, func(t *testing.T) { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{}, + }, + } + if version != "" { + pod.ObjectMeta.Annotations[constants.AnnotationConsulK8sVersion] = version + } + + require.Equal(t, c.expIsConsulDataplaneSupported, isConsulDataplaneSupported(pod)) + }) + } +} + +func TestConsulClientForNodeAgent(t *testing.T) { + cases := map[string]struct { + tls bool + autoEncrypt bool + enableNamespaces bool + }{ + "no tls and auto-encrypt": {}, + "with tls but no auto-encrypt": {tls: true}, + "with tls and auto-encrypt": {tls: true, autoEncrypt: true}, + "with namespaces": {enableNamespaces: true}, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + // Create test Consul server. + testClient := test.TestServerWithMockConnMgrWatcher(t, func(c *testutil.TestServerConfig) { + c.Connect["enabled"] = true + }) + testClient.TestServer.WaitForActiveCARoot(t) + if c.tls { + testClient.Cfg.APIClientConfig.Scheme = "https" + } + + ctrl := Controller{ + ConsulClientConfig: testClient.Cfg, + EnableConsulNamespaces: c.enableNamespaces, + // We are only testing with mirroring enabled because other cases are tested elsewhere. + EnableNSMirroring: true, + EnableAutoEncrypt: c.autoEncrypt, + } + + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "test-ns", + }, + Status: corev1.PodStatus{ + HostIP: "1.2.3.4", + }, + } + + ccCfg, err := ctrl.consulClientCfgForNodeAgent(testClient.APIClient, pod, discovery.State{Token: "test-token"}) + require.NoError(t, err) + require.Equal(t, "test-token", ccCfg.Token) + if c.tls { + require.Equal(t, "https", ccCfg.Scheme) + require.Equal(t, "1.2.3.4:8501", ccCfg.Address) + require.Empty(t, ccCfg.TLSConfig.Address) + if c.autoEncrypt { + caRoots, _, err := testClient.APIClient.Agent().ConnectCARoots(nil) + require.NoError(t, err) + require.Equal(t, []byte(caRoots.Roots[0].RootCertPEM), ccCfg.TLSConfig.CAPem) + } + } else { + require.Equal(t, ccCfg.Address, "1.2.3.4:8500") + } + + if c.enableNamespaces { + require.Equal(t, "test-ns", ccCfg.Namespace) + } + }) + } +} + +func TestUpdateHealthCheckOnConsulClient(t *testing.T) { + cases := map[string]struct { + checks []*api.AgentServiceCheck + updateToStatus string + expError string + }{ + "service with one existing kubernetes health check becoming unhealthy": { + checks: []*api.AgentServiceCheck{ + { + CheckID: "default/test-pod-test-service/kubernetes-health-check", + Name: "Kubernetes Health Check", + TTL: "100000h", + Status: api.HealthPassing, + SuccessBeforePassing: 1, + FailuresBeforeCritical: 1, + }, + }, + updateToStatus: api.HealthCritical, + }, + "service with one existing kubernetes health check becoming healthy": { + checks: []*api.AgentServiceCheck{ + { + CheckID: "default/test-pod-test-service/kubernetes-health-check", + Name: "Kubernetes Health Check", + TTL: "100000h", + Status: api.HealthCritical, + SuccessBeforePassing: 1, + FailuresBeforeCritical: 1, + }, + }, + updateToStatus: api.HealthPassing, + }, + "service without health check is a no-op": { + checks: nil, + updateToStatus: api.HealthPassing, + }, + "service with more than one existing kubernetes health check becoming healthy": { + checks: []*api.AgentServiceCheck{ + { + CheckID: "default/test-pod-test-service/kubernetes-health-check", + Name: "Kubernetes Health Check", + TTL: "100000h", + Status: api.HealthCritical, + SuccessBeforePassing: 1, + FailuresBeforeCritical: 1, + }, + { + CheckID: "default/test-pod-test-service/kubernetes-health-check-2", + Name: "Kubernetes Health Check", + TTL: "100000h", + Status: api.HealthPassing, + SuccessBeforePassing: 1, + FailuresBeforeCritical: 1, + }, + }, + updateToStatus: api.HealthPassing, + expError: "more than one Kubernetes health check found", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + testClient := test.TestServerWithMockConnMgrWatcher(t, nil) + consulClient := testClient.APIClient + + consulSvcs := []*api.AgentServiceRegistration{ + { + ID: "test-pod-test-service", + Name: "test-service", + Port: 80, + Address: "1.2.3.4", + Checks: c.checks, + }, + { + Kind: api.ServiceKindConnectProxy, + ID: "test-pod-test-service-sidecar-proxy", + Name: "test-service-sidecar-proxy", + Port: 20000, + Address: "1.2.3.4", + Proxy: &api.AgentServiceConnectProxyConfig{ + DestinationServiceName: "test-service", + DestinationServiceID: "test-pod-test-service", + }, + }, + } + + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Status: corev1.PodStatus{ + PodIP: "1.2.3.4", + }, + } + endpoints := corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + Namespace: "default", + }, + Subsets: []corev1.EndpointSubset{ + { + NotReadyAddresses: []corev1.EndpointAddress{ + { + IP: "1.2.3.4", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "test-pod", + Namespace: "default", + }, + }, + }, + }, + }, + } + + for _, svc := range consulSvcs { + err := consulClient.Agent().ServiceRegister(svc) + require.NoError(t, err) + } + + ctrl := Controller{ + ConsulClientConfig: testClient.Cfg, + Log: logrtest.TestLogger{T: t}, + } + + err := ctrl.updateHealthCheckOnConsulClient(testClient.Cfg.APIClientConfig, pod, endpoints, c.updateToStatus) + if c.expError == "" { + require.NoError(t, err) + status, agentHealthInfo, err := consulClient.Agent().AgentHealthServiceByName("test-service") + require.NoError(t, err) + if c.checks != nil { + require.NotEmpty(t, agentHealthInfo) + require.Equal(t, c.updateToStatus, status) + } else { + require.Empty(t, agentHealthInfo[0].Checks) + } + } else { + require.EqualError(t, err, c.expError) + } + }) + } + +} diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go index bbdc9bdcf1..6eb05c30d9 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller.go @@ -113,11 +113,18 @@ type Controller struct { // whenever service instances are deregistered. AuthMethod string + // EnableAutoEncrypt indicates whether we should use auto-encrypt when talking + // to Consul client agents. + EnableAutoEncrypt bool + MetricsConfig metrics.Config Log logr.Logger Scheme *runtime.Scheme context.Context + + // consulClientHttpPort is only used in tests. + consulClientHttpPort int } // Reconcile reads the state of an Endpoints object for a Kubernetes Service and reconciles Consul services which @@ -199,9 +206,29 @@ func (r *Controller) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if hasBeenInjected(pod) { endpointPods.Add(address.TargetRef.Name) - if err = r.registerServicesAndHealthCheck(apiClient, pod, serviceEndpoints, healthStatus, endpointAddressMap); err != nil { - r.Log.Error(err, "failed to register services or health check", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) - errs = multierror.Append(errs, err) + if isConsulDataplaneSupported(pod) { + if err = r.registerServicesAndHealthCheck(apiClient, pod, serviceEndpoints, healthStatus, endpointAddressMap); err != nil { + r.Log.Error(err, "failed to register services or health check", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) + errs = multierror.Append(errs, err) + } + } else { + r.Log.Info("detected an update to pre-consul-dataplane service", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) + nodeAgentClientCfg, err := r.consulClientCfgForNodeAgent(apiClient, pod, serverState) + if err != nil { + r.Log.Error(err, "failed to create node-local Consul API client", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) + errs = multierror.Append(errs, err) + continue + } + r.Log.Info("updating health check on the Consul client", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) + if err = r.updateHealthCheckOnConsulClient(nodeAgentClientCfg, pod, serviceEndpoints, healthStatus); err != nil { + r.Log.Error(err, "failed to update health check on Consul client", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace, "consul-client-ip", pod.Status.HostIP) + errs = multierror.Append(errs, err) + } + // We want to skip the rest of the reconciliation because we only care about updating health checks for existing services + // in the case when Consul clients are running in the cluster. If endpoints are deleted, consul clients + // will detect that they are unhealthy, and we don't need to worry about keeping them up-to-date. + // This is so that health checks are still updated during an upgrade to consul-dataplane. + continue } } if isGateway(pod) { @@ -249,7 +276,7 @@ func (r *Controller) registerServicesAndHealthCheck(apiClient *api.Client, pod c // For pods managed by this controller, create and register the service instance. if managedByEndpointsController { // Get information from the pod to create service instance registrations. - serviceRegistration, proxyServiceRegistration, err := r.createServiceRegistrations(apiClient, pod, serviceEndpoints, healthStatus) + serviceRegistration, proxyServiceRegistration, err := r.createServiceRegistrations(pod, serviceEndpoints, healthStatus) if err != nil { r.Log.Error(err, "failed to create service registrations for endpoints", "name", serviceEndpoints.Name, "ns", serviceEndpoints.Namespace) return err @@ -332,18 +359,18 @@ func serviceID(pod corev1.Pod, serviceEndpoints corev1.Endpoints) string { } func proxyServiceName(pod corev1.Pod, serviceEndpoints corev1.Endpoints) string { - serviceName := serviceName(pod, serviceEndpoints) - return fmt.Sprintf("%s-sidecar-proxy", serviceName) + svcName := serviceName(pod, serviceEndpoints) + return fmt.Sprintf("%s-sidecar-proxy", svcName) } func proxyServiceID(pod corev1.Pod, serviceEndpoints corev1.Endpoints) string { - proxyServiceName := proxyServiceName(pod, serviceEndpoints) - return fmt.Sprintf("%s-%s", pod.Name, proxyServiceName) + proxySvcName := proxyServiceName(pod, serviceEndpoints) + return fmt.Sprintf("%s-%s", pod.Name, proxySvcName) } // createServiceRegistrations creates the service and proxy service instance registrations with the information from the // Pod. -func (r *Controller) createServiceRegistrations(apiClient *api.Client, pod corev1.Pod, serviceEndpoints corev1.Endpoints, healthStatus string) (*api.CatalogRegistration, *api.CatalogRegistration, error) { +func (r *Controller) createServiceRegistrations(pod corev1.Pod, serviceEndpoints corev1.Endpoints, healthStatus string) (*api.CatalogRegistration, *api.CatalogRegistration, error) { // If a port is specified, then we determine the value of that port // and register that port for the host service. // The meshWebhook will always set the port annotation if one is not provided on the pod. diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go index a367aa791d..2290c4432d 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_ent_test.go @@ -2109,10 +2109,12 @@ func TestReconcileDeleteGatewayWithNamespaces(t *testing.T) { func createPodWithNamespace(name, namespace, ip string, inject bool, managedByEndpointsController bool) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{}, - Annotations: map[string]string{}, + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{ + constants.AnnotationConsulK8sVersion: "1.0.0", + }, }, Status: corev1.PodStatus{ PodIP: ip, diff --git a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go index d56f99407d..09bf48293e 100644 --- a/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go +++ b/control-plane/connect-inject/controllers/endpoints/endpoints_controller_test.go @@ -3418,6 +3418,209 @@ func TestReconcileUpdateEndpoint(t *testing.T) { } } +// TestReconcileUpdateEndpoint_LegacyService tests that we can update health checks on a consul client. +func TestReconcileUpdateEndpoint_LegacyService(t *testing.T) { + t.Parallel() + cases := []struct { + name string + k8sObjects func() []runtime.Object + initialConsulSvcs []*api.AgentServiceRegistration + expectedHealthChecks []*api.AgentCheck + }{ + { + name: "Health check changes from unhealthy to healthy", + k8sObjects: func() []runtime.Object { + pod1 := createServicePod("pod1", "1.2.3.4", true, true) + pod1.Status.HostIP = "127.0.0.1" + pod1.Annotations[constants.AnnotationConsulK8sVersion] = "0.99.0" // We want a version less than 1.0.0. + endpoint := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-updated", + Namespace: "default", + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "1.2.3.4", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "pod1", + Namespace: "default", + }, + }, + }, + }, + }, + } + return []runtime.Object{pod1, endpoint} + }, + initialConsulSvcs: []*api.AgentServiceRegistration{ + { + ID: "pod1-service-updated", + Name: "service-updated", + Port: 80, + Address: "1.2.3.4", + Check: &api.AgentServiceCheck{ + CheckID: "default/pod1-service-updated/kubernetes-health-check", + TTL: "100000h", + Name: "Kubernetes Health Check", + Status: api.HealthCritical, + }, + }, + { + Kind: api.ServiceKindConnectProxy, + ID: "pod1-service-updated-sidecar-proxy", + Name: "service-updated-sidecar-proxy", + Port: 20000, + Address: "1.2.3.4", + Proxy: &api.AgentServiceConnectProxyConfig{ + DestinationServiceName: "service-updated", + DestinationServiceID: "pod1-service-updated", + }, + }, + }, + expectedHealthChecks: []*api.AgentCheck{ + { + CheckID: "default/pod1-service-updated/kubernetes-health-check", + ServiceName: "service-updated", + ServiceID: "pod1-service-updated", + Name: "Kubernetes Health Check", + Status: api.HealthPassing, + Output: "Kubernetes health checks passing", + Type: "ttl", + }, + }, + }, + { + name: "Health check changes from healthy to unhealthy", + k8sObjects: func() []runtime.Object { + pod1 := createServicePod("pod1", "1.2.3.4", true, true) + pod1.Status.HostIP = "127.0.0.1" + pod1.Annotations[constants.AnnotationConsulK8sVersion] = "0.99.0" // We want a version less than 1.0.0. + endpoint := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-updated", + Namespace: "default", + }, + Subsets: []corev1.EndpointSubset{ + { + NotReadyAddresses: []corev1.EndpointAddress{ + { + IP: "1.2.3.4", + TargetRef: &corev1.ObjectReference{ + Kind: "Pod", + Name: "pod1", + Namespace: "default", + }, + }, + }, + }, + }, + } + return []runtime.Object{pod1, endpoint} + }, + initialConsulSvcs: []*api.AgentServiceRegistration{ + { + ID: "pod1-service-updated", + Name: "service-updated", + Port: 80, + Address: "1.2.3.4", + Check: &api.AgentServiceCheck{ + CheckID: "default/pod1-service-updated/kubernetes-health-check", + TTL: "100000h", + Name: "Kubernetes Health Check", + Status: api.HealthPassing, + }, + }, + { + Kind: api.ServiceKindConnectProxy, + ID: "pod1-service-updated-sidecar-proxy", + Name: "service-updated-sidecar-proxy", + Port: 20000, + Address: "1.2.3.4", + Proxy: &api.AgentServiceConnectProxyConfig{ + DestinationServiceName: "service-updated", + DestinationServiceID: "pod1-service-updated", + }, + }, + }, + expectedHealthChecks: []*api.AgentCheck{ + { + CheckID: "default/pod1-service-updated/kubernetes-health-check", + ServiceName: "service-updated", + ServiceID: "pod1-service-updated", + Name: "Kubernetes Health Check", + Status: api.HealthCritical, + Output: "Pod \"default/pod1\" is not ready", + Type: "ttl", + }, + }, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + // Add the default namespace. + ns := corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "default"}} + // Create fake k8s client. + k8sObjects := append(tt.k8sObjects(), &ns) + fakeClient := fake.NewClientBuilder().WithRuntimeObjects(k8sObjects...).Build() + + // Create test consulServer server + testClient := test.TestServerWithMockConnMgrWatcher(t, nil) + + // Create a consul client joined with this server. + var consulClientHttpPort int + consulClientAgent, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) { + c.Server = false + c.Bootstrap = false + consulClientHttpPort = c.Ports.HTTP + }) + require.NoError(t, err) + consulClientAgent.JoinLAN(t, testClient.TestServer.LANAddr) + consulClientAgent.WaitForSerfCheck(t) + + consulClient, err := api.NewClient(&api.Config{Address: consulClientAgent.HTTPAddr}) + require.NoError(t, err) + + // Register service and proxy in consul. + for _, svc := range tt.initialConsulSvcs { + err := consulClient.Agent().ServiceRegister(svc) + require.NoError(t, err) + } + + // Create the endpoints controller. + ep := &Controller{ + Client: fakeClient, + Log: logrtest.TestLogger{T: t}, + ConsulClientConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSetWith(), + ReleaseName: "consul", + ReleaseNamespace: "default", + consulClientHttpPort: consulClientHttpPort, + } + namespacedName := types.NamespacedName{Namespace: "default", Name: "service-updated"} + + resp, err := ep.Reconcile(context.Background(), ctrl.Request{NamespacedName: namespacedName}) + require.NoError(t, err) + require.False(t, resp.Requeue) + + // After reconciliation, Consul should have service-updated with the correct health check status. + for _, expectedCheck := range tt.expectedHealthChecks { + filter := fmt.Sprintf("ServiceID == %q", expectedCheck.ServiceID) + checks, err := consulClient.Agent().ChecksWithFilter(filter) + require.NoError(t, err) + require.Equal(t, 1, len(checks)) + // Ignoring Namespace because the response from ENT includes it and OSS does not. + var ignoredFields = []string{"Node", "Definition", "Namespace", "Partition"} + require.True(t, cmp.Equal(checks[expectedCheck.CheckID], expectedCheck, cmpopts.IgnoreFields(api.AgentCheck{}, ignoredFields...))) + } + }) + } +} + // Tests deleting an Endpoints object, with and without matching Consul and K8s service names. // This test covers Controller.deregisterService when the map is nil (not selectively deregistered). func TestReconcileDeleteEndpoint(t *testing.T) { @@ -5473,7 +5676,7 @@ func TestCreateServiceRegistrations_withTransparentProxy(t *testing.T) { Log: logrtest.TestLogger{T: t}, } - serviceRegistration, proxyServiceRegistration, err := epCtrl.createServiceRegistrations(nil, *pod, *endpoints, api.HealthPassing) + serviceRegistration, proxyServiceRegistration, err := epCtrl.createServiceRegistrations(*pod, *endpoints, api.HealthPassing) if c.expErr != "" { require.EqualError(t, err, c.expErr) } else { @@ -6119,10 +6322,12 @@ func Test_GetWANData(t *testing.T) { func createServicePod(name, ip string, inject bool, managedByEndpointsController bool) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: "default", - Labels: map[string]string{}, - Annotations: map[string]string{}, + Name: name, + Namespace: "default", + Labels: map[string]string{}, + Annotations: map[string]string{ + constants.AnnotationConsulK8sVersion: "1.0.0", + }, }, Status: corev1.PodStatus{ PodIP: ip, diff --git a/control-plane/connect-inject/webhook/mesh_webhook.go b/control-plane/connect-inject/webhook/mesh_webhook.go index e0cac079de..503d3182b4 100644 --- a/control-plane/connect-inject/webhook/mesh_webhook.go +++ b/control-plane/connect-inject/webhook/mesh_webhook.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/hashicorp/consul-k8s/control-plane/version" "gomodules.xyz/jsonpatch/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -575,6 +576,7 @@ func (w *MeshWebhook) defaultAnnotations(pod *corev1.Pod, podJson string) error } } pod.Annotations[constants.AnnotationOriginalPod] = podJson + pod.Annotations[constants.AnnotationConsulK8sVersion] = version.GetHumanVersion() return nil } diff --git a/control-plane/connect-inject/webhook/mesh_webhook_test.go b/control-plane/connect-inject/webhook/mesh_webhook_test.go index 58620e32ad..602ba63239 100644 --- a/control-plane/connect-inject/webhook/mesh_webhook_test.go +++ b/control-plane/connect-inject/webhook/mesh_webhook_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/hashicorp/consul-k8s/control-plane/version" "github.com/stretchr/testify/require" "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" @@ -171,6 +172,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/spec/volumes", @@ -260,6 +265,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -311,6 +320,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -381,6 +394,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -437,6 +454,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -518,6 +539,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -651,6 +676,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "replace", Path: "/spec/containers/0/livenessProbe/httpGet/port", @@ -710,6 +739,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -765,6 +798,10 @@ func TestHandlerHandle(t *testing.T) { Operation: "add", Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, { Operation: "add", Path: "/metadata/labels", @@ -808,7 +845,8 @@ func TestHandlerDefaultAnnotations(t *testing.T) { "empty", &corev1.Pod{}, map[string]string{ - constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":null},\"status\":{}}", + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":null},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), }, "", }, @@ -828,7 +866,8 @@ func TestHandlerDefaultAnnotations(t *testing.T) { }, }, map[string]string{ - constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), }, "", }, @@ -856,6 +895,7 @@ func TestHandlerDefaultAnnotations(t *testing.T) { map[string]string{ "consul.hashicorp.com/connect-service": "foo", constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"consul.hashicorp.com/connect-service\":\"foo\"}},\"spec\":{\"containers\":[{\"name\":\"web\",\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), }, "", @@ -882,8 +922,9 @@ func TestHandlerDefaultAnnotations(t *testing.T) { }, }, map[string]string{ - constants.AnnotationPort: "http", - constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"ports\":[{\"name\":\"http\",\"containerPort\":8080}],\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationPort: "http", + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"ports\":[{\"name\":\"http\",\"containerPort\":8080}],\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), }, "", }, @@ -908,8 +949,9 @@ func TestHandlerDefaultAnnotations(t *testing.T) { }, }, map[string]string{ - constants.AnnotationPort: "8080", - constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"ports\":[{\"containerPort\":8080}],\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationPort: "8080", + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"ports\":[{\"containerPort\":8080}],\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), }, "", }, @@ -917,10 +959,8 @@ func TestHandlerDefaultAnnotations(t *testing.T) { for _, tt := range cases { t.Run(tt.Name, func(t *testing.T) { - require := require.New(t) - podJson, err := json.Marshal(tt.Pod) - require.NoError(err) + require.NoError(t, err) var w MeshWebhook err = w.defaultAnnotations(tt.Pod, string(podJson)) @@ -928,7 +968,7 @@ func TestHandlerDefaultAnnotations(t *testing.T) { t.Fatalf("actual: %v, expected err: %v", err, tt.Err) } if tt.Err != "" { - require.Contains(err.Error(), tt.Err) + require.Contains(t, err.Error(), tt.Err) return } @@ -936,7 +976,7 @@ func TestHandlerDefaultAnnotations(t *testing.T) { if len(actual) == 0 { actual = nil } - require.Equal(tt.Expected, actual) + require.Equal(t, tt.Expected, actual) }) } } diff --git a/control-plane/go.mod b/control-plane/go.mod index 3aef2d1424..0ad9ba992b 100644 --- a/control-plane/go.mod +++ b/control-plane/go.mod @@ -17,6 +17,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-netaddrs v0.0.0-20220509001840-90ed9d26ec46 github.com/hashicorp/go-rootcerts v1.0.2 + github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/serf v0.10.1 github.com/kr/text v0.2.0 github.com/miekg/dns v1.1.41 diff --git a/control-plane/go.sum b/control-plane/go.sum index 41ed7e8566..a4106ab324 100644 --- a/control-plane/go.sum +++ b/control-plane/go.sum @@ -389,6 +389,8 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/control-plane/subcommand/connect-init/command_test.go b/control-plane/subcommand/connect-init/command_test.go index bc45bcd335..7736e13796 100644 --- a/control-plane/subcommand/connect-init/command_test.go +++ b/control-plane/subcommand/connect-init/command_test.go @@ -812,7 +812,7 @@ func TestRun_TrafficRedirection(t *testing.T) { t.Run(name, func(t *testing.T) { proxyFile := fmt.Sprintf("/tmp/%d", rand.Int()) t.Cleanup(func() { - _ = os.Remove(proxyFile) + _ = os.RemoveAll(proxyFile) }) // Start Consul server. diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index 1b210b5f00..cb4b619a07 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -83,12 +83,6 @@ type Command struct { flagDefaultPrometheusScrapePort string flagDefaultPrometheusScrapePath string - // Consul sidecar resource settings. - flagDefaultConsulSidecarCPULimit string - flagDefaultConsulSidecarCPURequest string - flagDefaultConsulSidecarMemoryLimit string - flagDefaultConsulSidecarMemoryRequest string - // Init container resource settings. flagInitContainerCPULimit string flagInitContainerCPURequest string @@ -108,6 +102,8 @@ type Command struct { // WAN Federation flags. flagEnableFederation bool + flagEnableAutoEncrypt bool + // Consul DNS flags. flagEnableConsulDNS bool flagResourcePrefix string @@ -187,6 +183,8 @@ func (c *Command) init() { "Indicates that the command runs in an OpenShift cluster.") c.flagSet.BoolVar(&c.flagEnableWebhookCAUpdate, "enable-webhook-ca-update", false, "Enables updating the CABundle on the webhook within this controller rather than using the web cert manager.") + c.flagSet.BoolVar(&c.flagEnableAutoEncrypt, "enable-auto-encrypt", false, + "Indicates whether TLS with auto-encrypt should be used when talking to Consul clients.") c.flagSet.StringVar(&c.flagLogLevel, "log-level", zapcore.InfoLevel.String(), fmt.Sprintf("Log verbosity level. Supported values (in order of detail) are "+ "%q, %q, %q, and %q.", zapcore.DebugLevel.String(), zapcore.InfoLevel.String(), zapcore.WarnLevel.String(), zapcore.ErrorLevel.String())) @@ -213,11 +211,6 @@ func (c *Command) init() { c.flagSet.StringVar(&c.flagInitContainerMemoryRequest, "init-container-memory-request", "25Mi", "Init container memory request.") c.flagSet.StringVar(&c.flagInitContainerMemoryLimit, "init-container-memory-limit", "150Mi", "Init container memory limit.") - // Consul sidecar resource setting flags. - c.flagSet.StringVar(&c.flagDefaultConsulSidecarCPURequest, "default-consul-sidecar-cpu-request", "20m", "Default consul sidecar CPU request.") - c.flagSet.StringVar(&c.flagDefaultConsulSidecarCPULimit, "default-consul-sidecar-cpu-limit", "20m", "Default consul sidecar CPU limit.") - c.flagSet.StringVar(&c.flagDefaultConsulSidecarMemoryRequest, "default-consul-sidecar-memory-request", "25Mi", "Default consul sidecar memory request.") - c.flagSet.StringVar(&c.flagDefaultConsulSidecarMemoryLimit, "default-consul-sidecar-memory-limit", "50Mi", "Default consul sidecar memory limit.") c.flagSet.IntVar(&c.flagDefaultEnvoyProxyConcurrency, "default-envoy-proxy-concurrency", 2, "Default Envoy proxy concurrency.") c.consul = &flags.ConsulFlags{} @@ -436,6 +429,7 @@ func (c *Command) Run(args []string) int { Scheme: mgr.GetScheme(), ReleaseName: c.flagReleaseName, ReleaseNamespace: c.flagReleaseNamespace, + EnableAutoEncrypt: c.flagEnableAutoEncrypt, Context: ctx, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", endpoints.Controller{}) From 39ce48a3d2850282812e99455c3d868352fb5b9d Mon Sep 17 00:00:00 2001 From: Iryna Shustava Date: Thu, 10 Nov 2022 11:12:58 -0700 Subject: [PATCH 5/5] changelog and review comments --- CHANGELOG.md | 1 + .../controllers/endpoints/consul_client_health_checks.go | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1556070952..6374c7538a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ IMPROVEMENTS: * Control Plane * Update minimum go version for project to 1.19 [[GH-1633](https://github.com/hashicorp/consul-k8s/pull/1633)] * Remove unneeded `agent:read` ACL permissions from mesh gateway policy. [[GH-1255](https://github.com/hashicorp/consul-k8s/pull/1255)] + * Support updating health checks on consul clients during an upgrade to agentless. [[GH-1690](https://github.com/hashicorp/consul-k8s/pull/1690)] * Helm: * Remove deprecated annotation `service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"` in the `server-service` template. [[GH-1619](https://github.com/hashicorp/consul-k8s/pull/1619)] * Support `minAvailable` on connect injector `PodDisruptionBudget`. [[GH-1557](https://github.com/hashicorp/consul-k8s/pull/1557)] diff --git a/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go b/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go index 3496ee2212..0ab1b9f79f 100644 --- a/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go +++ b/control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go @@ -11,6 +11,8 @@ import ( corev1 "k8s.io/api/core/v1" ) +const minSupportedConsulDataplaneVersion = "v1.0.0-beta1" + // isConsulDataplaneSupported returns true if the consul-k8s version on the pod supports // consul-dataplane architecture of Consul. func isConsulDataplaneSupported(pod corev1.Pod) bool { @@ -19,7 +21,7 @@ func isConsulDataplaneSupported(pod corev1.Pod) bool { if err != nil { return false } - consulDPSupportedVersion, err := version.NewVersion("v1.0.0-beta1") + consulDPSupportedVersion, err := version.NewVersion(minSupportedConsulDataplaneVersion) if err != nil { return false }