From c73605efb9a58ce0962688829662140f68ec0dad Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesh Date: Tue, 12 Sep 2023 12:08:10 -0400 Subject: [PATCH] Create mesh webhook to support v2 resources (#2930) * mesh webhook v2 --- .../constants/annotations_and_labels.go | 6 + .../metrics/metrics_configuration.go | 3 +- .../webhook_v2/consul_dataplane_sidecar.go | 471 ++++ .../consul_dataplane_sidecar_test.go | 1103 +++++++++ .../webhook_v2/container_env.go | 37 + .../webhook_v2/container_env_test.go | 58 + .../webhook_v2/container_init.go | 300 +++ .../webhook_v2/container_init_test.go | 844 +++++++ .../webhook_v2/container_volume.go | 23 + .../connect-inject/webhook_v2/dns.go | 93 + .../connect-inject/webhook_v2/dns_test.go | 105 + .../webhook_v2/health_checks_test.go | 56 + .../connect-inject/webhook_v2/heath_checks.go | 30 + .../connect-inject/webhook_v2/mesh_webhook.go | 563 +++++ .../webhook_v2/mesh_webhook_ent_test.go | 656 ++++++ .../webhook_v2/mesh_webhook_test.go | 2043 +++++++++++++++++ .../webhook_v2/redirect_traffic.go | 137 ++ .../webhook_v2/redirect_traffic_test.go | 481 ++++ .../inject-connect/v2controllers.go | 72 +- 19 files changed, 7071 insertions(+), 10 deletions(-) create mode 100644 control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar.go create mode 100644 control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar_test.go create mode 100644 control-plane/connect-inject/webhook_v2/container_env.go create mode 100644 control-plane/connect-inject/webhook_v2/container_env_test.go create mode 100644 control-plane/connect-inject/webhook_v2/container_init.go create mode 100644 control-plane/connect-inject/webhook_v2/container_init_test.go create mode 100644 control-plane/connect-inject/webhook_v2/container_volume.go create mode 100644 control-plane/connect-inject/webhook_v2/dns.go create mode 100644 control-plane/connect-inject/webhook_v2/dns_test.go create mode 100644 control-plane/connect-inject/webhook_v2/health_checks_test.go create mode 100644 control-plane/connect-inject/webhook_v2/heath_checks.go create mode 100644 control-plane/connect-inject/webhook_v2/mesh_webhook.go create mode 100644 control-plane/connect-inject/webhook_v2/mesh_webhook_ent_test.go create mode 100644 control-plane/connect-inject/webhook_v2/mesh_webhook_test.go create mode 100644 control-plane/connect-inject/webhook_v2/redirect_traffic.go create mode 100644 control-plane/connect-inject/webhook_v2/redirect_traffic_test.go diff --git a/control-plane/connect-inject/constants/annotations_and_labels.go b/control-plane/connect-inject/constants/annotations_and_labels.go index 5c14cb15f2..cd563f2436 100644 --- a/control-plane/connect-inject/constants/annotations_and_labels.go +++ b/control-plane/connect-inject/constants/annotations_and_labels.go @@ -229,6 +229,12 @@ const ( // ManagedByPodValue is used in Consul metadata to identify the manager // of resources. ManagedByPodValue = "consul-k8s-pod-controller" + + // AnnotationMeshDestinations is a list of upstreams to register with the + // proxy. The service name should map to a Consul service namd and the local + // port is the local port in the pod that the listener will bind to. It can + // be a named port. + AnnotationMeshDestinations = "consul.hashicorp.com/mesh-service-destinations" ) // Annotations used by Prometheus. diff --git a/control-plane/connect-inject/metrics/metrics_configuration.go b/control-plane/connect-inject/metrics/metrics_configuration.go index 6f9c29c85b..2f217f233d 100644 --- a/control-plane/connect-inject/metrics/metrics_configuration.go +++ b/control-plane/connect-inject/metrics/metrics_configuration.go @@ -8,9 +8,10 @@ import ( "fmt" "strconv" + corev1 "k8s.io/api/core/v1" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" - corev1 "k8s.io/api/core/v1" ) // Config represents configuration common to connect-inject components related to metrics. diff --git a/control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar.go b/control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar.go new file mode 100644 index 0000000000..5576f919d4 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar.go @@ -0,0 +1,471 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/google/shlex" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" +) + +const ( + consulDataplaneDNSBindHost = "127.0.0.1" + consulDataplaneDNSBindPort = 8600 +) + +func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod corev1.Pod) (corev1.Container, error) { + resources, err := w.sidecarResources(pod) + if err != nil { + return corev1.Container{}, err + } + + // Extract the service account token's volume mount. + var bearerTokenFile string + var saTokenVolumeMount corev1.VolumeMount + if w.AuthMethod != "" { + saTokenVolumeMount, bearerTokenFile, err = findServiceAccountVolumeMount(pod) + if err != nil { + return corev1.Container{}, err + } + } + + args, err := w.getContainerSidecarArgs(namespace, bearerTokenFile, pod) + if err != nil { + return corev1.Container{}, err + } + + containerName := sidecarContainer + + var probe *corev1.Probe + if useProxyHealthCheck(pod) { + // If using the proxy health check for a service, configure an HTTP handler + // that queries the '/ready' endpoint of the proxy. + probe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(constants.ProxyDefaultHealthPort), + Path: "/ready", + }, + }, + InitialDelaySeconds: 1, + } + } else { + probe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(constants.ProxyDefaultInboundPort), + }, + }, + InitialDelaySeconds: 1, + } + } + + container := corev1.Container{ + Name: containerName, + Image: w.ImageConsulDataplane, + Resources: resources, + // We need to set tmp dir to an ephemeral volume that we're mounting so that + // consul-dataplane can write files to it. Otherwise, it wouldn't be able to + // because we set file system to be read-only. + Env: []corev1.EnvVar{ + { + Name: "TMPDIR", + Value: "/consul/mesh-inject", + }, + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + // The pod name isn't known currently, so we must rely on the environment variable to fill it in rather than using args. + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + { + Name: "DP_PROXY_ID", + Value: "$(POD_NAME)", + }, + { + Name: "DP_CREDENTIAL_LOGIN_META", + Value: "pod=$(POD_NAMESPACE)/$(POD_NAME)", + }, + // This entry exists to support certain versions of consul dataplane, where environment variable entries + // utilize this numbered notation to indicate individual KV pairs in a map. + { + Name: "DP_CREDENTIAL_LOGIN_META1", + Value: "pod=$(POD_NAMESPACE)/$(POD_NAME)", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: volumeName, + MountPath: "/consul/mesh-inject", + }, + }, + Args: args, + ReadinessProbe: probe, + } + + if w.AuthMethod != "" { + container.VolumeMounts = append(container.VolumeMounts, saTokenVolumeMount) + } + + if useProxyHealthCheck(pod) { + // Configure the Readiness Address for the proxy's health check to be the Pod IP. + container.Env = append(container.Env, corev1.EnvVar{ + Name: "DP_ENVOY_READY_BIND_ADDRESS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }, + }) + // Configure the port on which the readiness probe will query the proxy for its health. + container.Ports = append(container.Ports, corev1.ContainerPort{ + Name: "proxy-health", + ContainerPort: int32(constants.ProxyDefaultHealthPort), + }) + } + + // Add any extra VolumeMounts. + if userVolMount, ok := pod.Annotations[constants.AnnotationConsulSidecarUserVolumeMount]; ok { + var volumeMounts []corev1.VolumeMount + err := json.Unmarshal([]byte(userVolMount), &volumeMounts) + if err != nil { + return corev1.Container{}, err + } + container.VolumeMounts = append(container.VolumeMounts, volumeMounts...) + } + + tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy) + if err != nil { + return corev1.Container{}, err + } + + // If not running in transparent proxy mode and in an OpenShift environment, + // skip setting the security context and let OpenShift set it for us. + // When transparent proxy is enabled, then consul-dataplane needs to run as our specific user + // so that traffic redirection will work. + if tproxyEnabled || !w.EnableOpenShift { + if pod.Spec.SecurityContext != nil { + // User container and consul-dataplane container cannot have the same UID. + if pod.Spec.SecurityContext.RunAsUser != nil && *pod.Spec.SecurityContext.RunAsUser == sidecarUserAndGroupID { + return corev1.Container{}, fmt.Errorf("pod's security context cannot have the same UID as consul-dataplane: %v", sidecarUserAndGroupID) + } + } + // Ensure that none of the user's containers have the same UID as consul-dataplane. At this point in injection the meshWebhook + // has only injected init containers so all containers defined in pod.Spec.Containers are from the user. + for _, c := range pod.Spec.Containers { + // User container and consul-dataplane container cannot have the same UID. + if c.SecurityContext != nil && c.SecurityContext.RunAsUser != nil && *c.SecurityContext.RunAsUser == sidecarUserAndGroupID && c.Image != w.ImageConsulDataplane { + return corev1.Container{}, fmt.Errorf("container %q has runAsUser set to the same UID \"%d\" as consul-dataplane which is not allowed", c.Name, sidecarUserAndGroupID) + } + } + container.SecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + RunAsGroup: pointer.Int64(sidecarUserAndGroupID), + RunAsNonRoot: pointer.Bool(true), + ReadOnlyRootFilesystem: pointer.Bool(true), + } + } + + return container, nil +} + +func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, bearerTokenFile string, pod corev1.Pod) ([]string, error) { + envoyConcurrency := w.DefaultEnvoyProxyConcurrency + + // Check to see if the user has overriden concurrency via an annotation. + if envoyConcurrencyAnnotation, ok := pod.Annotations[constants.AnnotationEnvoyProxyConcurrency]; ok { + val, err := strconv.ParseUint(envoyConcurrencyAnnotation, 10, 64) + if err != nil { + return nil, fmt.Errorf("unable to parse annotation %q: %w", constants.AnnotationEnvoyProxyConcurrency, err) + } + envoyConcurrency = int(val) + } + + args := []string{ + "-addresses", w.ConsulAddress, + "-grpc-port=" + strconv.Itoa(w.ConsulConfig.GRPCPort), + "-log-level=" + w.LogLevel, + "-log-json=" + strconv.FormatBool(w.LogJSON), + "-envoy-concurrency=" + strconv.Itoa(envoyConcurrency), + } + + if w.SkipServerWatch { + args = append(args, "-server-watch-disabled=true") + } + + if w.AuthMethod != "" { + args = append(args, + "-credential-type=login", + "-login-auth-method="+w.AuthMethod, + "-login-bearer-token-path="+bearerTokenFile, + // We don't know the pod name at this time, so we must use environment variables to populate the login-meta instead. + ) + if w.EnableNamespaces { + if w.EnableK8SNSMirroring { + args = append(args, "-login-namespace=default") + } else { + args = append(args, "-login-namespace="+w.consulNamespace(namespace.Name)) + } + } + if w.ConsulPartition != "" { + args = append(args, "-login-partition="+w.ConsulPartition) + } + } + if w.EnableNamespaces { + args = append(args, "-proxy-namespace="+w.consulNamespace(namespace.Name)) + } + if w.ConsulPartition != "" { + args = append(args, "-proxy-partition="+w.ConsulPartition) + } + if w.TLSEnabled { + if w.ConsulTLSServerName != "" { + args = append(args, "-tls-server-name="+w.ConsulTLSServerName) + } + if w.ConsulCACert != "" { + args = append(args, "-ca-certs="+constants.ConsulCAFile) + } + } else { + args = append(args, "-tls-disabled") + } + + // Configure the readiness port on the dataplane sidecar if proxy health checks are enabled. + if useProxyHealthCheck(pod) { + args = append(args, fmt.Sprintf("%s=%d", "-envoy-ready-bind-port", constants.ProxyDefaultHealthPort)) + } + + // The consul-dataplane HTTP listener always starts for graceful shutdown. To avoid port conflicts, the + // graceful port always needs to be set + gracefulPort, err := w.LifecycleConfig.GracefulPort(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle graceful port: %w", err) + } + + args = append(args, fmt.Sprintf("-graceful-port=%d", gracefulPort)) + + enableProxyLifecycle, err := w.LifecycleConfig.EnableProxyLifecycle(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if proxy lifecycle management is enabled: %w", err) + } + if enableProxyLifecycle { + shutdownDrainListeners, err := w.LifecycleConfig.EnableShutdownDrainListeners(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if proxy lifecycle shutdown listener draining is enabled: %w", err) + } + if shutdownDrainListeners { + args = append(args, "-shutdown-drain-listeners") + } + + shutdownGracePeriodSeconds, err := w.LifecycleConfig.ShutdownGracePeriodSeconds(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle shutdown grace period: %w", err) + } + args = append(args, fmt.Sprintf("-shutdown-grace-period-seconds=%d", shutdownGracePeriodSeconds)) + + gracefulShutdownPath := w.LifecycleConfig.GracefulShutdownPath(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle graceful shutdown path: %w", err) + } + args = append(args, fmt.Sprintf("-graceful-shutdown-path=%s", gracefulShutdownPath)) + } + + // Set a default scrape path that can be overwritten by the annotation. + prometheusScrapePath := w.MetricsConfig.PrometheusScrapePath(pod) + args = append(args, "-telemetry-prom-scrape-path="+prometheusScrapePath) + + metricsServer, err := w.MetricsConfig.ShouldRunMergedMetricsServer(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if merged metrics is enabled: %w", err) + } + if metricsServer { + + // (TODO) Figure out what port will be used for merged metrics and setup merged metrics + + mergedMetricsPort, err := w.MetricsConfig.MergedMetricsPort(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if merged metrics port: %w", err) + } + args = append(args, "-telemetry-prom-merge-port="+mergedMetricsPort) + + // Pull the TLS config from the relevant annotations. + var prometheusCAFile string + if raw, ok := pod.Annotations[constants.AnnotationPrometheusCAFile]; ok && raw != "" { + prometheusCAFile = raw + } + + var prometheusCAPath string + if raw, ok := pod.Annotations[constants.AnnotationPrometheusCAPath]; ok && raw != "" { + prometheusCAPath = raw + } + + var prometheusCertFile string + if raw, ok := pod.Annotations[constants.AnnotationPrometheusCertFile]; ok && raw != "" { + prometheusCertFile = raw + } + + var prometheusKeyFile string + if raw, ok := pod.Annotations[constants.AnnotationPrometheusKeyFile]; ok && raw != "" { + prometheusKeyFile = raw + } + + // Validate required Prometheus TLS config is present if set. + if prometheusCAFile != "" || prometheusCAPath != "" || prometheusCertFile != "" || prometheusKeyFile != "" { + if prometheusCAFile == "" && prometheusCAPath == "" { + return nil, fmt.Errorf("must set one of %q or %q when providing prometheus TLS config", constants.AnnotationPrometheusCAFile, constants.AnnotationPrometheusCAPath) + } + if prometheusCertFile == "" { + return nil, fmt.Errorf("must set %q when providing prometheus TLS config", constants.AnnotationPrometheusCertFile) + } + if prometheusKeyFile == "" { + return nil, fmt.Errorf("must set %q when providing prometheus TLS config", constants.AnnotationPrometheusKeyFile) + } + // TLS config has been validated, add them to the consul-dataplane cmd args + args = append(args, "-telemetry-prom-ca-certs-file="+prometheusCAFile, + "-telemetry-prom-ca-certs-path="+prometheusCAPath, + "-telemetry-prom-cert-file="+prometheusCertFile, + "-telemetry-prom-key-file="+prometheusKeyFile) + } + } + + // If Consul DNS is enabled, we want to configure consul-dataplane to be the DNS proxy + // for Consul DNS in the pod. + dnsEnabled, err := consulDNSEnabled(namespace, pod, w.EnableConsulDNS, w.EnableTransparentProxy) + if err != nil { + return nil, err + } + if dnsEnabled { + args = append(args, "-consul-dns-bind-port="+strconv.Itoa(consulDataplaneDNSBindPort)) + } + + var envoyExtraArgs []string + extraArgs, annotationSet := pod.Annotations[constants.AnnotationEnvoyExtraArgs] + + if annotationSet || w.EnvoyExtraArgs != "" { + extraArgsToUse := w.EnvoyExtraArgs + + // Prefer args set by pod annotation over the flag to the consul-k8s binary (h.EnvoyExtraArgs). + if annotationSet { + extraArgsToUse = extraArgs + } + + // Split string into tokens. + // e.g. "--foo bar --boo baz" --> ["--foo", "bar", "--boo", "baz"] + tokens, err := shlex.Split(extraArgsToUse) + if err != nil { + return []string{}, err + } + for _, t := range tokens { + if strings.Contains(t, " ") { + t = strconv.Quote(t) + } + envoyExtraArgs = append(envoyExtraArgs, t) + } + } + if envoyExtraArgs != nil { + args = append(args, "--") + args = append(args, envoyExtraArgs...) + } + return args, nil +} + +func (w *MeshWebhook) sidecarResources(pod corev1.Pod) (corev1.ResourceRequirements, error) { + resources := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{}, + Requests: corev1.ResourceList{}, + } + // zeroQuantity is used for comparison to see if a quantity was explicitly + // set. + var zeroQuantity resource.Quantity + + // NOTE: We only want to set the limit/request if the default or annotation + // was explicitly set. If it's not explicitly set, it will be the zero value + // which would show up in the pod spec as being explicitly set to zero if we + // set that key, e.g. "cpu" to zero. + // We want it to not show up in the pod spec at all if it's not explicitly + // set so that users aren't wondering why it's set to 0 when they didn't specify + // a request/limit. If they have explicitly set it to 0 then it will be set + // to 0 in the pod spec because we're doing a comparison to the zero-valued + // struct. + + // CPU Limit. + if anno, ok := pod.Annotations[constants.AnnotationSidecarProxyCPULimit]; ok { + cpuLimit, err := resource.ParseQuantity(anno) + if err != nil { + return corev1.ResourceRequirements{}, fmt.Errorf("parsing annotation %s:%q: %s", constants.AnnotationSidecarProxyCPULimit, anno, err) + } + resources.Limits[corev1.ResourceCPU] = cpuLimit + } else if w.DefaultProxyCPULimit != zeroQuantity { + resources.Limits[corev1.ResourceCPU] = w.DefaultProxyCPULimit + } + + // CPU Request. + if anno, ok := pod.Annotations[constants.AnnotationSidecarProxyCPURequest]; ok { + cpuRequest, err := resource.ParseQuantity(anno) + if err != nil { + return corev1.ResourceRequirements{}, fmt.Errorf("parsing annotation %s:%q: %s", constants.AnnotationSidecarProxyCPURequest, anno, err) + } + resources.Requests[corev1.ResourceCPU] = cpuRequest + } else if w.DefaultProxyCPURequest != zeroQuantity { + resources.Requests[corev1.ResourceCPU] = w.DefaultProxyCPURequest + } + + // Memory Limit. + if anno, ok := pod.Annotations[constants.AnnotationSidecarProxyMemoryLimit]; ok { + memoryLimit, err := resource.ParseQuantity(anno) + if err != nil { + return corev1.ResourceRequirements{}, fmt.Errorf("parsing annotation %s:%q: %s", constants.AnnotationSidecarProxyMemoryLimit, anno, err) + } + resources.Limits[corev1.ResourceMemory] = memoryLimit + } else if w.DefaultProxyMemoryLimit != zeroQuantity { + resources.Limits[corev1.ResourceMemory] = w.DefaultProxyMemoryLimit + } + + // Memory Request. + if anno, ok := pod.Annotations[constants.AnnotationSidecarProxyMemoryRequest]; ok { + memoryRequest, err := resource.ParseQuantity(anno) + if err != nil { + return corev1.ResourceRequirements{}, fmt.Errorf("parsing annotation %s:%q: %s", constants.AnnotationSidecarProxyMemoryRequest, anno, err) + } + resources.Requests[corev1.ResourceMemory] = memoryRequest + } else if w.DefaultProxyMemoryRequest != zeroQuantity { + resources.Requests[corev1.ResourceMemory] = w.DefaultProxyMemoryRequest + } + + return resources, nil +} + +// useProxyHealthCheck returns true if the pod has the annotation 'consul.hashicorp.com/use-proxy-health-check' +// set to truthy values. +func useProxyHealthCheck(pod corev1.Pod) bool { + if v, ok := pod.Annotations[constants.AnnotationUseProxyHealthCheck]; ok { + useProxyHealthCheck, err := strconv.ParseBool(v) + if err != nil { + return false + } + return useProxyHealthCheck + } + return false +} diff --git a/control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar_test.go b/control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar_test.go new file mode 100644 index 0000000000..aaa94a191d --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/consul_dataplane_sidecar_test.go @@ -0,0 +1,1103 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" + "github.com/hashicorp/consul-k8s/control-plane/consul" +) + +const nodeName = "test-node" + +func TestHandlerConsulDataplaneSidecar(t *testing.T) { + cases := map[string]struct { + webhookSetupFunc func(w *MeshWebhook) + additionalExpCmdArgs string + }{ + "default": { + webhookSetupFunc: nil, + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with custom gRPC port": { + webhookSetupFunc: func(w *MeshWebhook) { + w.ConsulConfig.GRPCPort = 8602 + }, + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with ACLs": { + webhookSetupFunc: func(w *MeshWebhook) { + w.AuthMethod = "test-auth-method" + }, + additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + + "-tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with ACLs and namespace mirroring": { + webhookSetupFunc: func(w *MeshWebhook) { + w.AuthMethod = "test-auth-method" + w.EnableNamespaces = true + w.EnableK8SNSMirroring = true + }, + additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + + "-login-namespace=default -proxy-namespace=k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with ACLs and single destination namespace": { + webhookSetupFunc: func(w *MeshWebhook) { + w.AuthMethod = "test-auth-method" + w.EnableNamespaces = true + w.ConsulDestinationNamespace = "test-ns" + }, + additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + + "-login-namespace=test-ns -proxy-namespace=test-ns -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with ACLs and partitions": { + webhookSetupFunc: func(w *MeshWebhook) { + w.AuthMethod = "test-auth-method" + w.ConsulPartition = "test-part" + }, + additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + + "-login-partition=test-part -proxy-partition=test-part -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with TLS and CA cert provided": { + webhookSetupFunc: func(w *MeshWebhook) { + w.TLSEnabled = true + w.ConsulTLSServerName = "server.dc1.consul" + w.ConsulCACert = "consul-ca-cert" + }, + additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -ca-certs=/consul/connect-inject/consul-ca.pem -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with TLS and no CA cert provided": { + webhookSetupFunc: func(w *MeshWebhook) { + w.TLSEnabled = true + w.ConsulTLSServerName = "server.dc1.consul" + }, + additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with single destination namespace": { + webhookSetupFunc: func(w *MeshWebhook) { + w.EnableNamespaces = true + w.ConsulDestinationNamespace = "consul-namespace" + }, + additionalExpCmdArgs: " -proxy-namespace=consul-namespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with namespace mirroring": { + webhookSetupFunc: func(w *MeshWebhook) { + w.EnableNamespaces = true + w.EnableK8SNSMirroring = true + }, + additionalExpCmdArgs: " -proxy-namespace=k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with namespace mirroring prefix": { + webhookSetupFunc: func(w *MeshWebhook) { + w.EnableNamespaces = true + w.EnableK8SNSMirroring = true + w.K8SNSMirroringPrefix = "foo-" + }, + additionalExpCmdArgs: " -proxy-namespace=foo-k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with partitions": { + webhookSetupFunc: func(w *MeshWebhook) { + w.ConsulPartition = "partition-1" + }, + additionalExpCmdArgs: " -proxy-partition=partition-1 -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with different log level": { + webhookSetupFunc: func(w *MeshWebhook) { + w.LogLevel = "debug" + }, + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "with different log level and log json": { + webhookSetupFunc: func(w *MeshWebhook) { + w.LogLevel = "debug" + w.LogJSON = true + }, + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "skip server watch enabled": { + webhookSetupFunc: func(w *MeshWebhook) { + w.SkipServerWatch = true + }, + additionalExpCmdArgs: " -server-watch-disabled=true -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", + }, + "custom prometheus scrape path": { + webhookSetupFunc: func(w *MeshWebhook) { + w.MetricsConfig.DefaultPrometheusScrapePath = "/scrape-path" // Simulate what would be passed as a flag + }, + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/scrape-path", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + w := &MeshWebhook{ + ConsulAddress: "1.1.1.1", + ConsulConfig: &consul.Config{GRPCPort: 8502}, + LogLevel: "info", + LogJSON: false, + } + if c.webhookSetupFunc != nil { + c.webhookSetupFunc(w) + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + { + Name: "auth-method-secret", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "service-account-secret", + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + }, + }, + }, + }, + ServiceAccountName: "web", + NodeName: nodeName, + }, + } + + container, err := w.consulDataplaneSidecar(testNS, pod) + require.NoError(t, err) + expCmd := "-addresses 1.1.1.1 -grpc-port=" + strconv.Itoa(w.ConsulConfig.GRPCPort) + + " -log-level=" + w.LogLevel + " -log-json=" + strconv.FormatBool(w.LogJSON) + " -envoy-concurrency=0" + c.additionalExpCmdArgs + require.Equal(t, expCmd, strings.Join(container.Args, " ")) + + if w.AuthMethod != "" { + require.Equal(t, container.VolumeMounts, []corev1.VolumeMount{ + { + Name: volumeName, + MountPath: "/consul/mesh-inject", + }, + { + Name: "service-account-secret", + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + }, + }) + } else { + require.Equal(t, container.VolumeMounts, []corev1.VolumeMount{ + { + Name: volumeName, + MountPath: "/consul/mesh-inject", + }, + }) + } + + expectedProbe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(constants.ProxyDefaultInboundPort), + }, + }, + InitialDelaySeconds: 1, + } + require.Equal(t, expectedProbe, container.ReadinessProbe) + require.Nil(t, container.StartupProbe) + require.Len(t, container.Env, 7) + require.Equal(t, container.Env[0].Name, "TMPDIR") + require.Equal(t, container.Env[0].Value, "/consul/mesh-inject") + require.Equal(t, container.Env[2].Name, "POD_NAME") + require.Equal(t, container.Env[3].Name, "POD_NAMESPACE") + require.Equal(t, container.Env[4].Name, "DP_PROXY_ID") + require.Equal(t, container.Env[4].Value, "$(POD_NAME)") + require.Equal(t, container.Env[5].Name, "DP_CREDENTIAL_LOGIN_META") + require.Equal(t, container.Env[5].Value, "pod=$(POD_NAMESPACE)/$(POD_NAME)") + require.Equal(t, container.Env[6].Name, "DP_CREDENTIAL_LOGIN_META1") + require.Equal(t, container.Env[6].Value, "pod=$(POD_NAMESPACE)/$(POD_NAME)") + }) + } +} + +func TestHandlerConsulDataplaneSidecar_Concurrency(t *testing.T) { + cases := map[string]struct { + annotations map[string]string + expFlags string + expErr string + }{ + "default settings, no annotations": { + annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + expFlags: "-envoy-concurrency=0", + }, + "default settings, annotation override": { + annotations: map[string]string{ + constants.AnnotationService: "foo", + constants.AnnotationEnvoyProxyConcurrency: "42", + }, + expFlags: "-envoy-concurrency=42", + }, + "default settings, invalid concurrency annotation negative number": { + annotations: map[string]string{ + constants.AnnotationService: "foo", + constants.AnnotationEnvoyProxyConcurrency: "-42", + }, + expErr: "unable to parse annotation \"consul.hashicorp.com/consul-envoy-proxy-concurrency\": strconv.ParseUint: parsing \"-42\": invalid syntax", + }, + "default settings, not-parseable concurrency annotation": { + annotations: map[string]string{ + constants.AnnotationService: "foo", + constants.AnnotationEnvoyProxyConcurrency: "not-int", + }, + expErr: "unable to parse annotation \"consul.hashicorp.com/consul-envoy-proxy-concurrency\": strconv.ParseUint: parsing \"not-int\": invalid syntax", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + h := MeshWebhook{ + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: c.annotations, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := h.consulDataplaneSidecar(testNS, pod) + if c.expErr != "" { + require.EqualError(t, err, c.expErr) + } else { + require.NoError(t, err) + require.Contains(t, strings.Join(container.Args, " "), c.expFlags) + } + }) + } +} + +// Test that we pass the dns proxy flag to dataplane correctly. +func TestHandlerConsulDataplaneSidecar_DNSProxy(t *testing.T) { + + // We only want the flag passed when DNS and tproxy are both enabled. DNS/tproxy can + // both be enabled/disabled with annotations/labels on the pod and namespace and then globally + // through the helm chart. To test this we use an outer loop with the possible DNS settings and then + // and inner loop with possible tproxy settings. + dnsCases := []struct { + GlobalConsulDNS bool + NamespaceDNS *bool + PodDNS *bool + ExpEnabled bool + }{ + { + GlobalConsulDNS: false, + ExpEnabled: false, + }, + { + GlobalConsulDNS: true, + ExpEnabled: true, + }, + { + GlobalConsulDNS: false, + NamespaceDNS: boolPtr(true), + ExpEnabled: true, + }, + { + GlobalConsulDNS: false, + PodDNS: boolPtr(true), + ExpEnabled: true, + }, + } + tproxyCases := []struct { + GlobalTProxy bool + NamespaceTProxy *bool + PodTProxy *bool + ExpEnabled bool + }{ + { + GlobalTProxy: false, + ExpEnabled: false, + }, + { + GlobalTProxy: true, + ExpEnabled: true, + }, + { + GlobalTProxy: false, + NamespaceTProxy: boolPtr(true), + ExpEnabled: true, + }, + { + GlobalTProxy: false, + PodTProxy: boolPtr(true), + ExpEnabled: true, + }, + } + + // Outer loop is permutations of dns being enabled. Inner loop is permutations of tproxy being enabled. + // Both must be enabled for dns to be enabled. + for i, dnsCase := range dnsCases { + for j, tproxyCase := range tproxyCases { + t.Run(fmt.Sprintf("dns=%d,tproxy=%d", i, j), func(t *testing.T) { + + // Test setup. + h := MeshWebhook{ + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + EnableTransparentProxy: tproxyCase.GlobalTProxy, + EnableConsulDNS: dnsCase.GlobalConsulDNS, + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + if dnsCase.PodDNS != nil { + pod.Annotations[constants.KeyConsulDNS] = strconv.FormatBool(*dnsCase.PodDNS) + } + if tproxyCase.PodTProxy != nil { + pod.Annotations[constants.KeyTransparentProxy] = strconv.FormatBool(*tproxyCase.PodTProxy) + } + + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sNamespace, + Labels: map[string]string{}, + }, + } + if dnsCase.NamespaceDNS != nil { + ns.Labels[constants.KeyConsulDNS] = strconv.FormatBool(*dnsCase.NamespaceDNS) + } + if tproxyCase.NamespaceTProxy != nil { + ns.Labels[constants.KeyTransparentProxy] = strconv.FormatBool(*tproxyCase.NamespaceTProxy) + } + + // Actual test here. + container, err := h.consulDataplaneSidecar(ns, pod) + require.NoError(t, err) + // Flag should only be passed if both tproxy and dns are enabled. + if tproxyCase.ExpEnabled && dnsCase.ExpEnabled { + require.Contains(t, container.Args, "-consul-dns-bind-port=8600") + } else { + require.NotContains(t, container.Args, "-consul-dns-bind-port=8600") + } + }) + } + } +} + +func TestHandlerConsulDataplaneSidecar_ProxyHealthCheck(t *testing.T) { + h := MeshWebhook{ + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + ConsulAddress: "1.1.1.1", + LogLevel: "info", + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationUseProxyHealthCheck: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := h.consulDataplaneSidecar(testNS, pod) + expectedProbe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(21000), + Path: "/ready", + }, + }, + InitialDelaySeconds: 1, + } + require.NoError(t, err) + require.Contains(t, container.Args, "-envoy-ready-bind-port=21000") + require.Equal(t, expectedProbe, container.ReadinessProbe) + require.Contains(t, container.Env, corev1.EnvVar{ + Name: "DP_ENVOY_READY_BIND_ADDRESS", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }, + }) + require.Contains(t, container.Ports, corev1.ContainerPort{ + Name: "proxy-health", + ContainerPort: 21000, + }) +} + +func TestHandlerConsulDataplaneSidecar_withSecurityContext(t *testing.T) { + cases := map[string]struct { + tproxyEnabled bool + openShiftEnabled bool + expSecurityContext *corev1.SecurityContext + }{ + "tproxy disabled; openshift disabled": { + tproxyEnabled: false, + openShiftEnabled: false, + expSecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + RunAsGroup: pointer.Int64(sidecarUserAndGroupID), + RunAsNonRoot: pointer.Bool(true), + ReadOnlyRootFilesystem: pointer.Bool(true), + }, + }, + "tproxy enabled; openshift disabled": { + tproxyEnabled: true, + openShiftEnabled: false, + expSecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + RunAsGroup: pointer.Int64(sidecarUserAndGroupID), + RunAsNonRoot: pointer.Bool(true), + ReadOnlyRootFilesystem: pointer.Bool(true), + }, + }, + "tproxy disabled; openshift enabled": { + tproxyEnabled: false, + openShiftEnabled: true, + expSecurityContext: nil, + }, + "tproxy enabled; openshift enabled": { + tproxyEnabled: true, + openShiftEnabled: true, + expSecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + RunAsGroup: pointer.Int64(sidecarUserAndGroupID), + RunAsNonRoot: pointer.Bool(true), + ReadOnlyRootFilesystem: pointer.Bool(true), + }, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + w := MeshWebhook{ + EnableTransparentProxy: c.tproxyEnabled, + EnableOpenShift: c.openShiftEnabled, + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + } + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + ec, err := w.consulDataplaneSidecar(testNS, pod) + require.NoError(t, err) + require.Equal(t, c.expSecurityContext, ec.SecurityContext) + }) + } +} + +// Test that if the user specifies a pod security context with the same uid as `sidecarUserAndGroupID` that we return +// an error to the meshWebhook. +func TestHandlerConsulDataplaneSidecar_FailsWithDuplicatePodSecurityContextUID(t *testing.T) { + require := require.New(t) + w := MeshWebhook{ + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + } + pod := corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + }, + }, + } + _, err := w.consulDataplaneSidecar(testNS, pod) + require.EqualError(err, fmt.Sprintf("pod's security context cannot have the same UID as consul-dataplane: %v", sidecarUserAndGroupID)) +} + +// Test that if the user specifies a container with security context with the same uid as `sidecarUserAndGroupID` that we +// return an error to the meshWebhook. If a container using the consul-dataplane image has the same uid, we don't return an error +// because in multiport pod there can be multiple consul-dataplane sidecars. +func TestHandlerConsulDataplaneSidecar_FailsWithDuplicateContainerSecurityContextUID(t *testing.T) { + cases := []struct { + name string + pod corev1.Pod + webhook MeshWebhook + expErr bool + expErrMessage string + }{ + { + name: "fails with non consul-dataplane image", + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + // Setting RunAsUser: 1 should succeed. + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(1), + }, + }, + { + Name: "app", + // Setting RunAsUser: 5995 should fail. + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + }, + Image: "not-consul-dataplane", + }, + }, + }, + }, + webhook: MeshWebhook{}, + expErr: true, + expErrMessage: fmt.Sprintf("container \"app\" has runAsUser set to the same UID \"%d\" as consul-dataplane which is not allowed", sidecarUserAndGroupID), + }, + { + name: "doesn't fail with envoy image", + pod: corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + // Setting RunAsUser: 1 should succeed. + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(1), + }, + }, + { + Name: "sidecar", + // Setting RunAsUser: 5995 should succeed if the image matches h.ImageConsulDataplane. + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Int64(sidecarUserAndGroupID), + }, + Image: "envoy", + }, + }, + }, + }, + webhook: MeshWebhook{ + ImageConsulDataplane: "envoy", + }, + expErr: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.webhook.ConsulConfig = &consul.Config{HTTPPort: 8500, GRPCPort: 8502} + _, err := tc.webhook.consulDataplaneSidecar(testNS, tc.pod) + if tc.expErr { + require.EqualError(t, err, tc.expErrMessage) + } else { + require.NoError(t, err) + } + }) + } +} + +// Test that we can pass extra args to envoy via the extraEnvoyArgs flag +// or via pod annotations. When arguments are passed in both ways, the +// arguments set via pod annotations are used. +func TestHandlerConsulDataplaneSidecar_EnvoyExtraArgs(t *testing.T) { + cases := []struct { + name string + envoyExtraArgs string + pod *corev1.Pod + expectedExtraArgs string + }{ + { + name: "no extra options provided", + envoyExtraArgs: "", + pod: &corev1.Pod{}, + expectedExtraArgs: "", + }, + { + name: "via flag: extra log-level option", + envoyExtraArgs: "--log-level debug", + pod: &corev1.Pod{}, + expectedExtraArgs: "-- --log-level debug", + }, + { + name: "via flag: multiple arguments with quotes", + envoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + pod: &corev1.Pod{}, + expectedExtraArgs: "-- --log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + }, + { + name: "via annotation: multiple arguments with quotes", + envoyExtraArgs: "", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationEnvoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + }, + }, + }, + expectedExtraArgs: "-- --log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + }, + { + name: "via flag and annotation: should prefer setting via the annotation", + envoyExtraArgs: "this should be overwritten", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationEnvoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + }, + }, + }, + expectedExtraArgs: "-- --log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := MeshWebhook{ + ImageConsul: "hashicorp/consul:latest", + ImageConsulDataplane: "hashicorp/consul-k8s:latest", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + EnvoyExtraArgs: tc.envoyExtraArgs, + } + + c, err := h.consulDataplaneSidecar(testNS, *tc.pod) + require.NoError(t, err) + require.Contains(t, strings.Join(c.Args, " "), tc.expectedExtraArgs) + }) + } +} + +func TestHandlerConsulDataplaneSidecar_UserVolumeMounts(t *testing.T) { + cases := []struct { + name string + pod corev1.Pod + expectedContainerVolumeMounts []corev1.VolumeMount + expErr string + }{ + { + name: "able to set a sidecar container volume mount via annotation", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationEnvoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + constants.AnnotationConsulSidecarUserVolumeMount: "[{\"name\": \"tls-cert\", \"mountPath\": \"/custom/path\"}, {\"name\": \"tls-ca\", \"mountPath\": \"/custom/path2\"}]", + }, + }, + }, + expectedContainerVolumeMounts: []corev1.VolumeMount{ + { + Name: "consul-connect-inject-data", + MountPath: "/consul/mesh-inject", + }, + { + Name: "tls-cert", + MountPath: "/custom/path", + }, + { + Name: "tls-ca", + MountPath: "/custom/path2", + }, + }, + }, + { + name: "invalid annotation results in error", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationEnvoyExtraArgs: "--log-level debug --admin-address-path \"/tmp/consul/foo bar\"", + constants.AnnotationConsulSidecarUserVolumeMount: "[abcdefg]", + }, + }, + }, + expErr: "invalid character 'a' looking ", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := MeshWebhook{ + ImageConsul: "hashicorp/consul:latest", + ImageConsulDataplane: "hashicorp/consul-k8s:latest", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + } + c, err := h.consulDataplaneSidecar(testNS, tc.pod) + if tc.expErr == "" { + require.NoError(t, err) + require.Equal(t, tc.expectedContainerVolumeMounts, c.VolumeMounts) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expErr) + } + }) + } +} + +func TestHandlerConsulDataplaneSidecar_Resources(t *testing.T) { + mem1 := resource.MustParse("100Mi") + mem2 := resource.MustParse("200Mi") + cpu1 := resource.MustParse("100m") + cpu2 := resource.MustParse("200m") + zero := resource.MustParse("0") + + cases := map[string]struct { + webhook MeshWebhook + annotations map[string]string + expResources corev1.ResourceRequirements + expErr string + }{ + "no defaults, no annotations": { + webhook: MeshWebhook{}, + annotations: nil, + expResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{}, + Requests: corev1.ResourceList{}, + }, + }, + "all defaults, no annotations": { + webhook: MeshWebhook{ + DefaultProxyCPURequest: cpu1, + DefaultProxyCPULimit: cpu2, + DefaultProxyMemoryRequest: mem1, + DefaultProxyMemoryLimit: mem2, + }, + annotations: nil, + expResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: cpu2, + corev1.ResourceMemory: mem2, + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: cpu1, + corev1.ResourceMemory: mem1, + }, + }, + }, + "no defaults, all annotations": { + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationSidecarProxyCPURequest: "100m", + constants.AnnotationSidecarProxyMemoryRequest: "100Mi", + constants.AnnotationSidecarProxyCPULimit: "200m", + constants.AnnotationSidecarProxyMemoryLimit: "200Mi", + }, + expResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: cpu2, + corev1.ResourceMemory: mem2, + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: cpu1, + corev1.ResourceMemory: mem1, + }, + }, + }, + "annotations override defaults": { + webhook: MeshWebhook{ + DefaultProxyCPURequest: zero, + DefaultProxyCPULimit: zero, + DefaultProxyMemoryRequest: zero, + DefaultProxyMemoryLimit: zero, + }, + annotations: map[string]string{ + constants.AnnotationSidecarProxyCPURequest: "100m", + constants.AnnotationSidecarProxyMemoryRequest: "100Mi", + constants.AnnotationSidecarProxyCPULimit: "200m", + constants.AnnotationSidecarProxyMemoryLimit: "200Mi", + }, + expResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: cpu2, + corev1.ResourceMemory: mem2, + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: cpu1, + corev1.ResourceMemory: mem1, + }, + }, + }, + "defaults set to zero, no annotations": { + webhook: MeshWebhook{ + DefaultProxyCPURequest: zero, + DefaultProxyCPULimit: zero, + DefaultProxyMemoryRequest: zero, + DefaultProxyMemoryLimit: zero, + }, + annotations: nil, + expResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: zero, + corev1.ResourceMemory: zero, + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: zero, + corev1.ResourceMemory: zero, + }, + }, + }, + "annotations set to 0": { + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationSidecarProxyCPURequest: "0", + constants.AnnotationSidecarProxyMemoryRequest: "0", + constants.AnnotationSidecarProxyCPULimit: "0", + constants.AnnotationSidecarProxyMemoryLimit: "0", + }, + expResources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: zero, + corev1.ResourceMemory: zero, + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: zero, + corev1.ResourceMemory: zero, + }, + }, + }, + "invalid cpu request": { + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationSidecarProxyCPURequest: "invalid", + }, + expErr: "parsing annotation consul.hashicorp.com/sidecar-proxy-cpu-request:\"invalid\": quantities must match the regular expression", + }, + "invalid cpu limit": { + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationSidecarProxyCPULimit: "invalid", + }, + expErr: "parsing annotation consul.hashicorp.com/sidecar-proxy-cpu-limit:\"invalid\": quantities must match the regular expression", + }, + "invalid memory request": { + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationSidecarProxyMemoryRequest: "invalid", + }, + expErr: "parsing annotation consul.hashicorp.com/sidecar-proxy-memory-request:\"invalid\": quantities must match the regular expression", + }, + "invalid memory limit": { + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationSidecarProxyMemoryLimit: "invalid", + }, + expErr: "parsing annotation consul.hashicorp.com/sidecar-proxy-memory-limit:\"invalid\": quantities must match the regular expression", + }, + } + + for name, c := range cases { + t.Run(name, func(tt *testing.T) { + c.webhook.ConsulConfig = &consul.Config{HTTPPort: 8500, GRPCPort: 8502} + require := require.New(tt) + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: c.annotations, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := c.webhook.consulDataplaneSidecar(testNS, pod) + if c.expErr != "" { + require.NotNil(err) + require.Contains(err.Error(), c.expErr) + } else { + require.NoError(err) + require.Equal(c.expResources, container.Resources) + } + }) + } +} + +func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) { + gracefulShutdownSeconds := 10 + gracefulPort := "20307" + gracefulShutdownPath := "/exit" + + cases := []struct { + name string + webhook MeshWebhook + annotations map[string]string + expCmdArgs string + expErr string + }{ + { + name: "no defaults, no annotations", + webhook: MeshWebhook{}, + annotations: nil, + expCmdArgs: "", + }, + { + name: "all defaults, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: nil, + expCmdArgs: "graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit", + }, + { + name: "no defaults, all annotations", + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "true", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "true", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: fmt.Sprint(gracefulShutdownSeconds), + constants.AnnotationSidecarProxyLifecycleGracefulPort: gracefulPort, + constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath: gracefulShutdownPath, + }, + expCmdArgs: "-graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit", + }, + { + name: "annotations override defaults", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "true", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "false", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: fmt.Sprint(gracefulShutdownSeconds + 5), + constants.AnnotationSidecarProxyLifecycleGracefulPort: "20317", + constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath: "/foo", + }, + expCmdArgs: "-graceful-port=20317 -shutdown-grace-period-seconds=15 -graceful-shutdown-path=/foo", + }, + { + name: "lifecycle disabled, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: nil, + expCmdArgs: "-graceful-port=20307", + }, + { + name: "lifecycle enabled, defaults omited, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + }, + }, + annotations: nil, + expCmdArgs: "", + }, + { + name: "annotations disable lifecycle default", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "false", + }, + expCmdArgs: "-graceful-port=20307", + }, + { + name: "annotations skip graceful shutdown", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "false", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "false", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: "0", + }, + expCmdArgs: "", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.webhook.ConsulConfig = &consul.Config{HTTPPort: 8500, GRPCPort: 8502} + require := require.New(t) + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: c.annotations, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := c.webhook.consulDataplaneSidecar(testNS, pod) + if c.expErr != "" { + require.NotNil(err) + require.Contains(err.Error(), c.expErr) + } else { + require.NoError(err) + require.Contains(strings.Join(container.Args, " "), c.expCmdArgs) + } + }) + } +} + +// boolPtr returns pointer to b. +func boolPtr(b bool) *bool { + return &b +} diff --git a/control-plane/connect-inject/webhook_v2/container_env.go b/control-plane/connect-inject/webhook_v2/container_env.go new file mode 100644 index 0000000000..4c05a2ea72 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/container_env.go @@ -0,0 +1,37 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + corev1 "k8s.io/api/core/v1" +) + +func (w *MeshWebhook) containerEnvVars(pod corev1.Pod) []corev1.EnvVar { + // (TODO: ashwin) make this work with current upstreams + //raw, ok := pod.Annotations[constants.AnnotationMeshDestinations] + //if !ok || raw == "" { + // return []corev1.EnvVar{} + //} + // + //var result []corev1.EnvVar + //for _, raw := range strings.Split(raw, ",") { + // parts := strings.SplitN(raw, ":", 3) + // port, _ := common.PortValue(pod, strings.TrimSpace(parts[1])) + // if port > 0 { + // name := strings.TrimSpace(parts[0]) + // name = strings.ToUpper(strings.Replace(name, "-", "_", -1)) + // portStr := strconv.Itoa(int(port)) + // + // result = append(result, corev1.EnvVar{ + // Name: fmt.Sprintf("%s_CONNECT_SERVICE_HOST", name), + // Value: "127.0.0.1", + // }, corev1.EnvVar{ + // Name: fmt.Sprintf("%s_CONNECT_SERVICE_PORT", name), + // Value: portStr, + // }) + // } + //} + + return []corev1.EnvVar{} +} diff --git a/control-plane/connect-inject/webhook_v2/container_env_test.go b/control-plane/connect-inject/webhook_v2/container_env_test.go new file mode 100644 index 0000000000..f7cef104ea --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/container_env_test.go @@ -0,0 +1,58 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" +) + +func TestContainerEnvVars(t *testing.T) { + t.Skip() + // (TODO: ashwin) make these work once upstreams are fixed + cases := []struct { + Name string + Upstream string + }{ + { + "Upstream with datacenter", + "static-server:7890:dc1", + }, + { + "Upstream without datacenter", + "static-server:7890", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + + var w MeshWebhook + envVars := w.containerEnvVars(corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "foo", + constants.AnnotationMeshDestinations: tt.Upstream, + }, + }, + }) + + require.ElementsMatch(envVars, []corev1.EnvVar{ + { + Name: "STATIC_SERVER_CONNECT_SERVICE_HOST", + Value: "127.0.0.1", + }, { + Name: "STATIC_SERVER_CONNECT_SERVICE_PORT", + Value: "7890", + }, + }) + }) + } +} diff --git a/control-plane/connect-inject/webhook_v2/container_init.go b/control-plane/connect-inject/webhook_v2/container_init.go new file mode 100644 index 0000000000..ebf4b0e336 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/container_init.go @@ -0,0 +1,300 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "bytes" + "strconv" + "strings" + "text/template" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" +) + +const ( + injectInitContainerName = "consul-mesh-init" + rootUserAndGroupID = 0 + sidecarUserAndGroupID = 5995 + initContainersUserAndGroupID = 5996 + netAdminCapability = "NET_ADMIN" +) + +type initContainerCommandData struct { + ServiceName string + ServiceAccountName string + AuthMethod string + + // Log settings for the connect-init command. + LogLevel string + LogJSON bool +} + +// containerInit returns the init container spec for connect-init that polls for the service and the connect proxy service to be registered +// so that it can save the proxy service id to the shared volume and boostrap Envoy with the proxy-id. +func (w *MeshWebhook) containerInit(namespace corev1.Namespace, pod corev1.Pod) (corev1.Container, error) { + // Check if tproxy is enabled on this pod. + tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy) + if err != nil { + return corev1.Container{}, err + } + + data := initContainerCommandData{ + AuthMethod: w.AuthMethod, + LogLevel: w.LogLevel, + LogJSON: w.LogJSON, + } + + // Create expected volume mounts + volMounts := []corev1.VolumeMount{ + { + Name: volumeName, + MountPath: "/consul/connect-inject", + }, + } + + data.ServiceName = pod.Annotations[constants.AnnotationService] + var bearerTokenFile string + if w.AuthMethod != "" { + data.ServiceAccountName = pod.Spec.ServiceAccountName + // Extract the service account token's volume mount + var saTokenVolumeMount corev1.VolumeMount + saTokenVolumeMount, bearerTokenFile, err = findServiceAccountVolumeMount(pod) + if err != nil { + return corev1.Container{}, err + } + + // Append to volume mounts + volMounts = append(volMounts, saTokenVolumeMount) + } + + // Render the command + var buf bytes.Buffer + tpl := template.Must(template.New("root").Parse(strings.TrimSpace( + initContainerCommandTpl))) + err = tpl.Execute(&buf, &data) + if err != nil { + return corev1.Container{}, err + } + + initContainerName := injectInitContainerName + container := corev1.Container{ + Name: initContainerName, + Image: w.ImageConsulK8S, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }, + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: "CONSUL_ADDRESSES", + Value: w.ConsulAddress, + }, + { + Name: "CONSUL_GRPC_PORT", + Value: strconv.Itoa(w.ConsulConfig.GRPCPort), + }, + { + Name: "CONSUL_HTTP_PORT", + Value: strconv.Itoa(w.ConsulConfig.HTTPPort), + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: w.ConsulConfig.APITimeout.String(), + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + }, + Resources: w.InitContainerResources, + VolumeMounts: volMounts, + Command: []string{"/bin/sh", "-ec", buf.String()}, + } + + if w.TLSEnabled { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_USE_TLS", + Value: "true", + }, + corev1.EnvVar{ + Name: "CONSUL_CACERT_PEM", + Value: w.ConsulCACert, + }, + corev1.EnvVar{ + Name: "CONSUL_TLS_SERVER_NAME", + Value: w.ConsulTLSServerName, + }) + } + + if w.AuthMethod != "" { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_LOGIN_AUTH_METHOD", + Value: w.AuthMethod, + }, + corev1.EnvVar{ + Name: "CONSUL_LOGIN_BEARER_TOKEN_FILE", + Value: bearerTokenFile, + }, + corev1.EnvVar{ + Name: "CONSUL_LOGIN_META", + Value: "pod=$(POD_NAMESPACE)/$(POD_NAME)", + }) + + if w.EnableNamespaces { + if w.EnableK8SNSMirroring { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_LOGIN_NAMESPACE", + Value: "default", + }) + } else { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_LOGIN_NAMESPACE", + Value: w.consulNamespace(namespace.Name), + }) + } + } + + if w.ConsulPartition != "" { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_LOGIN_PARTITION", + Value: w.ConsulPartition, + }) + } + } + if w.EnableNamespaces { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_NAMESPACE", + Value: w.consulNamespace(namespace.Name), + }) + } + + if w.ConsulPartition != "" { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_PARTITION", + Value: w.ConsulPartition, + }) + } + + // OpenShift without CNI is the only environment where privileged must be true. + privileged := false + if w.EnableOpenShift && !w.EnableCNI { + privileged = true + } + + if tproxyEnabled { + if !w.EnableCNI { + // Set redirect traffic config for the container so that we can apply iptables rules. + redirectTrafficConfig, err := w.iptablesConfigJSON(pod, namespace) + if err != nil { + return corev1.Container{}, err + } + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "CONSUL_REDIRECT_TRAFFIC_CONFIG", + Value: redirectTrafficConfig, + }) + + // Running consul connect redirect-traffic with iptables + // requires both being a root user and having NET_ADMIN capability. + container.SecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64(rootUserAndGroupID), + RunAsGroup: pointer.Int64(rootUserAndGroupID), + // RunAsNonRoot overrides any setting in the Pod so that we can still run as root here as required. + RunAsNonRoot: pointer.Bool(false), + Privileged: pointer.Bool(privileged), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{netAdminCapability}, + }, + } + } else { + container.SecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64(initContainersUserAndGroupID), + RunAsGroup: pointer.Int64(initContainersUserAndGroupID), + RunAsNonRoot: pointer.Bool(true), + Privileged: pointer.Bool(privileged), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + } + } + } + + return container, nil +} + +// consulDNSEnabled returns true if Consul DNS should be enabled for this pod. +// It returns an error when the annotation value cannot be parsed by strconv.ParseBool or if we are unable +// to read the pod's namespace label when it exists. +func consulDNSEnabled(namespace corev1.Namespace, pod corev1.Pod, globalDNSEnabled bool, globalTProxyEnabled bool) (bool, error) { + // DNS is only possible when tproxy is also enabled because it relies + // on traffic being redirected. + tproxy, err := common.TransparentProxyEnabled(namespace, pod, globalTProxyEnabled) + if err != nil { + return false, err + } + if !tproxy { + return false, nil + } + + // First check to see if the pod annotation exists to override the namespace or global settings. + if raw, ok := pod.Annotations[constants.KeyConsulDNS]; ok { + return strconv.ParseBool(raw) + } + // Next see if the namespace has been defaulted. + if raw, ok := namespace.Labels[constants.KeyConsulDNS]; ok { + return strconv.ParseBool(raw) + } + // Else fall back to the global default. + return globalDNSEnabled, nil +} + +// splitCommaSeparatedItemsFromAnnotation takes an annotation and a pod +// and returns the comma-separated value of the annotation as a list of strings. +func splitCommaSeparatedItemsFromAnnotation(annotation string, pod corev1.Pod) []string { + var items []string + if raw, ok := pod.Annotations[annotation]; ok { + items = append(items, strings.Split(raw, ",")...) + } + + return items +} + +// initContainerCommandTpl is the template for the command executed by +// the init container. +const initContainerCommandTpl = ` +consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level={{ .LogLevel }} \ + -log-json={{ .LogJSON }} \ + {{- if .AuthMethod }} + -service-account-name="{{ .ServiceAccountName }}" \ + -service-name="{{ .ServiceName }}" \ + {{- end }} +` diff --git a/control-plane/connect-inject/webhook_v2/container_init_test.go b/control-plane/connect-inject/webhook_v2/container_init_test.go new file mode 100644 index 0000000000..6931122124 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/container_init_test.go @@ -0,0 +1,844 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" +) + +const k8sNamespace = "k8snamespace" + +func TestHandlerContainerInit(t *testing.T) { + minimal := func() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "test-namespace", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + Status: corev1.PodStatus{ + HostIP: "1.1.1.1", + PodIP: "2.2.2.2", + }, + } + } + + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + Webhook MeshWebhook + ExpCmd string // Strings.Contains test + ExpEnv []corev1.EnvVar + }{ + { + "default cmd and env", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "web" + return pod + }, + MeshWebhook{ + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + LogLevel: "info", + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "0s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + }, + }, + + { + "with auth method", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "web" + pod.Spec.ServiceAccountName = "a-service-account-name" + pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: "sa", + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + }, + } + return pod + }, + MeshWebhook{ + AuthMethod: "an-auth-method", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + LogLevel: "debug", + LogJSON: true, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=debug \ + -log-json=true \ + -service-account-name="a-service-account-name" \ + -service-name="web" \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_LOGIN_AUTH_METHOD", + Value: "an-auth-method", + }, + { + Name: "CONSUL_LOGIN_BEARER_TOKEN_FILE", + Value: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + { + Name: "CONSUL_LOGIN_META", + Value: "pod=$(POD_NAMESPACE)/$(POD_NAME)", + }, + }, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + w := tt.Webhook + pod := *tt.Pod(minimal()) + container, err := w.containerInit(testNS, pod) + require.NoError(t, err) + actual := strings.Join(container.Command, " ") + require.Contains(t, actual, tt.ExpCmd) + require.EqualValues(t, container.Env[3:], tt.ExpEnv) + }) + } +} + +func TestHandlerContainerInit_transparentProxy(t *testing.T) { + cases := map[string]struct { + globalEnabled bool + cniEnabled bool + annotations map[string]string + expTproxyEnabled bool + namespaceLabel map[string]string + openShiftEnabled bool + }{ + "enabled globally, ns not set, annotation not provided, cni disabled, openshift disabled": { + true, + false, + nil, + true, + nil, + false, + }, + "enabled globally, ns not set, annotation is false, cni disabled, openshift disabled": { + true, + false, + map[string]string{constants.KeyTransparentProxy: "false"}, + false, + nil, + false, + }, + "enabled globally, ns not set, annotation is true, cni disabled, openshift disabled": { + true, + false, + map[string]string{constants.KeyTransparentProxy: "true"}, + true, + nil, + false, + }, + "disabled globally, ns not set, annotation not provided, cni disabled, openshift disabled": { + false, + false, + nil, + false, + nil, + false, + }, + "disabled globally, ns not set, annotation is false, cni disabled, openshift disabled": { + false, + false, + map[string]string{constants.KeyTransparentProxy: "false"}, + false, + nil, + false, + }, + "disabled globally, ns not set, annotation is true, cni disabled, openshift disabled": { + false, + false, + map[string]string{constants.KeyTransparentProxy: "true"}, + true, + nil, + false, + }, + "disabled globally, ns enabled, annotation not set, cni disabled, openshift disabled": { + false, + false, + nil, + true, + map[string]string{constants.KeyTransparentProxy: "true"}, + false, + }, + "enabled globally, ns disabled, annotation not set, cni disabled, openshift disabled": { + true, + false, + nil, + false, + map[string]string{constants.KeyTransparentProxy: "false"}, + false, + }, + "disabled globally, ns enabled, annotation not set, cni enabled, openshift disabled": { + false, + true, + nil, + false, + map[string]string{constants.KeyTransparentProxy: "true"}, + false, + }, + + "enabled globally, ns not set, annotation not set, cni enabled, openshift disabled": { + true, + true, + nil, + false, + nil, + false, + }, + "enabled globally, ns not set, annotation not set, cni enabled, openshift enabled": { + true, + true, + nil, + false, + nil, + true, + }, + "enabled globally, ns not set, annotation not set, cni disabled, openshift enabled": { + true, + false, + nil, + true, + nil, + true, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + w := MeshWebhook{ + EnableTransparentProxy: c.globalEnabled, + EnableCNI: c.cniEnabled, + ConsulConfig: &consul.Config{HTTPPort: 8500}, + EnableOpenShift: c.openShiftEnabled, + } + pod := minimal() + pod.Annotations = c.annotations + + privileged := false + if c.openShiftEnabled && !c.cniEnabled { + privileged = true + } + + var expectedSecurityContext *corev1.SecurityContext + if c.cniEnabled { + expectedSecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64(initContainersUserAndGroupID), + RunAsGroup: pointer.Int64(initContainersUserAndGroupID), + RunAsNonRoot: pointer.Bool(true), + Privileged: pointer.Bool(privileged), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + } + } else if c.expTproxyEnabled { + expectedSecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64(0), + RunAsGroup: pointer.Int64(0), + RunAsNonRoot: pointer.Bool(false), + Privileged: pointer.Bool(privileged), + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{netAdminCapability}, + }, + } + } + ns := testNS + ns.Labels = c.namespaceLabel + container, err := w.containerInit(ns, *pod) + require.NoError(t, err) + + redirectTrafficEnvVarFound := false + for _, ev := range container.Env { + if ev.Name == "CONSUL_REDIRECT_TRAFFIC_CONFIG" { + redirectTrafficEnvVarFound = true + break + } + } + + require.Equal(t, c.expTproxyEnabled, redirectTrafficEnvVarFound) + require.Equal(t, expectedSecurityContext, container.SecurityContext) + }) + } +} + +func TestHandlerContainerInit_namespacesAndPartitionsEnabled(t *testing.T) { + minimal := func() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + { + Name: "auth-method-secret", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "service-account-secret", + MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", + }, + }, + }, + }, + ServiceAccountName: "web", + }, + } + } + + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + Webhook MeshWebhook + Cmd string + ExpEnv []corev1.EnvVar + }{ + { + "default namespace, no partition", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "web" + return pod + }, + MeshWebhook{ + EnableNamespaces: true, + ConsulDestinationNamespace: "default", + ConsulPartition: "", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_NAMESPACE", + Value: "default", + }, + }, + }, + { + "default namespace, default partition", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "web" + return pod + }, + MeshWebhook{ + EnableNamespaces: true, + ConsulDestinationNamespace: "default", + ConsulPartition: "default", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_NAMESPACE", + Value: "default", + }, + { + Name: "CONSUL_PARTITION", + Value: "default", + }, + }, + }, + { + "non-default namespace, no partition", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "web" + return pod + }, + MeshWebhook{ + EnableNamespaces: true, + ConsulDestinationNamespace: "non-default", + ConsulPartition: "", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_NAMESPACE", + Value: "non-default", + }, + }, + }, + { + "non-default namespace, non-default partition", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "web" + return pod + }, + MeshWebhook{ + EnableNamespaces: true, + ConsulDestinationNamespace: "non-default", + ConsulPartition: "non-default-part", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_NAMESPACE", + Value: "non-default", + }, + { + Name: "CONSUL_PARTITION", + Value: "non-default-part", + }, + }, + }, + { + "auth method, non-default namespace, mirroring disabled, default partition", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "" + return pod + }, + MeshWebhook{ + AuthMethod: "auth-method", + EnableNamespaces: true, + ConsulDestinationNamespace: "non-default", + ConsulPartition: "default", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \ + -service-account-name="web" \ + -service-name="" \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_LOGIN_AUTH_METHOD", + Value: "auth-method", + }, + { + Name: "CONSUL_LOGIN_BEARER_TOKEN_FILE", + Value: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + { + Name: "CONSUL_LOGIN_META", + Value: "pod=$(POD_NAMESPACE)/$(POD_NAME)", + }, + { + Name: "CONSUL_LOGIN_NAMESPACE", + Value: "non-default", + }, + { + Name: "CONSUL_LOGIN_PARTITION", + Value: "default", + }, + { + Name: "CONSUL_NAMESPACE", + Value: "non-default", + }, + { + Name: "CONSUL_PARTITION", + Value: "default", + }, + }, + }, + { + "auth method, non-default namespace, mirroring enabled, non-default partition", + func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationService] = "" + return pod + }, + MeshWebhook{ + AuthMethod: "auth-method", + EnableNamespaces: true, + ConsulDestinationNamespace: "non-default", // Overridden by mirroring + EnableK8SNSMirroring: true, + ConsulPartition: "non-default", + ConsulAddress: "10.0.0.0", + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502, APITimeout: 5 * time.Second}, + }, + `/bin/sh -ec consul-k8s-control-plane connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \ + -log-level=info \ + -log-json=false \ + -service-account-name="web" \ + -service-name="" \`, + []corev1.EnvVar{ + { + Name: "CONSUL_ADDRESSES", + Value: "10.0.0.0", + }, + { + Name: "CONSUL_GRPC_PORT", + Value: "8502", + }, + { + Name: "CONSUL_HTTP_PORT", + Value: "8500", + }, + { + Name: "CONSUL_API_TIMEOUT", + Value: "5s", + }, + { + Name: "CONSUL_NODE_NAME", + Value: "$(NODE_NAME)-virtual", + }, + { + Name: "CONSUL_LOGIN_AUTH_METHOD", + Value: "auth-method", + }, + { + Name: "CONSUL_LOGIN_BEARER_TOKEN_FILE", + Value: "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + { + Name: "CONSUL_LOGIN_META", + Value: "pod=$(POD_NAMESPACE)/$(POD_NAME)", + }, + { + Name: "CONSUL_LOGIN_NAMESPACE", + Value: "default", + }, + { + Name: "CONSUL_LOGIN_PARTITION", + Value: "non-default", + }, + { + Name: "CONSUL_NAMESPACE", + Value: "k8snamespace", + }, + { + Name: "CONSUL_PARTITION", + Value: "non-default", + }, + }, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + h := tt.Webhook + h.LogLevel = "info" + container, err := h.containerInit(testNS, *tt.Pod(minimal())) + require.NoError(t, err) + actual := strings.Join(container.Command, " ") + require.Equal(t, tt.Cmd, actual) + if tt.ExpEnv != nil { + require.Equal(t, tt.ExpEnv, container.Env[3:]) + } + }) + } +} + +// If TLSEnabled is set, +// Consul addresses should use HTTPS +// and CA cert should be set as env variable if provided. +// Additionally, test that the init container is correctly configured +// when http or gRPC ports are different from defaults. +func TestHandlerContainerInit_WithTLSAndCustomPorts(t *testing.T) { + for _, caProvided := range []bool{true, false} { + name := fmt.Sprintf("ca provided: %t", caProvided) + t.Run(name, func(t *testing.T) { + w := MeshWebhook{ + ConsulAddress: "10.0.0.0", + TLSEnabled: true, + ConsulConfig: &consul.Config{HTTPPort: 443, GRPCPort: 8503}, + } + if caProvided { + w.ConsulCACert = "consul-ca-cert" + } + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := w.containerInit(testNS, *pod) + require.NoError(t, err) + require.Equal(t, "CONSUL_ADDRESSES", container.Env[3].Name) + require.Equal(t, w.ConsulAddress, container.Env[3].Value) + require.Equal(t, "CONSUL_GRPC_PORT", container.Env[4].Name) + require.Equal(t, fmt.Sprintf("%d", w.ConsulConfig.GRPCPort), container.Env[4].Value) + require.Equal(t, "CONSUL_HTTP_PORT", container.Env[5].Name) + require.Equal(t, fmt.Sprintf("%d", w.ConsulConfig.HTTPPort), container.Env[5].Value) + if w.TLSEnabled { + require.Equal(t, "CONSUL_USE_TLS", container.Env[8].Name) + require.Equal(t, "true", container.Env[8].Value) + if caProvided { + require.Equal(t, "CONSUL_CACERT_PEM", container.Env[9].Name) + require.Equal(t, "consul-ca-cert", container.Env[9].Value) + } else { + for _, ev := range container.Env { + if ev.Name == "CONSUL_CACERT_PEM" { + require.Empty(t, ev.Value) + } + } + } + } + + }) + } +} + +func TestHandlerContainerInit_Resources(t *testing.T) { + w := MeshWebhook{ + InitContainerResources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("25Mi"), + }, + }, + ConsulConfig: &consul.Config{HTTPPort: 8500, APITimeout: 5 * time.Second}, + } + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := w.containerInit(testNS, *pod) + require.NoError(t, err) + require.Equal(t, corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("25Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + }, container.Resources) +} + +var testNS = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sNamespace, + Labels: map[string]string{}, + }, +} + +func minimal() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaces.DefaultNamespace, + Name: "minimal", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + } +} diff --git a/control-plane/connect-inject/webhook_v2/container_volume.go b/control-plane/connect-inject/webhook_v2/container_volume.go new file mode 100644 index 0000000000..8de3d5b6f5 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/container_volume.go @@ -0,0 +1,23 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + corev1 "k8s.io/api/core/v1" +) + +// volumeName is the name of the volume that is created to store the +// Consul Connect injection data. +const volumeName = "consul-connect-inject-data" + +// containerVolume returns the volume data to add to the pod. This volume +// is used for shared data between containers. +func (w *MeshWebhook) containerVolume() corev1.Volume { + return corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}, + }, + } +} diff --git a/control-plane/connect-inject/webhook_v2/dns.go b/control-plane/connect-inject/webhook_v2/dns.go new file mode 100644 index 0000000000..e7aaa67830 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/dns.go @@ -0,0 +1,93 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "fmt" + "strconv" + + "github.com/miekg/dns" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +const ( + // These defaults are taken from the /etc/resolv.conf man page + // and are used by the dns library. + defaultDNSOptionNdots = 1 + defaultDNSOptionTimeout = 5 + defaultDNSOptionAttempts = 2 + + // defaultEtcResolvConfFile is the default location of the /etc/resolv.conf file. + defaultEtcResolvConfFile = "/etc/resolv.conf" +) + +func (w *MeshWebhook) configureDNS(pod *corev1.Pod, k8sNS string) error { + // First, we need to determine the nameservers configured in this cluster from /etc/resolv.conf. + etcResolvConf := defaultEtcResolvConfFile + if w.etcResolvFile != "" { + etcResolvConf = w.etcResolvFile + } + cfg, err := dns.ClientConfigFromFile(etcResolvConf) + if err != nil { + return err + } + + // Set DNS policy on the pod to None because we want DNS to work according to the config we will provide. + pod.Spec.DNSPolicy = corev1.DNSNone + + // Set the consul-dataplane's DNS server as the first server in the list (i.e. localhost). + // We want to do that so that when consul cannot resolve the record, we will fall back to the nameservers + // configured in our /etc/resolv.conf. It's important to add Consul DNS as the first nameserver because + // if we put kube DNS first, it will return NXDOMAIN response and a DNS client will not fall back to other nameservers. + if pod.Spec.DNSConfig == nil { + nameservers := []string{consulDataplaneDNSBindHost} + nameservers = append(nameservers, cfg.Servers...) + var options []corev1.PodDNSConfigOption + if cfg.Ndots != defaultDNSOptionNdots { + ndots := strconv.Itoa(cfg.Ndots) + options = append(options, corev1.PodDNSConfigOption{ + Name: "ndots", + Value: &ndots, + }) + } + if cfg.Timeout != defaultDNSOptionTimeout { + options = append(options, corev1.PodDNSConfigOption{ + Name: "timeout", + Value: pointer.String(strconv.Itoa(cfg.Timeout)), + }) + } + if cfg.Attempts != defaultDNSOptionAttempts { + options = append(options, corev1.PodDNSConfigOption{ + Name: "attempts", + Value: pointer.String(strconv.Itoa(cfg.Attempts)), + }) + } + + // Replace release namespace in the searches with the pod namespace. + // This is so that the searches we generate will be for the pod's namespace + // instead of the namespace of the connect-injector. E.g. instead of + // consul.svc.cluster.local it should be .svc.cluster.local. + var searches []string + // Kubernetes will add a search domain for .svc.cluster.local so we can always + // expect it to be there. See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#namespaces-of-services. + consulReleaseNSSearchDomain := fmt.Sprintf("%s.svc.cluster.local", w.ReleaseNamespace) + for _, search := range cfg.Search { + if search == consulReleaseNSSearchDomain { + searches = append(searches, fmt.Sprintf("%s.svc.cluster.local", k8sNS)) + } else { + searches = append(searches, search) + } + } + + pod.Spec.DNSConfig = &corev1.PodDNSConfig{ + Nameservers: nameservers, + Searches: searches, + Options: options, + } + } else { + return fmt.Errorf("DNS redirection to Consul is not supported with an already defined DNSConfig on the pod") + } + return nil +} diff --git a/control-plane/connect-inject/webhook_v2/dns_test.go b/control-plane/connect-inject/webhook_v2/dns_test.go new file mode 100644 index 0000000000..7c45a5e577 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/dns_test.go @@ -0,0 +1,105 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" +) + +func TestMeshWebhook_configureDNS(t *testing.T) { + cases := map[string]struct { + etcResolv string + expDNSConfig *corev1.PodDNSConfig + }{ + "empty /etc/resolv.conf file": { + expDNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"127.0.0.1"}, + }, + }, + "one nameserver": { + etcResolv: `nameserver 1.1.1.1`, + expDNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"127.0.0.1", "1.1.1.1"}, + }, + }, + "mutiple nameservers, searches, and options": { + etcResolv: ` +nameserver 1.1.1.1 +nameserver 2.2.2.2 +search foo.bar bar.baz +options ndots:5 timeout:6 attempts:3`, + expDNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"127.0.0.1", "1.1.1.1", "2.2.2.2"}, + Searches: []string{"foo.bar", "bar.baz"}, + Options: []corev1.PodDNSConfigOption{ + { + Name: "ndots", + Value: pointer.String("5"), + }, + { + Name: "timeout", + Value: pointer.String("6"), + }, + { + Name: "attempts", + Value: pointer.String("3"), + }, + }, + }, + }, + "replaces release specific search domains": { + etcResolv: ` +nameserver 1.1.1.1 +nameserver 2.2.2.2 +search consul.svc.cluster.local svc.cluster.local cluster.local +options ndots:5`, + expDNSConfig: &corev1.PodDNSConfig{ + Nameservers: []string{"127.0.0.1", "1.1.1.1", "2.2.2.2"}, + Searches: []string{"default.svc.cluster.local", "svc.cluster.local", "cluster.local"}, + Options: []corev1.PodDNSConfigOption{ + { + Name: "ndots", + Value: pointer.String("5"), + }, + }, + }, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + etcResolvFile, err := os.CreateTemp("", "") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(etcResolvFile.Name()) + }) + _, err = etcResolvFile.WriteString(c.etcResolv) + require.NoError(t, err) + w := MeshWebhook{ + etcResolvFile: etcResolvFile.Name(), + ReleaseNamespace: "consul", + } + + pod := minimal() + err = w.configureDNS(pod, "default") + require.NoError(t, err) + require.Equal(t, corev1.DNSNone, pod.Spec.DNSPolicy) + require.Equal(t, c.expDNSConfig, pod.Spec.DNSConfig) + }) + } +} + +func TestMeshWebhook_configureDNS_error(t *testing.T) { + w := MeshWebhook{} + + pod := minimal() + pod.Spec.DNSConfig = &corev1.PodDNSConfig{Nameservers: []string{"1.1.1.1"}} + err := w.configureDNS(pod, "default") + require.EqualError(t, err, "DNS redirection to Consul is not supported with an already defined DNSConfig on the pod") +} diff --git a/control-plane/connect-inject/webhook_v2/health_checks_test.go b/control-plane/connect-inject/webhook_v2/health_checks_test.go new file mode 100644 index 0000000000..ce5f3937bf --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/health_checks_test.go @@ -0,0 +1,56 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReady(t *testing.T) { + + var cases = []struct { + name string + certFileContents *string + keyFileContents *string + expectError bool + }{ + {"Both cert and key files not present.", nil, nil, true}, + {"Cert file not empty and key file missing.", ptrToString("test"), nil, true}, + {"Key file not empty and cert file missing.", nil, ptrToString("test"), true}, + {"Both cert and key files are present and not empty.", ptrToString("test"), ptrToString("test"), false}, + {"Both cert and key files are present but both are empty.", ptrToString(""), ptrToString(""), true}, + {"Both cert and key files are present but key file is empty.", ptrToString("test"), ptrToString(""), true}, + {"Both cert and key files are present but cert file is empty.", ptrToString(""), ptrToString("test"), true}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "") + require.NoError(t, err) + if tt.certFileContents != nil { + err := os.WriteFile(filepath.Join(tmpDir, "tls.crt"), []byte(*tt.certFileContents), 0666) + require.NoError(t, err) + } + if tt.keyFileContents != nil { + err := os.WriteFile(filepath.Join(tmpDir, "tls.key"), []byte(*tt.keyFileContents), 0666) + require.NoError(t, err) + } + rc := ReadinessCheck{tmpDir} + err = rc.Ready(nil) + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func ptrToString(s string) *string { + return &s +} diff --git a/control-plane/connect-inject/webhook_v2/heath_checks.go b/control-plane/connect-inject/webhook_v2/heath_checks.go new file mode 100644 index 0000000000..6d92172e78 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/heath_checks.go @@ -0,0 +1,30 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "errors" + "net/http" + "os" + "path/filepath" +) + +type ReadinessCheck struct { + CertDir string +} + +func (r ReadinessCheck) Ready(_ *http.Request) error { + certFile, err := os.ReadFile(filepath.Join(r.CertDir, "tls.crt")) + if err != nil { + return err + } + keyFile, err := os.ReadFile(filepath.Join(r.CertDir, "tls.key")) + if err != nil { + return err + } + if len(certFile) == 0 || len(keyFile) == 0 { + return errors.New("certificate files have not been loaded") + } + return nil +} diff --git a/control-plane/connect-inject/webhook_v2/mesh_webhook.go b/control-plane/connect-inject/webhook_v2/mesh_webhook.go new file mode 100644 index 0000000000..efe33985f2 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/mesh_webhook.go @@ -0,0 +1,563 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + + mapset "github.com/deckarep/golang-set" + "github.com/go-logr/logr" + "golang.org/x/exp/slices" + "gomodules.xyz/jsonpatch/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" + "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/hashicorp/consul-k8s/control-plane/version" +) + +const ( + sidecarContainer = "consul-dataplane" + + // exposedPathsLivenessPortsRangeStart is the start of the port range that we will use as + // the ListenerPort for the Expose configuration of the proxy registration for a liveness probe. + exposedPathsLivenessPortsRangeStart = 20300 + + // exposedPathsReadinessPortsRangeStart is the start of the port range that we will use as + // the ListenerPort for the Expose configuration of the proxy registration for a readiness probe. + exposedPathsReadinessPortsRangeStart = 20400 + + // exposedPathsStartupPortsRangeStart is the start of the port range that we will use as + // the ListenerPort for the Expose configuration of the proxy registration for a startup probe. + exposedPathsStartupPortsRangeStart = 20500 +) + +// kubeSystemNamespaces is a set of namespaces that are considered +// "system" level namespaces and are always skipped (never injected). +var kubeSystemNamespaces = mapset.NewSetWith(metav1.NamespaceSystem, metav1.NamespacePublic) + +// MeshWebhook is the HTTP meshWebhook for admission webhooks. +type MeshWebhook struct { + Clientset kubernetes.Interface + + // ConsulClientConfig is the config to create a Consul API client. + ConsulConfig *consul.Config + + // ConsulServerConnMgr is the watcher for the Consul server addresses. + ConsulServerConnMgr consul.ServerConnectionManager + + // ImageConsul is the container image for Consul to use. + // ImageConsulDataplane is the container image for Envoy to use. + // + // Both of these MUST be set. + ImageConsul string + ImageConsulDataplane string + + // ImageConsulK8S is the container image for consul-k8s to use. + // This image is used for the consul-sidecar container. + ImageConsulK8S string + + // Optional: set when you need extra options to be set when running envoy + // See a list of args here: https://www.envoyproxy.io/docs/envoy/latest/operations/cli + EnvoyExtraArgs string + + // RequireAnnotation means that the annotation must be given to inject. + // If this is false, injection is default. + RequireAnnotation bool + + // AuthMethod is the name of the Kubernetes Auth Method to + // use for identity with connectInjection if ACLs are enabled. + AuthMethod string + + // The PEM-encoded CA certificate string + // to use when communicating with Consul clients over HTTPS. + // If not set, will use HTTP. + ConsulCACert string + + // TLSEnabled indicates whether we should use TLS for communicating to Consul. + TLSEnabled bool + + // ConsulAddress is the address of the Consul server. This should be only the + // host (i.e. not including port or protocol). + ConsulAddress string + + // ConsulTLSServerName is the SNI header to use to connect to the Consul servers + // over TLS. + ConsulTLSServerName string + + // ConsulPartition is the name of the Admin Partition that the controller + // is deployed in. It is an enterprise feature requiring Consul Enterprise 1.11+. + // Its value is an empty string if partitions aren't enabled. + ConsulPartition string + + // EnableNamespaces indicates that a user is running Consul Enterprise + // with version 1.7+ which is namespace aware. It enables Consul namespaces, + // with injection into either a single Consul namespace or mirrored from + // k8s namespaces. + EnableNamespaces bool + + // AllowK8sNamespacesSet is a set of k8s namespaces to explicitly allow for + // injection. It supports the special character `*` which indicates that + // all k8s namespaces are eligible unless explicitly denied. This filter + // is applied before checking pod annotations. + AllowK8sNamespacesSet mapset.Set + + // DenyK8sNamespacesSet is a set of k8s namespaces to explicitly deny + // injection and thus service registration with Consul. An empty set + // means that no namespaces are removed from consideration. This filter + // takes precedence over AllowK8sNamespacesSet. + DenyK8sNamespacesSet mapset.Set + + // ConsulDestinationNamespace is the name of the Consul namespace to register all + // injected services into if Consul namespaces are enabled and mirroring + // is disabled. This may be set, but will not be used if mirroring is enabled. + ConsulDestinationNamespace string + + // EnableK8SNSMirroring causes Consul namespaces to be created to match the + // k8s namespace of any service being registered into Consul. Services are + // registered into the Consul namespace that mirrors their k8s namespace. + EnableK8SNSMirroring bool + + // K8SNSMirroringPrefix is an optional prefix that can be added to the Consul + // namespaces created while mirroring. For example, if it is set to "k8s-", + // then the k8s `default` namespace will be mirrored in Consul's + // `k8s-default` namespace. + K8SNSMirroringPrefix string + + // CrossNamespaceACLPolicy is the name of the ACL policy to attach to + // any created Consul namespaces to allow cross namespace service discovery. + // Only necessary if ACLs are enabled. + CrossNamespaceACLPolicy string + + // Default resource settings for sidecar proxies. Some of these + // fields may be empty. + DefaultProxyCPURequest resource.Quantity + DefaultProxyCPULimit resource.Quantity + DefaultProxyMemoryRequest resource.Quantity + DefaultProxyMemoryLimit resource.Quantity + + // LifecycleConfig contains proxy lifecycle management configuration from the inject-connect command and has methods to determine whether + // configuration should come from the default flags or annotations. The meshWebhook uses this to configure container sidecar proxy args. + LifecycleConfig lifecycle.Config + + // Default Envoy concurrency flag, this is the number of worker threads to be used by the proxy. + DefaultEnvoyProxyConcurrency int + + // MetricsConfig contains metrics configuration from the inject-connect command and has methods to determine whether + // configuration should come from the default flags or annotations. The meshWebhook uses this to configure prometheus + // annotations and the merged metrics server. + MetricsConfig metrics.Config + + // Resource settings for init container. All of these fields + // will be populated by the defaults provided in the initial flags. + InitContainerResources corev1.ResourceRequirements + + // Resource settings for Consul sidecar. All of these fields + // will be populated by the defaults provided in the initial flags. + DefaultConsulSidecarResources corev1.ResourceRequirements + + // EnableTransparentProxy enables transparent proxy mode. + // This means that the injected init container will apply traffic redirection rules + // so that all traffic will go through the Envoy proxy. + EnableTransparentProxy bool + + // EnableCNI enables the CNI plugin and prevents the connect-inject init container + // from running the consul redirect-traffic command as the CNI plugin handles traffic + // redirection + EnableCNI bool + + // TProxyOverwriteProbes controls whether the webhook should mutate pod's HTTP probes + // to point them to the Envoy proxy. + TProxyOverwriteProbes bool + + // EnableConsulDNS enables traffic redirection so that DNS requests are directed to Consul + // from mesh services. + EnableConsulDNS bool + + // EnableOpenShift indicates that when tproxy is enabled, the security context for the Envoy and init + // containers should not be added because OpenShift sets a random user for those and will not allow + // those containers to be created otherwise. + EnableOpenShift bool + + // SkipServerWatch prevents consul-dataplane from consuming the server update stream. This is useful + // for situations where Consul servers are behind a load balancer. + SkipServerWatch bool + + // ReleaseNamespace is the Kubernetes namespace where this webhook is running. + ReleaseNamespace string + + // Log + Log logr.Logger + // Log settings for consul-dataplane and connect-init containers. + LogLevel string + LogJSON bool + + decoder *admission.Decoder + // etcResolvFile is only used in tests to stub out /etc/resolv.conf file. + etcResolvFile string +} + +// Handle is the admission.Webhook implementation that actually handles the +// webhook request for admission control. This should be registered or +// served via the controller runtime manager. +func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admission.Response { + var pod corev1.Pod + + // Decode the pod from the request + if err := w.decoder.Decode(req, &pod); err != nil { + w.Log.Error(err, "could not unmarshal request to pod") + return admission.Errored(http.StatusBadRequest, err) + } + + // Marshall the contents of the pod that was received. This is compared with the + // marshalled contents of the pod after it has been updated to create the jsonpatch. + origPodJson, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Setup the default annotation values that are used for the container. + // This MUST be done before shouldInject is called since that function + // uses these annotations. + if err := w.defaultAnnotations(&pod, string(origPodJson)); err != nil { + w.Log.Error(err, "error creating default annotations", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error creating default annotations: %s", err)) + } + + // Check if we should inject, for example we don't inject in the + // system namespaces. + if shouldInject, err := w.shouldInject(pod, req.Namespace); err != nil { + w.Log.Error(err, "error checking if should inject", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error checking if should inject: %s", err)) + } else if !shouldInject { + return admission.Allowed(fmt.Sprintf("%s %s does not require injection", pod.Kind, pod.Name)) + } + + w.Log.Info("received pod", "name", req.Name, "ns", req.Namespace) + + // Add our volume that will be shared by the init container and + // the sidecar for passing data in the pod. + pod.Spec.Volumes = append(pod.Spec.Volumes, w.containerVolume()) + + // Optionally mount data volume to other containers + w.injectVolumeMount(pod) + + // Optionally add any volumes that are to be used by the envoy sidecar. + if _, ok := pod.Annotations[constants.AnnotationConsulSidecarUserVolume]; ok { + var userVolumes []corev1.Volume + err := json.Unmarshal([]byte(pod.Annotations[constants.AnnotationConsulSidecarUserVolume]), &userVolumes) + if err != nil { + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error unmarshalling sidecar user volumes: %s", err)) + } + pod.Spec.Volumes = append(pod.Spec.Volumes, userVolumes...) + } + + // Add the upstream services as environment variables for easy + // service discovery. + containerEnvVars := w.containerEnvVars(pod) + for i := range pod.Spec.InitContainers { + pod.Spec.InitContainers[i].Env = append(pod.Spec.InitContainers[i].Env, containerEnvVars...) + } + + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, containerEnvVars...) + } + + // A user can enable/disable tproxy for an entire namespace via a label. + ns, err := w.Clientset.CoreV1().Namespaces().Get(ctx, req.Namespace, metav1.GetOptions{}) + if err != nil { + w.Log.Error(err, "error fetching namespace metadata for container", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error getting namespace metadata for container: %s", err)) + } + + lifecycleEnabled, ok := w.LifecycleConfig.EnableProxyLifecycle(pod) + if ok != nil { + w.Log.Error(err, "unable to get lifecycle enabled status") + } + // Add the init container that registers the service and sets up the Envoy configuration. + initContainer, err := w.containerInit(*ns, pod) + if err != nil { + w.Log.Error(err, "error configuring injection init container", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection init container: %s", err)) + } + pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) + + // Add the Envoy sidecar. + envoySidecar, err := w.consulDataplaneSidecar(*ns, pod) + if err != nil { + w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err)) + } + //Append the Envoy sidecar before the application container only if lifecycle enabled. + + if lifecycleEnabled && ok == nil { + pod.Spec.Containers = append([]corev1.Container{envoySidecar}, pod.Spec.Containers...) + } else { + pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar) + } + + // pod.Annotations has already been initialized by h.defaultAnnotations() + // and does not need to be checked for being a nil value. + pod.Annotations[constants.KeyMeshInjectStatus] = constants.Injected + + tproxyEnabled, err := common.TransparentProxyEnabled(*ns, pod, w.EnableTransparentProxy) + if err != nil { + w.Log.Error(err, "error determining if transparent proxy is enabled", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error determining if transparent proxy is enabled: %s", err)) + } + + // Add an annotation to the pod sets transparent-proxy-status to enabled or disabled. Used by the CNI plugin + // to determine if it should traffic redirect or not. + if tproxyEnabled { + pod.Annotations[constants.KeyTransparentProxyStatus] = constants.Enabled + } + + // If DNS redirection is enabled, we want to configure dns on the pod. + dnsEnabled, err := consulDNSEnabled(*ns, pod, w.EnableConsulDNS, w.EnableTransparentProxy) + if err != nil { + w.Log.Error(err, "error determining if dns redirection is enabled", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error determining if dns redirection is enabled: %s", err)) + } + if dnsEnabled { + if err = w.configureDNS(&pod, req.Namespace); err != nil { + w.Log.Error(err, "error configuring DNS on the pod", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring DNS on the pod: %s", err)) + } + } + + // Add annotations for metrics. + if err = w.prometheusAnnotations(&pod); err != nil { + w.Log.Error(err, "error configuring prometheus annotations", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring prometheus annotations: %s", err)) + } + + if pod.Labels == nil { + pod.Labels = make(map[string]string) + } + pod.Labels[constants.KeyMeshInjectStatus] = constants.Injected + + // Consul-ENT only: Add the Consul destination namespace as an annotation to the pod. + if w.EnableNamespaces { + pod.Annotations[constants.AnnotationConsulNamespace] = w.consulNamespace(req.Namespace) + } + + // Overwrite readiness/liveness probes if needed. + err = w.overwriteProbes(*ns, &pod) + if err != nil { + w.Log.Error(err, "error overwriting readiness or liveness probes", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error overwriting readiness or liveness probes: %s", err)) + } + + // When CNI and tproxy are enabled, we add an annotation to the pod that contains the iptables config so that the CNI + // plugin can apply redirect traffic rules on the pod. + if w.EnableCNI && tproxyEnabled { + if err = w.addRedirectTrafficConfigAnnotation(&pod, *ns); err != nil { + w.Log.Error(err, "error configuring annotation for CNI traffic redirection", "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring annotation for CNI traffic redirection: %s", err)) + } + } + + // Marshall the pod into JSON after it has the desired envs, annotations, labels, + // sidecars and initContainers appended to it. + updatedPodJson, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Create a patches based on the Pod that was received by the meshWebhook + // and the desired Pod spec. + patches, err := jsonpatch.CreatePatch(origPodJson, updatedPodJson) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + // Check and potentially create Consul resources. This is done after + // all patches are created to guarantee no errors were encountered in + // that process before modifying the Consul cluster. + if w.EnableNamespaces { + serverState, err := w.ConsulServerConnMgr.State() + if err != nil { + w.Log.Error(err, "error checking or creating namespace", + "ns", w.consulNamespace(req.Namespace), "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error checking or creating namespace: %s", err)) + } + apiClient, err := consul.NewClientFromConnMgrState(w.ConsulConfig, serverState) + if err != nil { + w.Log.Error(err, "error checking or creating namespace", + "ns", w.consulNamespace(req.Namespace), "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error checking or creating namespace: %s", err)) + } + if _, err := namespaces.EnsureExists(apiClient, w.consulNamespace(req.Namespace), w.CrossNamespaceACLPolicy); err != nil { + w.Log.Error(err, "error checking or creating namespace", + "ns", w.consulNamespace(req.Namespace), "request name", req.Name) + return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error checking or creating namespace: %s", err)) + } + } + + // Return a Patched response along with the patches we intend on applying to the + // Pod received by the meshWebhook. + return admission.Patched(fmt.Sprintf("valid %s request", pod.Kind), patches...) +} + +// overwriteProbes overwrites readiness/liveness probes of this pod when +// both transparent proxy is enabled and overwrite probes is true for the pod. +func (w *MeshWebhook) overwriteProbes(ns corev1.Namespace, pod *corev1.Pod) error { + tproxyEnabled, err := common.TransparentProxyEnabled(ns, *pod, w.EnableTransparentProxy) + if err != nil { + return err + } + + overwriteProbes, err := common.ShouldOverwriteProbes(*pod, w.TProxyOverwriteProbes) + if err != nil { + return err + } + + if tproxyEnabled && overwriteProbes { + // We don't use the loop index because this needs to line up w.withiptablesConfigJSON, + // which is performed before the sidecar is injected. + idx := 0 + for _, container := range pod.Spec.Containers { + // skip the "envoy-sidecar" container from having it's probes overridden + if container.Name == sidecarContainer { + continue + } + if container.LivenessProbe != nil && container.LivenessProbe.HTTPGet != nil { + container.LivenessProbe.HTTPGet.Port = intstr.FromInt(exposedPathsLivenessPortsRangeStart + idx) + } + if container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + container.ReadinessProbe.HTTPGet.Port = intstr.FromInt(exposedPathsReadinessPortsRangeStart + idx) + } + if container.StartupProbe != nil && container.StartupProbe.HTTPGet != nil { + container.StartupProbe.HTTPGet.Port = intstr.FromInt(exposedPathsStartupPortsRangeStart + idx) + } + idx++ + } + } + return nil +} + +func (w *MeshWebhook) injectVolumeMount(pod corev1.Pod) { + containersToInject := splitCommaSeparatedItemsFromAnnotation(constants.AnnotationInjectMountVolumes, pod) + + for index, container := range pod.Spec.Containers { + if slices.Contains(containersToInject, container.Name) { + pod.Spec.Containers[index].VolumeMounts = append(pod.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: "/consul/connect-inject", + }) + } + } +} + +func (w *MeshWebhook) shouldInject(pod corev1.Pod, namespace string) (bool, error) { + // Don't inject in the Kubernetes system namespaces + if kubeSystemNamespaces.Contains(namespace) { + return false, nil + } + + // Namespace logic + // If in deny list, don't inject + if w.DenyK8sNamespacesSet.Contains(namespace) { + return false, nil + } + + // If not in allow list or allow list is not *, don't inject + if !w.AllowK8sNamespacesSet.Contains("*") && !w.AllowK8sNamespacesSet.Contains(namespace) { + return false, nil + } + + // If we already injected then don't inject again + if pod.Annotations[constants.KeyMeshInjectStatus] != "" || pod.Annotations[constants.KeyInjectStatus] != "" { + return false, nil + } + + // If the explicit true/false is on, then take that value. Note that + // this has to be the last check since it sets a default value after + // all other checks. + if raw, ok := pod.Annotations[constants.AnnotationMeshInject]; ok { + return strconv.ParseBool(raw) + } + + return !w.RequireAnnotation, nil +} + +func (w *MeshWebhook) defaultAnnotations(pod *corev1.Pod, podJson string) error { + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + + pod.Annotations[constants.AnnotationOriginalPod] = podJson + pod.Annotations[constants.AnnotationConsulK8sVersion] = version.GetHumanVersion() + + return nil +} + +// prometheusAnnotations sets the Prometheus scraping configuration +// annotations on the Pod. +func (w *MeshWebhook) prometheusAnnotations(pod *corev1.Pod) error { + enableMetrics, err := w.MetricsConfig.EnableMetrics(*pod) + if err != nil { + return err + } + prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(*pod) + if err != nil { + return err + } + prometheusScrapePath := w.MetricsConfig.PrometheusScrapePath(*pod) + + if enableMetrics { + pod.Annotations[constants.AnnotationPrometheusScrape] = "true" + pod.Annotations[constants.AnnotationPrometheusPort] = prometheusScrapePort + pod.Annotations[constants.AnnotationPrometheusPath] = prometheusScrapePath + } + return nil +} + +// consulNamespace returns the namespace that a service should be +// registered in based on the namespace options. It returns an +// empty string if namespaces aren't enabled. +func (w *MeshWebhook) consulNamespace(ns string) string { + return namespaces.ConsulNamespace(ns, w.EnableNamespaces, w.ConsulDestinationNamespace, w.EnableK8SNSMirroring, w.K8SNSMirroringPrefix) +} + +func findServiceAccountVolumeMount(pod corev1.Pod) (corev1.VolumeMount, string, error) { + // Find the volume mount that is mounted at the known + // service account token location + var volumeMount corev1.VolumeMount + for _, container := range pod.Spec.Containers { + for _, vm := range container.VolumeMounts { + if vm.MountPath == "/var/run/secrets/kubernetes.io/serviceaccount" { + volumeMount = vm + break + } + } + } + + // Return an error if volumeMount is still empty + if (corev1.VolumeMount{}) == volumeMount { + return volumeMount, "", errors.New("unable to find service account token volumeMount") + } + + return volumeMount, "/var/run/secrets/kubernetes.io/serviceaccount/token", nil +} + +func (w *MeshWebhook) InjectDecoder(d *admission.Decoder) error { + w.decoder = d + return nil +} diff --git a/control-plane/connect-inject/webhook_v2/mesh_webhook_ent_test.go b/control-plane/connect-inject/webhook_v2/mesh_webhook_ent_test.go new file mode 100644 index 0000000000..d6920d83a2 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/mesh_webhook_ent_test.go @@ -0,0 +1,656 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build enterprise + +package webhook_v2 + +import ( + "context" + "testing" + + "github.com/deckarep/golang-set" + logrtest "github.com/go-logr/logr/testing" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/sdk/testutil" + "github.com/stretchr/testify/require" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/helper/test" +) + +// This tests the checkAndCreate namespace function that is called +// in meshWebhook.Mutate. Patch generation is tested in the non-enterprise +// tests. Other namespace-specific logic is tested directly in the +// specific methods (shouldInject, consulNamespace). +func TestHandler_MutateWithNamespaces(t *testing.T) { + t.Parallel() + + basicSpec := corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + } + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{Group: "", Version: "v1"}, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + + cases := []struct { + Name string + Webhook MeshWebhook + Req admission.Request + ExpectedNamespaces []string + }{ + { + Name: "single destination namespace 'default' from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default"}, + }, + + { + Name: "single destination namespace 'default' from k8s 'non-default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", + decoder: decoder, + Clientset: clientWithNamespace("non-default"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "non-default", + }, + }, + ExpectedNamespaces: []string{"default"}, + }, + + { + Name: "single destination namespace 'dest' from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "dest", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default", "dest"}, + }, + + { + Name: "single destination namespace 'dest' from k8s 'non-default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "dest", + decoder: decoder, + Clientset: clientWithNamespace("non-default"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "non-default", + }, + }, + ExpectedNamespaces: []string{"default", "dest"}, + }, + + { + Name: "mirroring from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default"}, + }, + + { + Name: "mirroring from k8s 'dest'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + decoder: decoder, + Clientset: clientWithNamespace("dest"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "dest", + }, + }, + ExpectedNamespaces: []string{"default", "dest"}, + }, + + { + Name: "mirroring with prefix from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + K8SNSMirroringPrefix: "k8s-", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default", "k8s-default"}, + }, + + { + Name: "mirroring with prefix from k8s 'dest'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + K8SNSMirroringPrefix: "k8s-", + decoder: decoder, + Clientset: clientWithNamespace("dest"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "dest", + }, + }, + ExpectedNamespaces: []string{"default", "k8s-dest"}, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + testClient := test.TestServerWithMockConnMgrWatcher(t, nil) + client := testClient.APIClient + + // Add the client config and watcher to the test's meshWebhook + tt.Webhook.ConsulConfig = testClient.Cfg + tt.Webhook.ConsulServerConnMgr = testClient.Watcher + + // Mutate! + resp := tt.Webhook.Handle(context.Background(), tt.Req) + require.Equal(t, resp.Allowed, true) + + // Check all the namespace things + // Check that we have the right number of namespaces + namespaces, _, err := client.Namespaces().List(&api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, namespaces, len(tt.ExpectedNamespaces)) + + // Check the namespace details + for _, ns := range tt.ExpectedNamespaces { + actNamespace, _, err := client.Namespaces().Read(ns, &api.QueryOptions{}) + require.NoErrorf(t, err, "error getting namespace %s", ns) + require.NotNilf(t, actNamespace, "namespace %s was nil", ns) + require.Equalf(t, ns, actNamespace.Name, "namespace %s was improperly named", ns) + + // Check created namespace properties + if ns != "default" { + require.Equalf(t, "Auto-generated by consul-k8s", actNamespace.Description, + "wrong namespace description for namespace %s", ns) + require.Containsf(t, actNamespace.Meta, "external-source", + "namespace %s does not contain external-source metadata key", ns) + require.Equalf(t, "kubernetes", actNamespace.Meta["external-source"], + "namespace %s has wrong value for external-source metadata key", ns) + } + + } + }) + } +} + +// Tests that the correct cross-namespace policy is +// added to created namespaces. +func TestHandler_MutateWithNamespaces_ACLs(t *testing.T) { + basicSpec := corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + } + + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{Group: "", Version: "v1"}, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + + cases := []struct { + Name string + Webhook MeshWebhook + Req admission.Request + ExpectedNamespaces []string + }{ + { + Name: "acls + single destination namespace 'default' from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default"}, + }, + + { + Name: "acls + single destination namespace 'default' from k8s 'non-default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: clientWithNamespace("non-default"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "non-default", + }, + }, + ExpectedNamespaces: []string{"default"}, + }, + + { + Name: "acls + single destination namespace 'dest' from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "dest", + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default", "dest"}, + }, + + { + Name: "acls + single destination namespace 'dest' from k8s 'non-default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "dest", + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: clientWithNamespace("non-default"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "non-default", + }, + }, + ExpectedNamespaces: []string{"default", "dest"}, + }, + + { + Name: "acls + mirroring from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default"}, + }, + + { + Name: "acls + mirroring from k8s 'dest'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: clientWithNamespace("dest"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "dest", + }, + }, + ExpectedNamespaces: []string{"default", "dest"}, + }, + + { + Name: "acls + mirroring with prefix from k8s 'default'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + K8SNSMirroringPrefix: "k8s-", + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "default", + }, + }, + ExpectedNamespaces: []string{"default", "k8s-default"}, + }, + + { + Name: "acls + mirroring with prefix from k8s 'dest'", + Webhook: MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: "default", // will be overridden + EnableK8SNSMirroring: true, + K8SNSMirroringPrefix: "k8s-", + CrossNamespaceACLPolicy: "cross-namespace-policy", + decoder: decoder, + Clientset: clientWithNamespace("dest"), + }, + Req: admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + Namespace: "dest", + }, + }, + ExpectedNamespaces: []string{"default", "k8s-dest"}, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + // Set up consul server + adminToken := "123e4567-e89b-12d3-a456-426614174000" + testClient := test.TestServerWithMockConnMgrWatcher(t, func(c *testutil.TestServerConfig) { + c.ACL.Enabled = true + c.ACL.Tokens.InitialManagement = adminToken + }) + client := testClient.APIClient + + // Add the client config and watcher to the test's meshWebhook + tt.Webhook.ConsulConfig = testClient.Cfg + tt.Webhook.ConsulServerConnMgr = testClient.Watcher + + // Create cross namespace policy + // This would have been created by the acl bootstrapper in the + // default namespace to be attached to all created namespaces. + crossNamespaceRules := `namespace_prefix "" { + service_prefix "" { + policy = "read" + } + node_prefix "" { + policy = "read" + } +} ` + + policyTmpl := api.ACLPolicy{ + Name: "cross-namespace-policy", + Description: "Policy to allow permissions to cross Consul namespaces for k8s services", + Rules: crossNamespaceRules, + } + + _, _, err = client.ACL().PolicyCreate(&policyTmpl, &api.WriteOptions{}) + require.NoError(t, err) + + // Mutate! + resp := tt.Webhook.Handle(context.Background(), tt.Req) + require.Equal(t, resp.Allowed, true) + + // Check all the namespace things + // Check that we have the right number of namespaces + namespaces, _, err := client.Namespaces().List(&api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, namespaces, len(tt.ExpectedNamespaces)) + + // Check the namespace details + for _, ns := range tt.ExpectedNamespaces { + actNamespace, _, err := client.Namespaces().Read(ns, &api.QueryOptions{}) + require.NoErrorf(t, err, "error getting namespace %s", ns) + require.NotNilf(t, actNamespace, "namespace %s was nil", ns) + require.Equalf(t, ns, actNamespace.Name, "namespace %s was improperly named", ns) + + // Check created namespace properties + if ns != "default" { + require.Equalf(t, "Auto-generated by consul-k8s", actNamespace.Description, + "wrong namespace description for namespace %s", ns) + require.Containsf(t, actNamespace.Meta, "external-source", + "namespace %s does not contain external-source metadata key", ns) + require.Equalf(t, "kubernetes", actNamespace.Meta["external-source"], + "namespace %s has wrong value for external-source metadata key", ns) + + // Check for ACL policy things + // The acl bootstrapper will update the `default` namespace, so that + // can't be tested here. + require.NotNilf(t, actNamespace.ACLs, "ACLs was nil for namespace %s", ns) + require.Lenf(t, actNamespace.ACLs.PolicyDefaults, 1, "wrong length for PolicyDefaults in namespace %s", ns) + require.Equalf(t, "cross-namespace-policy", actNamespace.ACLs.PolicyDefaults[0].Name, + "wrong policy name for namespace %s", ns) + } + + } + }) + } +} + +// Test that the annotation for the Consul namespace is added. +func TestHandler_MutateWithNamespaces_Annotation(t *testing.T) { + t.Parallel() + sourceKubeNS := "kube-ns" + + cases := map[string]struct { + ConsulDestinationNamespace string + Mirroring bool + MirroringPrefix string + ExpNamespaceAnnotation string + }{ + "dest: default": { + ConsulDestinationNamespace: "default", + ExpNamespaceAnnotation: "default", + }, + "dest: foo": { + ConsulDestinationNamespace: "foo", + ExpNamespaceAnnotation: "foo", + }, + "mirroring": { + Mirroring: true, + ExpNamespaceAnnotation: sourceKubeNS, + }, + "mirroring with prefix": { + Mirroring: true, + MirroringPrefix: "prefix-", + ExpNamespaceAnnotation: "prefix-" + sourceKubeNS, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + testClient := test.TestServerWithMockConnMgrWatcher(t, nil) + + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{Group: "", Version: "v1"}, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + + require.NoError(t, err) + + webhook := MeshWebhook{ + Log: logrtest.NewTestLogger(t), + AllowK8sNamespacesSet: mapset.NewSet("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableNamespaces: true, + ConsulDestinationNamespace: c.ConsulDestinationNamespace, + EnableK8SNSMirroring: c.Mirroring, + K8SNSMirroringPrefix: c.MirroringPrefix, + ConsulConfig: testClient.Cfg, + ConsulServerConnMgr: testClient.Watcher, + decoder: decoder, + Clientset: clientWithNamespace(sourceKubeNS), + } + + pod := corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Namespace: sourceKubeNS, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + request := admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &pod), + Namespace: sourceKubeNS, + }, + } + resp := webhook.Handle(context.Background(), request) + require.Equal(t, resp.Allowed, true) + + // Check that the annotation was added as a patch. + var consulNamespaceAnnotationValue string + for _, patch := range resp.Patches { + if patch.Path == "/metadata/annotations" { + for annotationName, annotationValue := range patch.Value.(map[string]interface{}) { + if annotationName == constants.AnnotationConsulNamespace { + consulNamespaceAnnotationValue = annotationValue.(string) + } + } + } + } + require.NotEmpty(t, consulNamespaceAnnotationValue, "no namespace annotation set") + require.Equal(t, c.ExpNamespaceAnnotation, consulNamespaceAnnotationValue) + }) + } +} diff --git a/control-plane/connect-inject/webhook_v2/mesh_webhook_test.go b/control-plane/connect-inject/webhook_v2/mesh_webhook_test.go new file mode 100644 index 0000000000..b2b1f47392 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/mesh_webhook_test.go @@ -0,0 +1,2043 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "context" + "encoding/json" + "strconv" + "strings" + "testing" + + mapset "github.com/deckarep/golang-set" + logrtest "github.com/go-logr/logr/testr" + "github.com/hashicorp/consul/sdk/iptables" + "github.com/stretchr/testify/require" + "gomodules.xyz/jsonpatch/v2" + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" + "github.com/hashicorp/consul-k8s/control-plane/consul" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/hashicorp/consul-k8s/control-plane/version" +) + +func TestHandlerHandle(t *testing.T) { + t.Parallel() + basicSpec := corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + } + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{ + Group: "", + Version: "v1", + }, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + + cases := []struct { + Name string + Webhook MeshWebhook + Req admission.Request + Err string // expected error string, not exact + Patches []jsonpatch.Operation + }{ + { + "kube-system namespace", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: metav1.NamespaceSystem, + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + }, + }, + "", + nil, + }, + + { + "already injected", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.KeyMeshInjectStatus: constants.Injected, + }, + }, + Spec: basicSpec, + }), + }, + }, + "", + nil, + }, + + { + "empty pod basic", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/metadata/labels", + }, + { + Operation: "add", + Path: "/metadata/annotations", + }, + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + }, + }, + { + "empty pod basic with lifecycle", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + LifecycleConfig: lifecycle.Config{DefaultEnableProxyLifecycle: true}, + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/metadata/labels", + }, + { + Operation: "add", + Path: "/metadata/annotations", + }, + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + + { + Operation: "add", + Path: "/spec/containers/0/readinessProbe", + }, + { + Operation: "add", + Path: "/spec/containers/0/securityContext", + }, + { + Operation: "replace", + Path: "/spec/containers/0/name", + }, + { + Operation: "add", + Path: "/spec/containers/0/args", + }, + { + Operation: "add", + Path: "/spec/containers/0/env", + }, + { + Operation: "add", + Path: "/spec/containers/0/volumeMounts", + }, + }, + }, + // (TODO: ashwin) fix this test once upstreams get correctly processed + //{ + // "pod with upstreams specified", + // MeshWebhook{ + // Log: logrtest.New(t), + // AllowK8sNamespacesSet: mapset.NewSetWith("*"), + // DenyK8sNamespacesSet: mapset.NewSet(), + // decoder: decoder, + // Clientset: defaultTestClientWithNamespace(), + // }, + // admission.Request{ + // AdmissionRequest: admissionv1.AdmissionRequest{ + // Namespace: namespaces.DefaultNamespace, + // Object: encodeRaw(t, &corev1.Pod{ + // ObjectMeta: metav1.ObjectMeta{ + // Annotations: map[string]string{ + // constants.AnnotationMeshDestinations: "echo:1234,db:1234", + // }, + // }, + // Spec: basicSpec, + // }), + // }, + // }, + // "", + // []jsonpatch.Operation{ + // { + // Operation: "add", + // Path: "/metadata/labels", + // }, + // { + // Operation: "add", + // Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + // }, + // { + // Operation: "add", + // Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + // }, + // { + // Operation: "add", + // Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + // }, + // { + // Operation: "add", + // Path: "/spec/volumes", + // }, + // { + // Operation: "add", + // Path: "/spec/initContainers", + // }, + // { + // Operation: "add", + // Path: "/spec/containers/1", + // }, + // { + // Operation: "add", + // Path: "/spec/containers/0/env", + // }, + // }, + //}, + + { + "empty pod with injection disabled", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationMeshInject: "false", + }, + }, + Spec: basicSpec, + }), + }, + }, + "", + nil, + }, + + { + "empty pod with injection truthy", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationMeshInject: "t", + }, + }, + Spec: basicSpec, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + }, + }, + + { + "pod with empty volume mount annotation", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationInjectMountVolumes: "", + }, + }, + Spec: basicSpec, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + }, + }, + { + "pod with volume mount annotation", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationInjectMountVolumes: "web,unknown,web_three_point_oh", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web_two_point_oh", + }, + { + Name: "web_three_point_oh", + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/containers/0/volumeMounts", + }, + { + Operation: "add", + Path: "/spec/containers/2/volumeMounts", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/3", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + }, + }, + { + "pod with sidecar volume mount annotation", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationConsulSidecarUserVolume: "[{\"name\":\"bbb\",\"csi\":{\"driver\":\"bob\"}}]", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + }, + }, + { + "pod with sidecar invalid volume mount annotation", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationConsulSidecarUserVolume: "[a]", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + }), + }, + }, + "error unmarshalling sidecar user volumes: invalid character 'a' looking for beginning of value", + nil, + }, + { + "pod with service annotation", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + Spec: basicSpec, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + }, + }, + + { + "pod with existing label", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "testLabel": "123", + }, + }, + Spec: basicSpec, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/annotations", + }, + { + Operation: "add", + Path: "/metadata/labels/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + }, + }, + { + "tproxy with overwriteProbes is enabled", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableTransparentProxy: true, + TProxyOverwriteProbes: true, + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + // We're setting an existing annotation so that we can assert on the + // specific annotations that are set as a result of probes being overwritten. + Annotations: map[string]string{"foo": "bar"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8081), + }, + }, + }, + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyTransparentProxyStatus), + }, + + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "replace", + Path: "/spec/containers/0/livenessProbe/httpGet/port", + }, + { + Operation: "replace", + Path: "/spec/containers/0/readinessProbe/httpGet/port", + }, + }, + }, + { + "dns redirection enabled", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableTransparentProxy: true, + TProxyOverwriteProbes: true, + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + Annotations: map[string]string{constants.KeyConsulDNS: "true"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyTransparentProxyStatus), + }, + + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + { + Operation: "add", + Path: "/spec/dnsPolicy", + }, + { + Operation: "add", + Path: "/spec/dnsConfig", + }, + }, + }, + { + "dns redirection only enabled if tproxy enabled", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableTransparentProxy: true, + TProxyOverwriteProbes: true, + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + Annotations: map[string]string{ + constants.KeyConsulDNS: "true", + constants.KeyTransparentProxy: "false", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + // Note: no DNS policy/config additions. + }, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + tt.Webhook.ConsulConfig = &consul.Config{HTTPPort: 8500} + ctx := context.Background() + resp := tt.Webhook.Handle(ctx, tt.Req) + if (tt.Err == "") != resp.Allowed { + t.Fatalf("allowed: %v, expected err: %v", resp.Allowed, tt.Err) + } + if tt.Err != "" { + require.Contains(t, resp.Result.Message, tt.Err) + return + } + + actual := resp.Patches + if len(actual) > 0 { + for i := range actual { + actual[i].Value = nil + } + } + require.ElementsMatch(t, tt.Patches, actual) + }) + } +} + +// This test validates that overwrite probes match the iptables configuration fromiptablesConfigJSON() +// Because they happen at different points in the injection, the port numbers can get out of sync. +func TestHandlerHandle_ValidateOverwriteProbes(t *testing.T) { + t.Parallel() + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{ + Group: "", + Version: "v1", + }, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + + cases := []struct { + Name string + Webhook MeshWebhook + Req admission.Request + Err string // expected error string, not exact + Patches []jsonpatch.Operation + }{ + { + "tproxy with overwriteProbes is enabled", + MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + EnableTransparentProxy: true, + TProxyOverwriteProbes: true, + LifecycleConfig: lifecycle.Config{DefaultEnableProxyLifecycle: true}, + decoder: decoder, + Clientset: defaultTestClientWithNamespace(), + }, + admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Namespace: namespaces.DefaultNamespace, + Object: encodeRaw(t, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + // We're setting an existing annotation so that we can assert on the + // specific annotations that are set as a result of probes being overwritten. + Annotations: map[string]string{"foo": "bar"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8081), + }, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8082), + }, + }, + }, + }, + }, + }, + }), + }, + }, + "", + []jsonpatch.Operation{ + { + Operation: "add", + Path: "/spec/volumes", + }, + { + Operation: "add", + Path: "/spec/initContainers", + }, + { + Operation: "add", + Path: "/spec/containers/1", + }, + { + Operation: "replace", + Path: "/spec/containers/0/name", + }, + { + Operation: "add", + Path: "/spec/containers/0/args", + }, + { + Operation: "add", + Path: "/spec/containers/0/env", + }, + { + Operation: "add", + Path: "/spec/containers/0/volumeMounts", + }, + { + Operation: "add", + Path: "/spec/containers/0/readinessProbe/tcpSocket", + }, + { + Operation: "add", + Path: "/spec/containers/0/readinessProbe/initialDelaySeconds", + }, + { + Operation: "remove", + Path: "/spec/containers/0/readinessProbe/httpGet", + }, + { + Operation: "add", + Path: "/spec/containers/0/securityContext", + }, + { + Operation: "remove", + Path: "/spec/containers/0/startupProbe", + }, + { + Operation: "remove", + Path: "/spec/containers/0/livenessProbe", + }, + { + Operation: "add", + Path: "/metadata/labels", + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyMeshInjectStatus), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.KeyTransparentProxyStatus), + }, + + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationOriginalPod), + }, + { + Operation: "add", + Path: "/metadata/annotations/" + escapeJSONPointer(constants.AnnotationConsulK8sVersion), + }, + }, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + tt.Webhook.ConsulConfig = &consul.Config{HTTPPort: 8500} + ctx := context.Background() + resp := tt.Webhook.Handle(ctx, tt.Req) + if (tt.Err == "") != resp.Allowed { + t.Fatalf("allowed: %v, expected err: %v", resp.Allowed, tt.Err) + } + if tt.Err != "" { + require.Contains(t, resp.Result.Message, tt.Err) + return + } + + var iptablesCfg iptables.Config + var overwritePorts []string + actual := resp.Patches + if len(actual) > 0 { + for i := range actual { + + // We want to grab the iptables configuration from the connect-init container's + // environment. + if actual[i].Path == "/spec/initContainers" { + value := actual[i].Value.([]any) + valueMap := value[0].(map[string]any) + envs := valueMap["env"].([]any) + redirectEnv := envs[8].(map[string]any) + require.Equal(t, redirectEnv["name"].(string), "CONSUL_REDIRECT_TRAFFIC_CONFIG") + iptablesJson := redirectEnv["value"].(string) + + err := json.Unmarshal([]byte(iptablesJson), &iptablesCfg) + require.NoError(t, err) + } + + // We want to accumulate the httpGet Probes from the application container to + // compare them to the iptables rules. This is now the second container in the spec + if strings.Contains(actual[i].Path, "/spec/containers/1") { + valueMap, ok := actual[i].Value.(map[string]any) + require.True(t, ok) + + for k, v := range valueMap { + if strings.Contains(k, "Probe") { + probe := v.(map[string]any) + httpProbe := probe["httpGet"] + httpProbeMap := httpProbe.(map[string]any) + port := httpProbeMap["port"] + portNum := port.(float64) + + overwritePorts = append(overwritePorts, strconv.Itoa(int(portNum))) + } + } + } + + // nil out all the patch values to just compare the keys changing. + actual[i].Value = nil + } + } + // Make sure the iptables excluded ports match the ports on the container + require.ElementsMatch(t, iptablesCfg.ExcludeInboundPorts, overwritePorts) + require.ElementsMatch(t, tt.Patches, actual) + }) + } +} + +func TestHandlerDefaultAnnotations(t *testing.T) { + cases := []struct { + Name string + Pod *corev1.Pod + Expected map[string]string + Err string + }{ + { + "empty", + &corev1.Pod{}, + map[string]string{ + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":null},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), + }, + "", + }, + { + "basic pod, no ports", + &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + }, + map[string]string{ + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), + }, + "", + }, + { + "basic pod, with ports", + &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + Ports: []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: 8080, + }, + }, + }, + { + Name: "web-side", + }, + }, + }, + }, + map[string]string{ + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"ports\":[{\"name\":\"http\",\"containerPort\":8080}],\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), + }, + "", + }, + + { + "basic pod, with unnamed ports", + &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8080, + }, + }, + }, + { + Name: "web-side", + }, + }, + }, + }, + map[string]string{ + constants.AnnotationOriginalPod: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"containers\":[{\"name\":\"web\",\"ports\":[{\"containerPort\":8080}],\"resources\":{}},{\"name\":\"web-side\",\"resources\":{}}]},\"status\":{}}", + constants.AnnotationConsulK8sVersion: version.GetHumanVersion(), + }, + "", + }, + } + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + podJson, err := json.Marshal(tt.Pod) + require.NoError(t, err) + + var w MeshWebhook + err = w.defaultAnnotations(tt.Pod, string(podJson)) + if (tt.Err != "") != (err != nil) { + t.Fatalf("actual: %v, expected err: %v", err, tt.Err) + } + if tt.Err != "" { + require.Contains(t, err.Error(), tt.Err) + return + } + + actual := tt.Pod.Annotations + if len(actual) == 0 { + actual = nil + } + require.Equal(t, tt.Expected, actual) + }) + } +} + +func TestHandlerPrometheusAnnotations(t *testing.T) { + cases := []struct { + Name string + Webhook MeshWebhook + Expected map[string]string + }{ + { + Name: "Sets the correct prometheus annotations on the pod if metrics are enabled", + Webhook: MeshWebhook{ + MetricsConfig: metrics.Config{ + DefaultEnableMetrics: true, + DefaultPrometheusScrapePort: "20200", + DefaultPrometheusScrapePath: "/metrics", + }, + }, + Expected: map[string]string{ + constants.AnnotationPrometheusScrape: "true", + constants.AnnotationPrometheusPort: "20200", + constants.AnnotationPrometheusPath: "/metrics", + }, + }, + { + Name: "Does not set annotations if metrics are not enabled", + Webhook: MeshWebhook{ + MetricsConfig: metrics.Config{ + DefaultEnableMetrics: false, + DefaultPrometheusScrapePort: "20200", + DefaultPrometheusScrapePath: "/metrics", + }, + }, + Expected: map[string]string{}, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + h := tt.Webhook + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}} + + err := h.prometheusAnnotations(pod) + require.NoError(err) + + require.Equal(pod.Annotations, tt.Expected) + }) + } +} + +// Test consulNamespace function. +func TestConsulNamespace(t *testing.T) { + cases := []struct { + Name string + EnableNamespaces bool + ConsulDestinationNamespace string + EnableK8SNSMirroring bool + K8SNSMirroringPrefix string + K8sNamespace string + Expected string + }{ + { + "namespaces disabled", + false, + "default", + false, + "", + "namespace", + "", + }, + + { + "namespaces disabled, mirroring enabled", + false, + "default", + true, + "", + "namespace", + "", + }, + + { + "namespaces disabled, mirroring enabled, prefix defined", + false, + "default", + true, + "test-", + "namespace", + "", + }, + + { + "namespaces enabled, mirroring disabled", + true, + "default", + false, + "", + "namespace", + "default", + }, + + { + "namespaces enabled, mirroring disabled, prefix defined", + true, + "default", + false, + "test-", + "namespace", + "default", + }, + + { + "namespaces enabled, mirroring enabled", + true, + "default", + true, + "", + "namespace", + "namespace", + }, + + { + "namespaces enabled, mirroring enabled, prefix defined", + true, + "default", + true, + "test-", + "namespace", + "test-namespace", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + + w := MeshWebhook{ + EnableNamespaces: tt.EnableNamespaces, + ConsulDestinationNamespace: tt.ConsulDestinationNamespace, + EnableK8SNSMirroring: tt.EnableK8SNSMirroring, + K8SNSMirroringPrefix: tt.K8SNSMirroringPrefix, + } + + ns := w.consulNamespace(tt.K8sNamespace) + + require.Equal(tt.Expected, ns) + }) + } +} + +// Test shouldInject function. +func TestShouldInject(t *testing.T) { + cases := []struct { + Name string + Pod *corev1.Pod + K8sNamespace string + EnableNamespaces bool + AllowK8sNamespacesSet mapset.Set + DenyK8sNamespacesSet mapset.Set + Expected bool + }{ + { + "kube-system not injected", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + // Service annotation is required for injection + constants.AnnotationService: "testing", + }, + }, + }, + "kube-system", + false, + mapset.NewSet(), + mapset.NewSet(), + false, + }, + { + "kube-public not injected", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "kube-public", + false, + mapset.NewSet(), + mapset.NewSet(), + false, + }, + { + "namespaces disabled, empty allow/deny lists", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSet(), + mapset.NewSet(), + false, + }, + { + "namespaces disabled, allow *", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSetWith("*"), + mapset.NewSet(), + true, + }, + { + "namespaces disabled, allow default", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSetWith("default"), + mapset.NewSet(), + true, + }, + { + "namespaces disabled, allow * and default", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSetWith("*", "default"), + mapset.NewSet(), + true, + }, + { + "namespaces disabled, allow only ns1 and ns2", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSetWith("ns1", "ns2"), + mapset.NewSet(), + false, + }, + { + "namespaces disabled, deny default ns", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSet(), + mapset.NewSetWith("default"), + false, + }, + { + "namespaces disabled, allow *, deny default ns", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSetWith("*"), + mapset.NewSetWith("default"), + false, + }, + { + "namespaces disabled, default ns in both allow and deny lists", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + false, + mapset.NewSetWith("default"), + mapset.NewSetWith("default"), + false, + }, + { + "namespaces enabled, empty allow/deny lists", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSet(), + mapset.NewSet(), + false, + }, + { + "namespaces enabled, allow *", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSetWith("*"), + mapset.NewSet(), + true, + }, + { + "namespaces enabled, allow default", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSetWith("default"), + mapset.NewSet(), + true, + }, + { + "namespaces enabled, allow * and default", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSetWith("*", "default"), + mapset.NewSet(), + true, + }, + { + "namespaces enabled, allow only ns1 and ns2", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSetWith("ns1", "ns2"), + mapset.NewSet(), + false, + }, + { + "namespaces enabled, deny default ns", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSet(), + mapset.NewSetWith("default"), + false, + }, + { + "namespaces enabled, allow *, deny default ns", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSetWith("*"), + mapset.NewSetWith("default"), + false, + }, + { + "namespaces enabled, default ns in both allow and deny lists", + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "testing", + }, + }, + }, + "default", + true, + mapset.NewSetWith("default"), + mapset.NewSetWith("default"), + false, + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + + w := MeshWebhook{ + RequireAnnotation: false, + EnableNamespaces: tt.EnableNamespaces, + AllowK8sNamespacesSet: tt.AllowK8sNamespacesSet, + DenyK8sNamespacesSet: tt.DenyK8sNamespacesSet, + } + + injected, err := w.shouldInject(*tt.Pod, tt.K8sNamespace) + + require.Equal(nil, err) + require.Equal(tt.Expected, injected) + }) + } +} + +func TestOverwriteProbes(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + tproxyEnabled bool + overwriteProbes bool + podContainers []corev1.Container + expLivenessPort []int + expReadinessPort []int + expStartupPort []int + additionalAnnotations map[string]string + }{ + "transparent proxy disabled; overwrites probes disabled": { + tproxyEnabled: false, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + }, + }, + expReadinessPort: []int{8080}, + }, + "transparent proxy enabled; overwrite probes disabled": { + tproxyEnabled: true, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + }, + }, + expReadinessPort: []int{8080}, + }, + "transparent proxy disabled; overwrite probes enabled": { + tproxyEnabled: false, + overwriteProbes: true, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + }, + }, + expReadinessPort: []int{8080}, + }, + "just the readiness probe": { + tproxyEnabled: true, + overwriteProbes: true, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + }, + }, + expReadinessPort: []int{exposedPathsReadinessPortsRangeStart}, + }, + "just the liveness probe": { + tproxyEnabled: true, + overwriteProbes: true, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8081), + }, + }, + }, + }, + }, + expLivenessPort: []int{exposedPathsLivenessPortsRangeStart}, + }, + "skips envoy sidecar": { + tproxyEnabled: true, + overwriteProbes: true, + podContainers: []corev1.Container{ + { + Name: sidecarContainer, + }, + }, + }, + "readiness, liveness and startup probes": { + tproxyEnabled: true, + overwriteProbes: true, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8081), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8082), + }, + }, + }, + }, + }, + expLivenessPort: []int{exposedPathsLivenessPortsRangeStart}, + expReadinessPort: []int{exposedPathsReadinessPortsRangeStart}, + expStartupPort: []int{exposedPathsStartupPortsRangeStart}, + }, + "readiness, liveness and startup probes multiple containers": { + tproxyEnabled: true, + overwriteProbes: true, + podContainers: []corev1.Container{ + { + Name: "test", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8081, + }, + { + Name: "http", + ContainerPort: 8080, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8081), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + }, + }, + }, + }, + { + Name: "test-2", + Ports: []corev1.ContainerPort{ + { + Name: "tcp", + ContainerPort: 8083, + }, + { + Name: "http", + ContainerPort: 8082, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8083), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8082), + }, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(8082), + }, + }, + }, + }, + }, + expLivenessPort: []int{exposedPathsLivenessPortsRangeStart, exposedPathsLivenessPortsRangeStart + 1}, + expReadinessPort: []int{exposedPathsReadinessPortsRangeStart, exposedPathsReadinessPortsRangeStart + 1}, + expStartupPort: []int{exposedPathsStartupPortsRangeStart, exposedPathsStartupPortsRangeStart + 1}, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + Containers: c.podContainers, + }, + } + if c.additionalAnnotations != nil { + pod.ObjectMeta.Annotations = c.additionalAnnotations + } + + w := MeshWebhook{ + EnableTransparentProxy: c.tproxyEnabled, + TProxyOverwriteProbes: c.overwriteProbes, + } + err := w.overwriteProbes(corev1.Namespace{}, pod) + require.NoError(t, err) + for i, container := range pod.Spec.Containers { + if container.ReadinessProbe != nil { + require.Equal(t, c.expReadinessPort[i], container.ReadinessProbe.HTTPGet.Port.IntValue()) + } + if container.LivenessProbe != nil { + require.Equal(t, c.expLivenessPort[i], container.LivenessProbe.HTTPGet.Port.IntValue()) + } + if container.StartupProbe != nil { + require.Equal(t, c.expStartupPort[i], container.StartupProbe.HTTPGet.Port.IntValue()) + } + } + }) + } +} + +// encodeRaw is a helper to encode some data into a RawExtension. +func encodeRaw(t *testing.T, input interface{}) runtime.RawExtension { + data, err := json.Marshal(input) + require.NoError(t, err) + return runtime.RawExtension{Raw: data} +} + +// https://tools.ietf.org/html/rfc6901 +func escapeJSONPointer(s string) string { + s = strings.Replace(s, "~", "~0", -1) + s = strings.Replace(s, "/", "~1", -1) + return s +} + +func defaultTestClientWithNamespace() kubernetes.Interface { + return clientWithNamespace("default") +} + +func clientWithNamespace(name string) kubernetes.Interface { + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + return fake.NewSimpleClientset(&ns) +} diff --git a/control-plane/connect-inject/webhook_v2/redirect_traffic.go b/control-plane/connect-inject/webhook_v2/redirect_traffic.go new file mode 100644 index 0000000000..ac45df125f --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/redirect_traffic.go @@ -0,0 +1,137 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/hashicorp/consul/sdk/iptables" + corev1 "k8s.io/api/core/v1" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" +) + +// addRedirectTrafficConfigAnnotation creates an iptables.Config in JSON format based on proxy configuration. +// iptables.Config: +// +// ConsulDNSIP: an environment variable named RESOURCE_PREFIX_DNS_SERVICE_HOST where RESOURCE_PREFIX is the consul.fullname in helm. +// ProxyUserID: a constant set in Annotations +// ProxyInboundPort: the service port or bind port +// ProxyOutboundPort: default transparent proxy outbound port or transparent proxy outbound listener port +// ExcludeInboundPorts: prometheus, envoy stats, expose paths, checks and excluded pod annotations +// ExcludeOutboundPorts: pod annotations +// ExcludeOutboundCIDRs: pod annotations +// ExcludeUIDs: pod annotations +func (w *MeshWebhook) iptablesConfigJSON(pod corev1.Pod, ns corev1.Namespace) (string, error) { + cfg := iptables.Config{ + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + } + + // Set the proxy's inbound port. + cfg.ProxyInboundPort = constants.ProxyDefaultInboundPort + + // Set the proxy's outbound port. + cfg.ProxyOutboundPort = iptables.DefaultTProxyOutboundPort + + // If metrics are enabled, get the prometheusScrapePort and exclude it from the inbound ports + enableMetrics, err := w.MetricsConfig.EnableMetrics(pod) + if err != nil { + return "", err + } + if enableMetrics { + prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod) + if err != nil { + return "", err + } + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, prometheusScrapePort) + } + + // Exclude any overwritten liveness/readiness/startup ports from redirection. + overwriteProbes, err := common.ShouldOverwriteProbes(pod, w.TProxyOverwriteProbes) + if err != nil { + return "", err + } + + // Exclude the port on which the proxy health check port will be configured if + // using the proxy health check for a service. + if useProxyHealthCheck(pod) { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(constants.ProxyDefaultHealthPort)) + } + + if overwriteProbes { + // We don't use the loop index because this needs to line up w.overwriteProbes(), + // which is performed after the sidecar is injected. + idx := 0 + for _, container := range pod.Spec.Containers { + // skip the "consul-dataplane" container from having its probes overridden + if container.Name == sidecarContainer { + continue + } + if container.LivenessProbe != nil && container.LivenessProbe.HTTPGet != nil { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposedPathsLivenessPortsRangeStart+idx)) + } + if container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposedPathsReadinessPortsRangeStart+idx)) + } + if container.StartupProbe != nil && container.StartupProbe.HTTPGet != nil { + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, strconv.Itoa(exposedPathsStartupPortsRangeStart+idx)) + } + idx++ + } + } + + // Inbound ports + excludeInboundPorts := splitCommaSeparatedItemsFromAnnotation(constants.AnnotationTProxyExcludeInboundPorts, pod) + cfg.ExcludeInboundPorts = append(cfg.ExcludeInboundPorts, excludeInboundPorts...) + + // Outbound ports + excludeOutboundPorts := splitCommaSeparatedItemsFromAnnotation(constants.AnnotationTProxyExcludeOutboundPorts, pod) + cfg.ExcludeOutboundPorts = append(cfg.ExcludeOutboundPorts, excludeOutboundPorts...) + + // Outbound CIDRs + excludeOutboundCIDRs := splitCommaSeparatedItemsFromAnnotation(constants.AnnotationTProxyExcludeOutboundCIDRs, pod) + cfg.ExcludeOutboundCIDRs = append(cfg.ExcludeOutboundCIDRs, excludeOutboundCIDRs...) + + // UIDs + excludeUIDs := splitCommaSeparatedItemsFromAnnotation(constants.AnnotationTProxyExcludeUIDs, pod) + cfg.ExcludeUIDs = append(cfg.ExcludeUIDs, excludeUIDs...) + + // Add init container user ID to exclude from traffic redirection. + cfg.ExcludeUIDs = append(cfg.ExcludeUIDs, strconv.Itoa(initContainersUserAndGroupID)) + + dnsEnabled, err := consulDNSEnabled(ns, pod, w.EnableConsulDNS, w.EnableTransparentProxy) + if err != nil { + return "", err + } + + if dnsEnabled { + // If Consul DNS is enabled, we find the environment variable that has the value + // of the ClusterIP of the Consul DNS Service. constructDNSServiceHostName returns + // the name of the env variable whose value is the ClusterIP of the Consul DNS Service. + cfg.ConsulDNSIP = consulDataplaneDNSBindHost + cfg.ConsulDNSPort = consulDataplaneDNSBindPort + } + + iptablesConfigJson, err := json.Marshal(&cfg) + if err != nil { + return "", fmt.Errorf("could not marshal iptables config: %w", err) + } + + return string(iptablesConfigJson), nil +} + +// addRedirectTrafficConfigAnnotation add the created iptables JSON config as an annotation on the provided pod. +func (w *MeshWebhook) addRedirectTrafficConfigAnnotation(pod *corev1.Pod, ns corev1.Namespace) error { + iptablesConfig, err := w.iptablesConfigJSON(*pod, ns) + if err != nil { + return err + } + + pod.Annotations[constants.AnnotationRedirectTraffic] = iptablesConfig + + return nil +} diff --git a/control-plane/connect-inject/webhook_v2/redirect_traffic_test.go b/control-plane/connect-inject/webhook_v2/redirect_traffic_test.go new file mode 100644 index 0000000000..077189ead4 --- /dev/null +++ b/control-plane/connect-inject/webhook_v2/redirect_traffic_test.go @@ -0,0 +1,481 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package webhook_v2 + +import ( + "encoding/json" + "fmt" + "strconv" + "testing" + + mapset "github.com/deckarep/golang-set" + logrtest "github.com/go-logr/logr/testr" + "github.com/hashicorp/consul/sdk/iptables" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/consul" +) + +const ( + defaultPodName = "fakePod" + defaultNamespace = "default" +) + +func TestAddRedirectTrafficConfig(t *testing.T) { + s := runtime.NewScheme() + s.AddKnownTypes(schema.GroupVersion{ + Group: "", + Version: "v1", + }, &corev1.Pod{}) + decoder, err := admission.NewDecoder(s) + require.NoError(t, err) + cases := []struct { + name string + webhook MeshWebhook + pod *corev1.Pod + namespace corev1.Namespace + dnsEnabled bool + expCfg iptables.Config + expErr error + }{ + { + name: "basic bare minimum pod", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + }, + }, + { + name: "proxy health checks enabled", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationUseProxyHealthCheck: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"21000"}, + }, + }, + { + name: "metrics enabled", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationEnableMetrics: "true", + constants.AnnotationPrometheusScrapePort: "13373", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"13373"}, + }, + }, + { + name: "metrics enabled with incorrect annotation", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationEnableMetrics: "invalid", + constants.AnnotationPrometheusScrapePort: "13373", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"13373"}, + }, + expErr: fmt.Errorf("%s annotation value of %s was invalid: %s", constants.AnnotationEnableMetrics, "invalid", "strconv.ParseBool: parsing \"invalid\": invalid syntax"), + }, + { + name: "overwrite probes, transparent proxy annotation set", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationTransparentProxyOverwriteProbes: "true", + constants.KeyTransparentProxy: "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Port: intstr.FromInt(exposedPathsLivenessPortsRangeStart), + }, + }, + }, + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{strconv.Itoa(exposedPathsLivenessPortsRangeStart)}, + }, + }, + { + name: "exclude inbound ports", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationTProxyExcludeInboundPorts: "1111,11111", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeInboundPorts: []string{"1111", "11111"}, + }, + }, + { + name: "exclude outbound ports", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationTProxyExcludeOutboundPorts: "2222,22222", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"5996"}, + ExcludeOutboundPorts: []string{"2222", "22222"}, + }, + }, + { + name: "exclude outbound CIDRs", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationTProxyExcludeOutboundCIDRs: "3.3.3.3,3.3.3.3/24", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{strconv.Itoa(initContainersUserAndGroupID)}, + ExcludeOutboundCIDRs: []string{"3.3.3.3", "3.3.3.3/24"}, + }, + }, + { + name: "exclude UIDs", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationTProxyExcludeUIDs: "4444,44444", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ConsulDNSIP: "", + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeUIDs: []string{"4444", "44444", strconv.Itoa(initContainersUserAndGroupID)}, + }, + }, + { + name: "exclude inbound ports, outbound ports, outbound CIDRs, and UIDs", + webhook: MeshWebhook{ + Log: logrtest.New(t), + AllowK8sNamespacesSet: mapset.NewSetWith("*"), + DenyK8sNamespacesSet: mapset.NewSet(), + decoder: decoder, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: defaultNamespace, + Name: defaultPodName, + Annotations: map[string]string{ + constants.AnnotationTProxyExcludeInboundPorts: "1111,11111", + constants.AnnotationTProxyExcludeOutboundPorts: "2222,22222", + constants.AnnotationTProxyExcludeOutboundCIDRs: "3.3.3.3,3.3.3.3/24", + constants.AnnotationTProxyExcludeUIDs: "4444,44444", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + }, + }, + }, + }, + expCfg: iptables.Config{ + ProxyUserID: strconv.Itoa(sidecarUserAndGroupID), + ProxyInboundPort: constants.ProxyDefaultInboundPort, + ProxyOutboundPort: iptables.DefaultTProxyOutboundPort, + ExcludeInboundPorts: []string{"1111", "11111"}, + ExcludeOutboundPorts: []string{"2222", "22222"}, + ExcludeOutboundCIDRs: []string{"3.3.3.3", "3.3.3.3/24"}, + ExcludeUIDs: []string{"4444", "44444", strconv.Itoa(initContainersUserAndGroupID)}, + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err = c.webhook.addRedirectTrafficConfigAnnotation(c.pod, c.namespace) + + // Only compare annotation and iptables config on successful runs + if c.expErr == nil { + require.NoError(t, err) + anno, ok := c.pod.Annotations[constants.AnnotationRedirectTraffic] + require.Equal(t, ok, true) + + actualConfig := iptables.Config{} + err = json.Unmarshal([]byte(anno), &actualConfig) + require.NoError(t, err) + require.Equal(t, c.expCfg, actualConfig) + } else { + require.EqualError(t, err, c.expErr.Error()) + } + }) + } +} + +func TestRedirectTraffic_consulDNS(t *testing.T) { + cases := map[string]struct { + globalEnabled bool + annotations map[string]string + namespaceLabel map[string]string + expectConsulDNSConfig bool + }{ + "enabled globally, ns not set, annotation not provided": { + globalEnabled: true, + expectConsulDNSConfig: true, + }, + "enabled globally, ns not set, annotation is false": { + globalEnabled: true, + annotations: map[string]string{constants.KeyConsulDNS: "false"}, + expectConsulDNSConfig: false, + }, + "enabled globally, ns not set, annotation is true": { + globalEnabled: true, + annotations: map[string]string{constants.KeyConsulDNS: "true"}, + expectConsulDNSConfig: true, + }, + "disabled globally, ns not set, annotation not provided": { + expectConsulDNSConfig: false, + }, + "disabled globally, ns not set, annotation is false": { + annotations: map[string]string{constants.KeyConsulDNS: "false"}, + expectConsulDNSConfig: false, + }, + "disabled globally, ns not set, annotation is true": { + annotations: map[string]string{constants.KeyConsulDNS: "true"}, + expectConsulDNSConfig: true, + }, + "disabled globally, ns enabled, annotation not set": { + namespaceLabel: map[string]string{constants.KeyConsulDNS: "true"}, + expectConsulDNSConfig: true, + }, + "enabled globally, ns disabled, annotation not set": { + globalEnabled: true, + namespaceLabel: map[string]string{constants.KeyConsulDNS: "false"}, + expectConsulDNSConfig: false, + }, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + w := MeshWebhook{ + EnableConsulDNS: c.globalEnabled, + EnableTransparentProxy: true, + ConsulConfig: &consul.Config{HTTPPort: 8500}, + } + + pod := minimal() + pod.Annotations = c.annotations + + ns := testNS + ns.Labels = c.namespaceLabel + iptablesConfig, err := w.iptablesConfigJSON(*pod, ns) + require.NoError(t, err) + + actualConfig := iptables.Config{} + err = json.Unmarshal([]byte(iptablesConfig), &actualConfig) + require.NoError(t, err) + if c.expectConsulDNSConfig { + require.Equal(t, "127.0.0.1", actualConfig.ConsulDNSIP) + require.Equal(t, 8600, actualConfig.ConsulDNSPort) + } else { + require.Empty(t, actualConfig.ConsulDNSIP) + } + }) + } +} diff --git a/control-plane/subcommand/inject-connect/v2controllers.go b/control-plane/subcommand/inject-connect/v2controllers.go index 75c01ea943..32b97ac245 100644 --- a/control-plane/subcommand/inject-connect/v2controllers.go +++ b/control-plane/subcommand/inject-connect/v2controllers.go @@ -5,13 +5,17 @@ package connectinject import ( "context" + "github.com/hashicorp/consul-server-connection-manager/discovery" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" + ctrlRuntimeWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/endpointsv2" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/pod" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" + webhookV2 "github.com/hashicorp/consul-k8s/control-plane/connect-inject/webhook_v2" "github.com/hashicorp/consul-k8s/control-plane/subcommand/flags" ) @@ -24,13 +28,13 @@ func (c *Command) configureV2Controllers(ctx context.Context, mgr manager.Manage allowK8sNamespaces := flags.ToSet(c.flagAllowK8sNamespacesList) denyK8sNamespaces := flags.ToSet(c.flagDenyK8sNamespacesList) - //lifecycleConfig := lifecycle.Config{ - // DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, - // DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, - // DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, - // DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, - // DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, - //} + lifecycleConfig := lifecycle.Config{ + DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, + DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, + DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, + DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, + DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, + } metricsConfig := metrics.Config{ DefaultEnableMetrics: c.flagDefaultEnableMetrics, @@ -99,8 +103,58 @@ func (c *Command) configureV2Controllers(ctx context.Context, mgr manager.Manage //} // TODO: register webhooks - - // TODO: Update Webhook CA Bundle + mgr.GetWebhookServer().CertDir = c.flagCertDir + + mgr.GetWebhookServer().Register("/mutate", + &ctrlRuntimeWebhook.Admission{Handler: &webhookV2.MeshWebhook{ + Clientset: c.clientset, + ReleaseNamespace: c.flagReleaseNamespace, + ConsulConfig: consulConfig, + ConsulServerConnMgr: watcher, + ImageConsul: c.flagConsulImage, + ImageConsulDataplane: c.flagConsulDataplaneImage, + EnvoyExtraArgs: c.flagEnvoyExtraArgs, + ImageConsulK8S: c.flagConsulK8sImage, + RequireAnnotation: !c.flagDefaultInject, + AuthMethod: c.flagACLAuthMethod, + ConsulCACert: string(c.caCertPem), + TLSEnabled: c.consul.UseTLS, + ConsulAddress: c.consul.Addresses, + SkipServerWatch: c.consul.SkipServerWatch, + ConsulTLSServerName: c.consul.TLSServerName, + DefaultProxyCPURequest: c.sidecarProxyCPURequest, + DefaultProxyCPULimit: c.sidecarProxyCPULimit, + DefaultProxyMemoryRequest: c.sidecarProxyMemoryRequest, + DefaultProxyMemoryLimit: c.sidecarProxyMemoryLimit, + DefaultEnvoyProxyConcurrency: c.flagDefaultEnvoyProxyConcurrency, + LifecycleConfig: lifecycleConfig, + MetricsConfig: metricsConfig, + InitContainerResources: c.initContainerResources, + ConsulPartition: c.consul.Partition, + AllowK8sNamespacesSet: allowK8sNamespaces, + DenyK8sNamespacesSet: denyK8sNamespaces, + EnableNamespaces: c.flagEnableNamespaces, + ConsulDestinationNamespace: c.flagConsulDestinationNamespace, + EnableK8SNSMirroring: c.flagEnableK8SNSMirroring, + K8SNSMirroringPrefix: c.flagK8SNSMirroringPrefix, + CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy, + EnableTransparentProxy: c.flagDefaultEnableTransparentProxy, + EnableCNI: c.flagEnableCNI, + TProxyOverwriteProbes: c.flagTransparentProxyDefaultOverwriteProbes, + EnableConsulDNS: c.flagEnableConsulDNS, + EnableOpenShift: c.flagEnableOpenShift, + Log: ctrl.Log.WithName("handler").WithName("consul-mesh"), + LogLevel: c.flagLogLevel, + LogJSON: c.flagLogJSON, + }}) + + if c.flagEnableWebhookCAUpdate { + err := c.updateWebhookCABundle(ctx) + if err != nil { + setupLog.Error(err, "problem getting CA Cert") + return err + } + } return nil }