diff --git a/.changelog/3222.txt b/.changelog/3222.txt new file mode 100644 index 0000000000..9347bd077a --- /dev/null +++ b/.changelog/3222.txt @@ -0,0 +1,3 @@ +```release-note:feature +control-plane: adds a named port, `prometheus`, to the `consul-dataplane` sidecar for use with [Prometheus operator](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmetricsendpoint). +``` diff --git a/control-plane/connect-inject/controllers/pod/pod_controller.go b/control-plane/connect-inject/controllers/pod/pod_controller.go index c4b867f8c3..4cf7580d5c 100644 --- a/control-plane/connect-inject/controllers/pod/pod_controller.go +++ b/control-plane/connect-inject/controllers/pod/pod_controller.go @@ -498,8 +498,8 @@ func (r *Controller) getBootstrapConfig(pod corev1.Pod) (*pbmesh.BootstrapConfig bootstrap := &pbmesh.BootstrapConfig{} // If metrics are enabled, the BootstrapConfig should set envoy_prometheus_bind_addr to a listener on 0.0.0.0 on - // the PrometheusScrapePort that points to a metrics backend. The backend for this listener will be determined by - // the envoy bootstrapping command (consul connect envoy) or the consul-dataplane GetBoostrapParams rpc. + // the PrometheusScrapePort. The backend for this listener will be determined by + // the consul-dataplane command line flags generated by the webhook. // If there is a merged metrics server, the backend would be that server. // If we are not running the merged metrics server, the backend should just be the Envoy metrics endpoint. enableMetrics, err := r.MetricsConfig.EnableMetrics(pod) diff --git a/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go b/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go index 05a67c3736..03f47a8e84 100644 --- a/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go +++ b/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go @@ -162,6 +162,15 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor container.VolumeMounts = append(container.VolumeMounts, volumeMounts...) } + // Container Ports + metricsPorts, err := w.getMetricsPorts(pod) + if err != nil { + return corev1.Container{}, err + } + if metricsPorts != nil { + container.Ports = append(container.Ports, metricsPorts...) + } + tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy) if err != nil { return corev1.Container{}, err @@ -500,3 +509,37 @@ func useProxyHealthCheck(pod corev1.Pod) bool { } return false } + +// getMetricsPorts creates container ports for exposing services such as prometheus. +// Prometheus in particular needs a named port for use with the operator. +// https://github.com/hashicorp/consul-k8s/pull/1440 +func (w *MeshWebhook) getMetricsPorts(pod corev1.Pod) ([]corev1.ContainerPort, error) { + enableMetrics, err := w.MetricsConfig.EnableMetrics(pod) + if err != nil { + return nil, fmt.Errorf("error determining if metrics are enabled: %w", err) + } + if !enableMetrics { + return nil, nil + } + + prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod) + if err != nil { + return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err) + } + if prometheusScrapePort == "" { + return nil, nil + } + + port, err := strconv.Atoi(prometheusScrapePort) + if err != nil { + return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err) + } + + return []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: int32(port), + Protocol: corev1.ProtocolTCP, + }, + }, nil +} diff --git a/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go b/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go index f89703d5ad..4261887b12 100644 --- a/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go +++ b/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go @@ -9,15 +9,17 @@ import ( "strings" "testing" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" - "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" - "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" + "github.com/hashicorp/consul-k8s/control-plane/consul" ) const nodeName = "test-node" @@ -1187,6 +1189,7 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { name string pod corev1.Pod expCmdArgs string + expPorts []corev1.ContainerPort expErr string }{ { @@ -1209,6 +1212,37 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { }, }, expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics", + expPorts: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 20200, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + { + name: "metrics with prometheus port override", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20123", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + constants.AnnotationPrometheusScrapePort: "6789", + }, + }, + }, + expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20123 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics", + expPorts: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 6789, + Protocol: corev1.ProtocolTCP, + }, + }, }, { name: "merged metrics with TLS enabled", @@ -1229,6 +1263,13 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { }, }, expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics -telemetry-prom-ca-certs-file=/certs/ca.crt -telemetry-prom-ca-certs-path=/certs/ca -telemetry-prom-cert-file=/certs/server.crt -telemetry-prom-key-file=/certs/key.pem", + expPorts: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 20200, + Protocol: corev1.ProtocolTCP, + }, + }, }, { name: "merge metrics with TLS enabled, missing CA gives an error", @@ -1293,6 +1334,12 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { t.Run(c.name, func(t *testing.T) { h := MeshWebhook{ ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + MetricsConfig: metrics.Config{ + // These are all the default values passed from the CLI + DefaultPrometheusScrapePort: "20200", + DefaultPrometheusScrapePath: "/metrics", + DefaultMergedMetricsPort: "20100", + }, } container, err := h.consulDataplaneSidecar(testNS, c.pod, multiPortInfo{}) if c.expErr != "" { @@ -1301,6 +1348,9 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { } else { require.NoError(t, err) require.Contains(t, strings.Join(container.Args, " "), c.expCmdArgs) + if c.expPorts != nil { + require.ElementsMatch(t, container.Ports, c.expPorts) + } } }) } diff --git a/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar.go b/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar.go index f8a8406d23..434899d67e 100644 --- a/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar.go +++ b/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar.go @@ -159,6 +159,15 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor container.VolumeMounts = append(container.VolumeMounts, volumeMounts...) } + // Container Ports + metricsPorts, err := w.getMetricsPorts(pod) + if err != nil { + return corev1.Container{}, err + } + if metricsPorts != nil { + container.Ports = append(container.Ports, metricsPorts...) + } + tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy) if err != nil { return corev1.Container{}, err @@ -302,15 +311,22 @@ func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, bearer return nil, fmt.Errorf("unable to determine if merged metrics is enabled: %w", err) } if metricsServer { - - // (TODO) Figure out what port will be used for merged metrics and setup merged metrics - mergedMetricsPort, err := w.MetricsConfig.MergedMetricsPort(pod) if err != nil { return nil, fmt.Errorf("unable to determine if merged metrics port: %w", err) } args = append(args, "-telemetry-prom-merge-port="+mergedMetricsPort) + serviceMetricsPath := w.MetricsConfig.ServiceMetricsPath(pod) + serviceMetricsPort, err := w.MetricsConfig.ServiceMetricsPort(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if service metrics port: %w", err) + } + + if serviceMetricsPath != "" && serviceMetricsPort != "" { + args = append(args, "-telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath)) + } + // Pull the TLS config from the relevant annotations. var prometheusCAFile string if raw, ok := pod.Annotations[constants.AnnotationPrometheusCAFile]; ok && raw != "" { @@ -470,3 +486,37 @@ func useProxyHealthCheck(pod corev1.Pod) bool { } return false } + +// getMetricsPorts creates container ports for exposing services such as prometheus. +// Prometheus in particular needs a named port for use with the operator. +// https://github.com/hashicorp/consul-k8s/pull/1440 +func (w *MeshWebhook) getMetricsPorts(pod corev1.Pod) ([]corev1.ContainerPort, error) { + enableMetrics, err := w.MetricsConfig.EnableMetrics(pod) + if err != nil { + return nil, fmt.Errorf("error determining if metrics are enabled: %w", err) + } + if !enableMetrics { + return nil, nil + } + + prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod) + if err != nil { + return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err) + } + if prometheusScrapePort == "" { + return nil, nil + } + + port, err := strconv.Atoi(prometheusScrapePort) + if err != nil { + return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err) + } + + return []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: int32(port), + Protocol: corev1.ProtocolTCP, + }, + }, nil +} diff --git a/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar_test.go b/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar_test.go index 81b3520511..994bf4e446 100644 --- a/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar_test.go +++ b/control-plane/connect-inject/webhookv2/consul_dataplane_sidecar_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/consul" ) @@ -945,6 +946,178 @@ func TestHandlerConsulDataplaneSidecar_Resources(t *testing.T) { } } +func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { + cases := []struct { + name string + pod corev1.Pod + expCmdArgs string + expPorts []corev1.ContainerPort + expErr string + }{ + { + name: "default", + pod: corev1.Pod{}, + expCmdArgs: "", + }, + { + name: "turning on merged metrics", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20100", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + }, + }, + }, + expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics", + expPorts: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 20200, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + { + name: "metrics with prometheus port override", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20123", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + constants.AnnotationPrometheusScrapePort: "6789", + }, + }, + }, + expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20123 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics", + expPorts: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 6789, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + { + name: "merged metrics with TLS enabled", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20100", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + constants.AnnotationPrometheusCAFile: "/certs/ca.crt", + constants.AnnotationPrometheusCAPath: "/certs/ca", + constants.AnnotationPrometheusCertFile: "/certs/server.crt", + constants.AnnotationPrometheusKeyFile: "/certs/key.pem", + }, + }, + }, + expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics -telemetry-prom-ca-certs-file=/certs/ca.crt -telemetry-prom-ca-certs-path=/certs/ca -telemetry-prom-cert-file=/certs/server.crt -telemetry-prom-key-file=/certs/key.pem", + expPorts: []corev1.ContainerPort{ + { + Name: "prometheus", + ContainerPort: 20200, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + { + name: "merge metrics with TLS enabled, missing CA gives an error", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20100", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + constants.AnnotationPrometheusCertFile: "/certs/server.crt", + constants.AnnotationPrometheusKeyFile: "/certs/key.pem", + }, + }, + }, + expCmdArgs: "", + expErr: fmt.Sprintf("must set one of %q or %q when providing prometheus TLS config", constants.AnnotationPrometheusCAFile, constants.AnnotationPrometheusCAPath), + }, + { + name: "merge metrics with TLS enabled, missing cert gives an error", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20100", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + constants.AnnotationPrometheusCAFile: "/certs/ca.crt", + constants.AnnotationPrometheusKeyFile: "/certs/key.pem", + }, + }, + }, + expCmdArgs: "", + expErr: fmt.Sprintf("must set %q when providing prometheus TLS config", constants.AnnotationPrometheusCertFile), + }, + { + name: "merge metrics with TLS enabled, missing key file gives an error", + pod: corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + constants.AnnotationService: "web", + constants.AnnotationEnableMetrics: "true", + constants.AnnotationEnableMetricsMerging: "true", + constants.AnnotationMergedMetricsPort: "20100", + constants.AnnotationPort: "1234", + constants.AnnotationPrometheusScrapePath: "/scrape-path", + constants.AnnotationPrometheusCAPath: "/certs/ca", + constants.AnnotationPrometheusCertFile: "/certs/server.crt", + }, + }, + }, + expCmdArgs: "", + expErr: fmt.Sprintf("must set %q when providing prometheus TLS config", constants.AnnotationPrometheusKeyFile), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + h := MeshWebhook{ + ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502}, + MetricsConfig: metrics.Config{ + // These are all the default values passed from the CLI + DefaultPrometheusScrapePort: "20200", + DefaultPrometheusScrapePath: "/metrics", + DefaultMergedMetricsPort: "20100", + }, + } + container, err := h.consulDataplaneSidecar(testNS, c.pod) + if c.expErr != "" { + require.NotNil(t, err) + require.Contains(t, err.Error(), c.expErr) + } else { + require.NoError(t, err) + require.Contains(t, strings.Join(container.Args, " "), c.expCmdArgs) + if c.expPorts != nil { + require.ElementsMatch(t, container.Ports, c.expPorts) + } + } + }) + } +} + func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) { gracefulShutdownSeconds := 10 gracefulPort := "20307"