feat: add named prom port to dataplane sidecar

.changelog/3222.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+control-plane: adds a named port, `prometheus`, to the `consul-dataplane` sidecar for use with [Prometheus operator](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmetricsendpoint).
+```

control-plane/connect-inject/controllers/pod/pod_controller.go

@@ -498,8 +498,8 @@ func (r *Controller) getBootstrapConfig(pod corev1.Pod) (*pbmesh.BootstrapConfig
 	bootstrap := &pbmesh.BootstrapConfig{}
 
 	// If metrics are enabled, the BootstrapConfig should set envoy_prometheus_bind_addr to a listener on 0.0.0.0 on
-	// the PrometheusScrapePort that points to a metrics backend. The backend for this listener will be determined by
-	// the envoy bootstrapping command (consul connect envoy) or the consul-dataplane GetBoostrapParams rpc.
+	// the PrometheusScrapePort. The backend for this listener will be determined by
+	// the consul-dataplane command line flags generated by the webhook.
 	// If there is a merged metrics server, the backend would be that server.
 	// If we are not running the merged metrics server, the backend should just be the Envoy metrics endpoint.
 	enableMetrics, err := r.MetricsConfig.EnableMetrics(pod)

control-plane/connect-inject/webhook/consul_dataplane_sidecar.go

@@ -162,6 +162,15 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		container.VolumeMounts = append(container.VolumeMounts, volumeMounts...)
 	}
 
+	// Container Ports
+	metricsPorts, err := w.getMetricsPorts(pod)
+	if err != nil {
+		return corev1.Container{}, err
+	}
+	if metricsPorts != nil {
+		container.Ports = append(container.Ports, metricsPorts...)
+	}
+
 	tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy)
 	if err != nil {
 		return corev1.Container{}, err
@@ -500,3 +509,37 @@ func useProxyHealthCheck(pod corev1.Pod) bool {
 	}
 	return false
 }
+
+// getMetricsPorts creates container ports for exposing services such as prometheus.
+// Prometheus in particular needs a named port for use with the operator.
+// https://github.com/hashicorp/consul-k8s/pull/1440
+func (w *MeshWebhook) getMetricsPorts(pod corev1.Pod) ([]corev1.ContainerPort, error) {
+	enableMetrics, err := w.MetricsConfig.EnableMetrics(pod)
+	if err != nil {
+		return nil, fmt.Errorf("error determining if metrics are enabled: %w", err)
+	}
+	if !enableMetrics {
+		return nil, nil
+	}
+
+	prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
+	}
+	if prometheusScrapePort == "" {
+		return nil, nil
+	}
+
+	port, err := strconv.Atoi(prometheusScrapePort)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
+	}
+
+	return []corev1.ContainerPort{
+		{
+			Name:          "prometheus",
+			ContainerPort: int32(port),
+			Protocol:      corev1.ProtocolTCP,
+		},
+	}, nil
+}

control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go

@@ -9,15 +9,17 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
-	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle"
-	"github.com/hashicorp/consul-k8s/control-plane/consul"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/pointer"
+
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle"
+	"github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics"
+	"github.com/hashicorp/consul-k8s/control-plane/consul"
 )
 
 const nodeName = "test-node"
@@ -1187,6 +1189,7 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
 		name       string
 		pod        corev1.Pod
 		expCmdArgs string
+		expPorts   []corev1.ContainerPort
 		expErr     string
 	}{
 		{
@@ -1209,6 +1212,37 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
 				},
 			},
 			expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics",
+			expPorts: []corev1.ContainerPort{
+				{
+					Name:          "prometheus",
+					ContainerPort: 20200,
+					Protocol:      corev1.ProtocolTCP,
+				},
+			},
+		},
+		{
+			name: "metrics with prometheus port override",
+			pod: corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						constants.AnnotationService:              "web",
+						constants.AnnotationEnableMetrics:        "true",
+						constants.AnnotationEnableMetricsMerging: "true",
+						constants.AnnotationMergedMetricsPort:    "20123",
+						constants.AnnotationPort:                 "1234",
+						constants.AnnotationPrometheusScrapePath: "/scrape-path",
+						constants.AnnotationPrometheusScrapePort: "6789",
+					},
+				},
+			},
+			expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20123 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics",
+			expPorts: []corev1.ContainerPort{
+				{
+					Name:          "prometheus",
+					ContainerPort: 6789,
+					Protocol:      corev1.ProtocolTCP,
+				},
+			},
 		},
 		{
 			name: "merged metrics with TLS enabled",
@@ -1229,6 +1263,13 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
 				},
 			},
 			expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics -telemetry-prom-ca-certs-file=/certs/ca.crt -telemetry-prom-ca-certs-path=/certs/ca -telemetry-prom-cert-file=/certs/server.crt -telemetry-prom-key-file=/certs/key.pem",
+			expPorts: []corev1.ContainerPort{
+				{
+					Name:          "prometheus",
+					ContainerPort: 20200,
+					Protocol:      corev1.ProtocolTCP,
+				},
+			},
 		},
 		{
 			name: "merge metrics with TLS enabled, missing CA gives an error",
@@ -1293,6 +1334,12 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
 		t.Run(c.name, func(t *testing.T) {
 			h := MeshWebhook{
 				ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502},
+				MetricsConfig: metrics.Config{
+					// These are all the default values passed from the CLI
+					DefaultPrometheusScrapePort: "20200",
+					DefaultPrometheusScrapePath: "/metrics",
+					DefaultMergedMetricsPort:    "20100",
+				},
 			}
 			container, err := h.consulDataplaneSidecar(testNS, c.pod, multiPortInfo{})
 			if c.expErr != "" {
@@ -1301,6 +1348,9 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
 			} else {
 				require.NoError(t, err)
 				require.Contains(t, strings.Join(container.Args, " "), c.expCmdArgs)
+				if c.expPorts != nil {
+					require.ElementsMatch(t, container.Ports, c.expPorts)
+				}
 			}
 		})
 	}

control-plane/connect-inject/webhookv2/consul_dataplane_sidecar.go

@@ -159,6 +159,15 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
 		container.VolumeMounts = append(container.VolumeMounts, volumeMounts...)
 	}
 
+	// Container Ports
+	metricsPorts, err := w.getMetricsPorts(pod)
+	if err != nil {
+		return corev1.Container{}, err
+	}
+	if metricsPorts != nil {
+		container.Ports = append(container.Ports, metricsPorts...)
+	}
+
 	tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy)
 	if err != nil {
 		return corev1.Container{}, err
@@ -302,15 +311,22 @@ func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, bearer
 		return nil, fmt.Errorf("unable to determine if merged metrics is enabled: %w", err)
 	}
 	if metricsServer {
-
-		// (TODO) Figure out what port will be used for merged metrics and setup merged metrics
-
 		mergedMetricsPort, err := w.MetricsConfig.MergedMetricsPort(pod)
 		if err != nil {
 			return nil, fmt.Errorf("unable to determine if merged metrics port: %w", err)
 		}
 		args = append(args, "-telemetry-prom-merge-port="+mergedMetricsPort)
 
+		serviceMetricsPath := w.MetricsConfig.ServiceMetricsPath(pod)
+		serviceMetricsPort, err := w.MetricsConfig.ServiceMetricsPort(pod)
+		if err != nil {
+			return nil, fmt.Errorf("unable to determine if service metrics port: %w", err)
+		}
+
+		if serviceMetricsPath != "" && serviceMetricsPort != "" {
+			args = append(args, "-telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath))
+		}
+
 		// Pull the TLS config from the relevant annotations.
 		var prometheusCAFile string
 		if raw, ok := pod.Annotations[constants.AnnotationPrometheusCAFile]; ok && raw != "" {
@@ -470,3 +486,37 @@ func useProxyHealthCheck(pod corev1.Pod) bool {
 	}
 	return false
 }
+
+// getMetricsPorts creates container ports for exposing services such as prometheus.
+// Prometheus in particular needs a named port for use with the operator.
+// https://github.com/hashicorp/consul-k8s/pull/1440
+func (w *MeshWebhook) getMetricsPorts(pod corev1.Pod) ([]corev1.ContainerPort, error) {
+	enableMetrics, err := w.MetricsConfig.EnableMetrics(pod)
+	if err != nil {
+		return nil, fmt.Errorf("error determining if metrics are enabled: %w", err)
+	}
+	if !enableMetrics {
+		return nil, nil
+	}
+
+	prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
+	}
+	if prometheusScrapePort == "" {
+		return nil, nil
+	}
+
+	port, err := strconv.Atoi(prometheusScrapePort)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
+	}
+
+	return []corev1.ContainerPort{
+		{
+			Name:          "prometheus",
+			ContainerPort: int32(port),
+			Protocol:      corev1.ProtocolTCP,
+		},
+	}, nil
+}