Skip to content

Commit

Permalink
feat: add named prom port to dataplane sidecar
Browse files Browse the repository at this point in the history 
Co-authored-by: Hamish <hamish.forbes@gmail.com>
  • Loading branch information
@DanStough @hamishforbes
DanStough and hamishforbes committed Nov 16, 2023
1 parent b18b849 commit b08f8a5
Show file tree
Hide file tree
Showing 5 changed files with 324 additions and 8 deletions.
4 changes: 2 additions & 2 deletions control-plane/connect-inject/controllers/pod/pod_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,8 +498,8 @@ func (r *Controller) getBootstrapConfig(pod corev1.Pod) (*pbmesh.BootstrapConfig
bootstrap := &pbmesh.BootstrapConfig{}

// If metrics are enabled, the BootstrapConfig should set envoy_prometheus_bind_addr to a listener on 0.0.0.0 on
// the PrometheusScrapePort that points to a metrics backend. The backend for this listener will be determined by
// the envoy bootstrapping command (consul connect envoy) or the consul-dataplane GetBoostrapParams rpc.
// the PrometheusScrapePort. The backend for this listener will be determined by
// the consul-dataplane command line flags generated by the webhook.
// If there is a merged metrics server, the backend would be that server.
// If we are not running the merged metrics server, the backend should just be the Envoy metrics endpoint.
enableMetrics, err := r.MetricsConfig.EnableMetrics(pod)
Expand Down
43 changes: 43 additions & 0 deletions control-plane/connect-inject/webhook/consul_dataplane_sidecar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,15 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
container.VolumeMounts = append(container.VolumeMounts, volumeMounts...)
}

// Container Ports
ports, err := w.getContainerPorts(pod)
if err != nil {
return corev1.Container{}, err
}
if ports != nil {
container.Ports = ports
}

tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy)
if err != nil {
return corev1.Container{}, err
Expand Down Expand Up @@ -500,3 +509,37 @@ func useProxyHealthCheck(pod corev1.Pod) bool {
}
return false
}

// getContainerPorts creates container ports for exposing services such as prometheus.
// Prometheus in particular needs a named port for use with the operator.
// https://github.com/hashicorp/consul-k8s/pull/1440
func (w *MeshWebhook) getContainerPorts(pod corev1.Pod) ([]corev1.ContainerPort, error) {
enableMetrics, err := w.MetricsConfig.EnableMetrics(pod)
if err != nil {
return nil, fmt.Errorf("error determining if metrics are enabled: %w", err)
}
if !enableMetrics {
return nil, nil
}

prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod)
if err != nil {
return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
}
if prometheusScrapePort == "" {
return nil, nil
}

port, err := strconv.Atoi(prometheusScrapePort)
if err != nil {
return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
}

return []corev1.ContainerPort{
{
Name: "prometheus",
ContainerPort: int32(port),
Protocol: corev1.ProtocolTCP,
},
}, nil
}
56 changes: 53 additions & 3 deletions control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,17 @@ import (
"strings"
"testing"

"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
"github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle"
"github.com/hashicorp/consul-k8s/control-plane/consul"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"

"github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants"
"github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle"
"github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics"
"github.com/hashicorp/consul-k8s/control-plane/consul"
)

const nodeName = "test-node"
Expand Down Expand Up @@ -1187,6 +1189,7 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
name string
pod corev1.Pod
expCmdArgs string
expPorts []corev1.ContainerPort
expErr string
}{
{
Expand All @@ -1209,6 +1212,37 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
},
},
expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics",
expPorts: []corev1.ContainerPort{
{
Name: "prometheus",
ContainerPort: 20200,
Protocol: corev1.ProtocolTCP,
},
},
},
{
name: "metrics with prometheus port override",
pod: corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
constants.AnnotationService: "web",
constants.AnnotationEnableMetrics: "true",
constants.AnnotationEnableMetricsMerging: "true",
constants.AnnotationMergedMetricsPort: "20123",
constants.AnnotationPort: "1234",
constants.AnnotationPrometheusScrapePath: "/scrape-path",
constants.AnnotationPrometheusScrapePort: "6789",
},
},
},
expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20123 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics",
expPorts: []corev1.ContainerPort{
{
Name: "prometheus",
ContainerPort: 6789,
Protocol: corev1.ProtocolTCP,
},
},
},
{
name: "merged metrics with TLS enabled",
Expand All @@ -1229,6 +1263,13 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
},
},
expCmdArgs: "-telemetry-prom-scrape-path=/scrape-path -telemetry-prom-merge-port=20100 -telemetry-prom-service-metrics-url=http://127.0.0.1:1234/metrics -telemetry-prom-ca-certs-file=/certs/ca.crt -telemetry-prom-ca-certs-path=/certs/ca -telemetry-prom-cert-file=/certs/server.crt -telemetry-prom-key-file=/certs/key.pem",
expPorts: []corev1.ContainerPort{
{
Name: "prometheus",
ContainerPort: 20200,
Protocol: corev1.ProtocolTCP,
},
},
},
{
name: "merge metrics with TLS enabled, missing CA gives an error",
Expand Down Expand Up @@ -1293,6 +1334,12 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
t.Run(c.name, func(t *testing.T) {
h := MeshWebhook{
ConsulConfig: &consul.Config{HTTPPort: 8500, GRPCPort: 8502},
MetricsConfig: metrics.Config{
// These are all the default values passed from the CLI
DefaultPrometheusScrapePort: "20200",
DefaultPrometheusScrapePath: "/metrics",
DefaultMergedMetricsPort: "20100",
},
}
container, err := h.consulDataplaneSidecar(testNS, c.pod, multiPortInfo{})
if c.expErr != "" {
Expand All @@ -1301,6 +1348,9 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) {
} else {
require.NoError(t, err)
require.Contains(t, strings.Join(container.Args, " "), c.expCmdArgs)
if c.expPorts != nil {
require.ElementsMatch(t, container.Ports, c.expPorts)
}
}
})
}
Expand Down
56 changes: 53 additions & 3 deletions control-plane/connect-inject/webhookv2/consul_dataplane_sidecar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,15 @@ func (w *MeshWebhook) consulDataplaneSidecar(namespace corev1.Namespace, pod cor
container.VolumeMounts = append(container.VolumeMounts, volumeMounts...)
}

// Container Ports
ports, err := w.getContainerPorts(pod)
if err != nil {
return corev1.Container{}, err
}
if ports != nil {
container.Ports = ports
}

tproxyEnabled, err := common.TransparentProxyEnabled(namespace, pod, w.EnableTransparentProxy)
if err != nil {
return corev1.Container{}, err
Expand Down Expand Up @@ -302,15 +311,22 @@ func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, bearer
return nil, fmt.Errorf("unable to determine if merged metrics is enabled: %w", err)
}
if metricsServer {

// (TODO) Figure out what port will be used for merged metrics and setup merged metrics

mergedMetricsPort, err := w.MetricsConfig.MergedMetricsPort(pod)
if err != nil {
return nil, fmt.Errorf("unable to determine if merged metrics port: %w", err)
}
args = append(args, "-telemetry-prom-merge-port="+mergedMetricsPort)

serviceMetricsPath := w.MetricsConfig.ServiceMetricsPath(pod)
serviceMetricsPort, err := w.MetricsConfig.ServiceMetricsPort(pod)
if err != nil {
return nil, fmt.Errorf("unable to determine if service metrics port: %w", err)
}

if serviceMetricsPath != "" && serviceMetricsPort != "" {
args = append(args, "-telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath))
}

// Pull the TLS config from the relevant annotations.
var prometheusCAFile string
if raw, ok := pod.Annotations[constants.AnnotationPrometheusCAFile]; ok && raw != "" {
Expand Down Expand Up @@ -470,3 +486,37 @@ func useProxyHealthCheck(pod corev1.Pod) bool {
}
return false
}

// getContainerPorts creates container ports for exposing services such as prometheus.
// Prometheus in particular needs a named port for use with the operator.
// https://github.com/hashicorp/consul-k8s/pull/1440
func (w *MeshWebhook) getContainerPorts(pod corev1.Pod) ([]corev1.ContainerPort, error) {
enableMetrics, err := w.MetricsConfig.EnableMetrics(pod)
if err != nil {
return nil, fmt.Errorf("error determining if metrics are enabled: %w", err)
}
if !enableMetrics {
return nil, nil
}

prometheusScrapePort, err := w.MetricsConfig.PrometheusScrapePort(pod)
if err != nil {
return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
}
if prometheusScrapePort == "" {
return nil, nil
}

port, err := strconv.Atoi(prometheusScrapePort)
if err != nil {
return nil, fmt.Errorf("error parsing prometheus port from pod: %w", err)
}

return []corev1.ContainerPort{
{
Name: "prometheus",
ContainerPort: int32(port),
Protocol: corev1.ProtocolTCP,
},
}, nil
}
Loading

0 comments on commit b08f8a5

Please sign in to comment.