diff --git a/acceptance/tests/fixtures/bases/static-metrics-app/deployment.yaml b/acceptance/tests/fixtures/bases/static-metrics-app/deployment.yaml index 9283a8aae6..146730479e 100644 --- a/acceptance/tests/fixtures/bases/static-metrics-app/deployment.yaml +++ b/acceptance/tests/fixtures/bases/static-metrics-app/deployment.yaml @@ -14,6 +14,8 @@ spec: annotations: 'consul.hashicorp.com/connect-inject': 'true' 'consul.hashicorp.com/connect-service': 'server' + 'consul.hashicorp.com/service-metrics-path': '/metrics' + 'consul.hashicorp.com/service-metrics-port': '9090' labels: app: static-metrics-app spec: @@ -24,4 +26,4 @@ spec: - name: METRICS_ENABLE_PROMETHEUS value: "true" ports: - - containerPort: 9090 \ No newline at end of file + - containerPort: 9090 diff --git a/acceptance/tests/metrics/metrics_test.go b/acceptance/tests/metrics/metrics_test.go index b923658736..76db3f4594 100644 --- a/acceptance/tests/metrics/metrics_test.go +++ b/acceptance/tests/metrics/metrics_test.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/consul/sdk/testutil/retry" "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/environment" "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" "github.com/hashicorp/consul-k8s/acceptance/framework/k8s" "github.com/hashicorp/consul-k8s/acceptance/framework/logger" @@ -21,16 +22,19 @@ const StaticClientName = "static-client" // Test that prometheus metrics, when enabled, are accessible from the // endpoints that have been exposed on the server, client and gateways. func TestComponentMetrics(t *testing.T) { - t.Skipf("Skipping this test because it's not yet supported with agentless") env := suite.Environment() cfg := suite.Config() ctx := env.DefaultContext(t) ns := ctx.KubectlOptions(t).Namespace helmValues := map[string]string{ + // Remove before merging + "global.imageConsulDataplane": "curtbushko/consul-dataplane:latest", + "global.imageK8S": "curtbushko/consul-k8s-control-plane-dev:latest", "global.datacenter": "dc1", "global.metrics.enabled": "true", "global.metrics.enableAgentMetrics": "true", + "client.enabled": "true", "connectInject.enabled": "true", "controller.enabled": "true", @@ -77,28 +81,28 @@ func TestComponentMetrics(t *testing.T) { require.NoError(t, err) require.Contains(t, metricsOutput, `consul_acl_ResolveToken{quantile="0.5"}`) - //// Ingress Gateway Metrics - //assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway"`) - // - //// Terminating Gateway Metrics - //assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway"`) - // - //// Mesh Gateway Metrics - //assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway"`) + // Ingress Gateway Metrics + assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway"`) + // Terminating Gateway Metrics + assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway"`) + // Mesh Gateway Metrics + assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway"`) } // Test that merged service and envoy metrics are accessible from the // endpoints that have been exposed on the service. func TestAppMetrics(t *testing.T) { - t.Skipf("Skipping this test because it's not yet supported with agentless") env := suite.Environment() cfg := suite.Config() ctx := env.DefaultContext(t) ns := ctx.KubectlOptions(t).Namespace helmValues := map[string]string{ - "global.datacenter": "dc1", - "global.metrics.enabled": "true", + // Remove before merging + "global.imageConsulDataplane": "curtbushko/consul-dataplane:latest", + "global.imageK8S": "curtbushko/consul-k8s-control-plane-dev:latest", + "global.datacenter": "dc1", + "global.metrics.enabled": "true", "connectInject.enabled": "true", "connectInject.metrics.defaultEnableMerging": "true", @@ -128,7 +132,7 @@ func TestAppMetrics(t *testing.T) { // Retry because sometimes the merged metrics server takes a couple hundred milliseconds // to start. - retry.RunWith(&retry.Counter{Count: 3, Wait: 1 * time.Second}, t, func(r *retry.R) { + retry.RunWith(&retry.Counter{Count: 10, Wait: 1 * time.Second}, t, func(r *retry.R) { metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP)) require.NoError(r, err) // This assertion represents the metrics from the envoy sidecar. @@ -138,13 +142,13 @@ func TestAppMetrics(t *testing.T) { }) } -//func assertGatewayMetricsEnabled(t *testing.T, ctx environment.TestContext, ns, label, metricsAssertion string) { -// pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("component=%s", label)}) -// require.NoError(t, err) -// for _, pod := range pods.Items { -// podIP := pod.Status.PodIP -// metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP)) -// require.NoError(t, err) -// require.Contains(t, metricsOutput, metricsAssertion) -// } -//} +func assertGatewayMetricsEnabled(t *testing.T, ctx environment.TestContext, ns, label, metricsAssertion string) { + pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("component=%s", label)}) + require.NoError(t, err) + for _, pod := range pods.Items { + podIP := pod.Status.PodIP + metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP)) + require.NoError(t, err) + require.Contains(t, metricsOutput, metricsAssertion) + } +} diff --git a/charts/consul/templates/ingress-gateways-deployment.yaml b/charts/consul/templates/ingress-gateways-deployment.yaml index 2c8a5d65da..f37abf816b 100644 --- a/charts/consul/templates/ingress-gateways-deployment.yaml +++ b/charts/consul/templates/ingress-gateways-deployment.yaml @@ -290,7 +290,11 @@ spec: -service-partition={{ $root.Values.global.adminPartitions.name }} \ {{- end }} -log-level={{ default $root.Values.global.logLevel }} \ - -log-json={{ $root.Values.global.logJSON }} + -log-json={{ $root.Values.global.logJSON }} \ + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + -telemetry-prom-scrape-path={{ $root.Values.connectInject.metrics.defaultPrometheusScrapePath }} \ + -telemetry-prom-merge-port={{ $root.Values.connectInject.metrics.defaultMergedMetricsPort }} + {{- end }} livenessProbe: tcpSocket: port: 21000 diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml index daec987816..1ba39e2d17 100644 --- a/charts/consul/templates/mesh-gateway-deployment.yaml +++ b/charts/consul/templates/mesh-gateway-deployment.yaml @@ -240,7 +240,11 @@ spec: -service-partition={{ .Values.global.adminPartitions.name }} \ {{- end }} -log-level={{ default .Values.global.logLevel }} \ - -log-json={{ .Values.global.logJSON }} + -log-json={{ .Values.global.logJSON }} \ + {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }} + -telemetry-prom-scrape-path={{ .Values.connectInject.metrics.defaultPrometheusScrapePath }} \ + -telemetry-prom-merge-port={{ .Values.connectInject.metrics.defaultMergedMetricsPort }} + {{- end }} livenessProbe: tcpSocket: port: {{ .Values.meshGateway.containerPort }} diff --git a/charts/consul/templates/terminating-gateways-deployment.yaml b/charts/consul/templates/terminating-gateways-deployment.yaml index 1c36275375..c00fb49991 100644 --- a/charts/consul/templates/terminating-gateways-deployment.yaml +++ b/charts/consul/templates/terminating-gateways-deployment.yaml @@ -281,7 +281,11 @@ spec: -service-partition={{ $root.Values.global.adminPartitions.name }} \ {{- end }} -log-level={{ default $root.Values.global.logLevel }} \ - -log-json={{ $root.Values.global.logJSON }} + -log-json={{ $root.Values.global.logJSON }} \ + {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }} + -telemetry-prom-scrape-path={{ $root.Values.connectInject.metrics.defaultPrometheusScrapePath }} \ + -telemetry-prom-merge-port={{ $root.Values.connectInject.metrics.defaultMergedMetricsPort }} + {{- end }} livenessProbe: tcpSocket: port: 8443 diff --git a/control-plane/connect-inject/consul_dataplane_sidecar.go b/control-plane/connect-inject/consul_dataplane_sidecar.go index 61c713ae3b..abcd0a1d9f 100644 --- a/control-plane/connect-inject/consul_dataplane_sidecar.go +++ b/control-plane/connect-inject/consul_dataplane_sidecar.go @@ -202,20 +202,14 @@ func (w *MeshWebhook) getContainerSidecarCommand(namespace corev1.Namespace, mpi } cmd = append(cmd, "-telemetry-prom-scrape-path="+prometheusScrapePath, "-telemetry-prom-merge-port="+mergedMetricsPort) - // Pull the TLS config from the relevant annotations. - - // - telemetry-prom-ca-certs-path - // - telemetry-prom-key-file - // - telemetry-prom-cert-file - // - telemetry-prom-service-metrics-url - // - telemetry-prom-scrape-path serviceMetricsPath := pod.Annotations[annotationServiceMetricsPath] serviceMetricsPort := pod.Annotations[annotationServiceMetricsPort] if serviceMetricsPath != "" && serviceMetricsPort != "" { - cmd = append(cmd, "telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath)) + cmd = append(cmd, "-telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath)) } + // Pull the TLS config from the relevant annotations. var prometheusCAFile string if prometheusCAFile, ok := pod.Annotations[annotationPrometheusCAFile]; ok && prometheusCAFile != "" { cmd = append(cmd, "-telemetry-prom-ca-certs-file="+prometheusCAFile)