Acceptance tests

Get acceptance tests running
hashicorp · Oct 19, 2022 · ea62a71 · ea62a71
1 parent 87630a7
commit ea62a71
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 35 deletions.
diff --git a/acceptance/tests/fixtures/bases/static-metrics-app/deployment.yaml b/acceptance/tests/fixtures/bases/static-metrics-app/deployment.yaml
@@ -14,6 +14,8 @@ spec:
       annotations:
         'consul.hashicorp.com/connect-inject': 'true'
         'consul.hashicorp.com/connect-service': 'server'
+        'consul.hashicorp.com/service-metrics-path': '/metrics'
+        'consul.hashicorp.com/service-metrics-port': '9090'
       labels:
         app: static-metrics-app
     spec:
@@ -24,4 +26,4 @@ spec:
         - name: METRICS_ENABLE_PROMETHEUS
           value: "true"
         ports:
-        - containerPort: 9090
+        - containerPort: 9090
diff --git a/acceptance/tests/metrics/metrics_test.go b/acceptance/tests/metrics/metrics_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
@@ -21,16 +22,19 @@ const StaticClientName = "static-client"
 // Test that prometheus metrics, when enabled, are accessible from the
 // endpoints that have been exposed on the server, client and gateways.
 func TestComponentMetrics(t *testing.T) {
-	t.Skipf("Skipping this test because it's not yet supported with agentless")
 	env := suite.Environment()
 	cfg := suite.Config()
 	ctx := env.DefaultContext(t)
 	ns := ctx.KubectlOptions(t).Namespace
 
 	helmValues := map[string]string{
+		// Remove before merging
+		"global.imageConsulDataplane":       "curtbushko/consul-dataplane:latest",
+		"global.imageK8S":                   "curtbushko/consul-k8s-control-plane-dev:latest",
 		"global.datacenter":                 "dc1",
 		"global.metrics.enabled":            "true",
 		"global.metrics.enableAgentMetrics": "true",
+		"client.enabled":                    "true",
 
 		"connectInject.enabled": "true",
 		"controller.enabled":    "true",
@@ -77,28 +81,28 @@ func TestComponentMetrics(t *testing.T) {
 	require.NoError(t, err)
 	require.Contains(t, metricsOutput, `consul_acl_ResolveToken{quantile="0.5"}`)
 
-	//// Ingress Gateway Metrics
-	//assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway"`)
-	//
-	//// Terminating Gateway Metrics
-	//assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway"`)
-	//
-	//// Mesh Gateway Metrics
-	//assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway"`)
+	// Ingress Gateway Metrics
+	assertGatewayMetricsEnabled(t, ctx, ns, "ingress-gateway", `envoy_cluster_assignment_stale{local_cluster="ingress-gateway",consul_source_service="ingress-gateway"`)
+	// Terminating Gateway Metrics
+	assertGatewayMetricsEnabled(t, ctx, ns, "terminating-gateway", `envoy_cluster_assignment_stale{local_cluster="terminating-gateway",consul_source_service="terminating-gateway"`)
+	// Mesh Gateway Metrics
+	assertGatewayMetricsEnabled(t, ctx, ns, "mesh-gateway", `envoy_cluster_assignment_stale{local_cluster="mesh-gateway",consul_source_service="mesh-gateway"`)
 }
 
 // Test that merged service and envoy metrics are accessible from the
 // endpoints that have been exposed on the service.
 func TestAppMetrics(t *testing.T) {
-	t.Skipf("Skipping this test because it's not yet supported with agentless")
 	env := suite.Environment()
 	cfg := suite.Config()
 	ctx := env.DefaultContext(t)
 	ns := ctx.KubectlOptions(t).Namespace
 
 	helmValues := map[string]string{
-		"global.datacenter":      "dc1",
-		"global.metrics.enabled": "true",
+		// Remove before merging
+		"global.imageConsulDataplane": "curtbushko/consul-dataplane:latest",
+		"global.imageK8S":             "curtbushko/consul-k8s-control-plane-dev:latest",
+		"global.datacenter":           "dc1",
+		"global.metrics.enabled":      "true",
 
 		"connectInject.enabled":                      "true",
 		"connectInject.metrics.defaultEnableMerging": "true",
@@ -128,7 +132,7 @@ func TestAppMetrics(t *testing.T) {
 
 	// Retry because sometimes the merged metrics server takes a couple hundred milliseconds
 	// to start.
-	retry.RunWith(&retry.Counter{Count: 3, Wait: 1 * time.Second}, t, func(r *retry.R) {
+	retry.RunWith(&retry.Counter{Count: 10, Wait: 1 * time.Second}, t, func(r *retry.R) {
 		metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP))
 		require.NoError(r, err)
 		// This assertion represents the metrics from the envoy sidecar.
@@ -138,13 +142,13 @@ func TestAppMetrics(t *testing.T) {
 	})
 }
 
-//func assertGatewayMetricsEnabled(t *testing.T, ctx environment.TestContext, ns, label, metricsAssertion string) {
-//	pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("component=%s", label)})
-//	require.NoError(t, err)
-//	for _, pod := range pods.Items {
-//		podIP := pod.Status.PodIP
-//		metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP))
-//		require.NoError(t, err)
-//		require.Contains(t, metricsOutput, metricsAssertion)
-//	}
-//}
+func assertGatewayMetricsEnabled(t *testing.T, ctx environment.TestContext, ns, label, metricsAssertion string) {
+	pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("component=%s", label)})
+	require.NoError(t, err)
+	for _, pod := range pods.Items {
+		podIP := pod.Status.PodIP
+		metricsOutput, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "exec", "deploy/"+StaticClientName, "--", "curl", "--silent", "--show-error", fmt.Sprintf("http://%s:20200/metrics", podIP))
+		require.NoError(t, err)
+		require.Contains(t, metricsOutput, metricsAssertion)
+	}
+}
diff --git a/charts/consul/templates/ingress-gateways-deployment.yaml b/charts/consul/templates/ingress-gateways-deployment.yaml
@@ -290,7 +290,11 @@ spec:
             -service-partition={{ $root.Values.global.adminPartitions.name }} \
             {{- end }}
             -log-level={{ default $root.Values.global.logLevel }} \
-            -log-json={{ $root.Values.global.logJSON }}
+            -log-json={{ $root.Values.global.logJSON }} \
+            {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+            -telemetry-prom-scrape-path={{ $root.Values.connectInject.metrics.defaultPrometheusScrapePath }} \
+            -telemetry-prom-merge-port={{ $root.Values.connectInject.metrics.defaultMergedMetricsPort }}
+            {{- end }}
         livenessProbe:
           tcpSocket:
             port: 21000

diff --git a/charts/consul/templates/mesh-gateway-deployment.yaml b/charts/consul/templates/mesh-gateway-deployment.yaml
@@ -240,7 +240,11 @@ spec:
             -service-partition={{ .Values.global.adminPartitions.name }} \
             {{- end }}
             -log-level={{ default .Values.global.logLevel }} \
-            -log-json={{ .Values.global.logJSON }}
+            -log-json={{ .Values.global.logJSON }} \
+            {{- if (and .Values.global.metrics.enabled .Values.global.metrics.enableGatewayMetrics) }}
+            -telemetry-prom-scrape-path={{ .Values.connectInject.metrics.defaultPrometheusScrapePath }} \
+            -telemetry-prom-merge-port={{ .Values.connectInject.metrics.defaultMergedMetricsPort }}
+            {{- end }}
         livenessProbe:
           tcpSocket:
             port: {{ .Values.meshGateway.containerPort }}

diff --git a/charts/consul/templates/terminating-gateways-deployment.yaml b/charts/consul/templates/terminating-gateways-deployment.yaml
@@ -281,7 +281,11 @@ spec:
                 -service-partition={{ $root.Values.global.adminPartitions.name }} \
                 {{- end }}
                 -log-level={{ default $root.Values.global.logLevel }} \
-                -log-json={{ $root.Values.global.logJSON }}
+                -log-json={{ $root.Values.global.logJSON }} \
+                {{- if (and $root.Values.global.metrics.enabled $root.Values.global.metrics.enableGatewayMetrics) }}
+                -telemetry-prom-scrape-path={{ $root.Values.connectInject.metrics.defaultPrometheusScrapePath }} \
+                -telemetry-prom-merge-port={{ $root.Values.connectInject.metrics.defaultMergedMetricsPort }}
+                {{- end }}
           livenessProbe:
             tcpSocket:
               port: 8443

diff --git a/control-plane/connect-inject/consul_dataplane_sidecar.go b/control-plane/connect-inject/consul_dataplane_sidecar.go
@@ -202,20 +202,14 @@ func (w *MeshWebhook) getContainerSidecarCommand(namespace corev1.Namespace, mpi
 		}
 		cmd = append(cmd, "-telemetry-prom-scrape-path="+prometheusScrapePath,
 			"-telemetry-prom-merge-port="+mergedMetricsPort)
-		// Pull the TLS config from the relevant annotations.
-
-		// - telemetry-prom-ca-certs-path
-		// - telemetry-prom-key-file
-		// - telemetry-prom-cert-file
-		// - telemetry-prom-service-metrics-url
-		// - telemetry-prom-scrape-path
 
 		serviceMetricsPath := pod.Annotations[annotationServiceMetricsPath]
 		serviceMetricsPort := pod.Annotations[annotationServiceMetricsPort]
 		if serviceMetricsPath != "" && serviceMetricsPort != "" {
-			cmd = append(cmd, "telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath))
+			cmd = append(cmd, "-telemetry-prom-service-metrics-url="+fmt.Sprintf("http://127.0.0.1:%s%s", serviceMetricsPort, serviceMetricsPath))
 		}
 
+		// Pull the TLS config from the relevant annotations.
 		var prometheusCAFile string
 		if prometheusCAFile, ok := pod.Annotations[annotationPrometheusCAFile]; ok && prometheusCAFile != "" {
 			cmd = append(cmd, "-telemetry-prom-ca-certs-file="+prometheusCAFile)