test: test Prometheus TLS

test/e2e/monitoring_stack_controller_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"os/exec"
 	"strings"
@@ -23,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/util/cert"
 	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -114,6 +116,9 @@ func TestMonitoringStackController(t *testing.T) {
 	}, {
 		name:     "managed fields in Prometheus object",
 		scenario: assertPrometheusManagedFields,
+	}, {
+		name:     "Prometheus stacks can scrape themselves behind TLS",
+		scenario: assertPrometheusScrapesItselfTLS,
 	}}
 	for _, tc := range ts {
 		t.Run(tc.name, tc.scenario)
@@ -655,6 +660,107 @@ func assertPrometheusManagedFields(t *testing.T) {
 	assert.DeepEqual(t, have, expected)
 }
 
+func assertPrometheusScrapesItselfTLS(t *testing.T) {
+	// TODO: Test Alertmanager TLS too once it's available
+	//       how to do this can be partialy seen at:
+	//       https://github.com/vyzigold/observability-operator/commit/adc714f4792654978f02899429e05c4e26a404ef
+
+	monitoringStackName := "self-scrape-tls"
+	prometheusServiceName := monitoringStackName + "-prometheus"
+
+	certs, key, err := cert.GenerateSelfSignedCertKey(prometheusServiceName, []net.IP{}, []string{})
+	assert.NilError(t, err)
+
+	promKey := string(key)
+	promCerts := strings.SplitAfter(string(certs), "-----END CERTIFICATE-----")
+
+	promTLSSecret := corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "prom-test-tls-secret",
+			Namespace: e2eTestNamespace,
+		},
+		StringData: map[string]string{
+			"tls.key": promKey,
+			"tls.crt": promCerts[0],
+			"ca.crt":  promCerts[1],
+		},
+	}
+
+	err = f.K8sClient.Create(context.Background(), &promTLSSecret)
+	assert.NilError(t, err)
+
+	ms := newMonitoringStack(t, monitoringStackName)
+	ms.Spec.PrometheusConfig = &stack.PrometheusConfig{
+		WebTLSConfig: &stack.WebTLSConfig{
+			Cert: stack.SecretKeySelector{
+				Name: "prom-test-tls-secret",
+				Key:  "tls.crt",
+			},
+			Key: stack.SecretKeySelector{
+				Name: "prom-test-tls-secret",
+				Key:  "tls.key",
+			},
+			CA: stack.SecretKeySelector{
+				Name: "prom-test-tls-secret",
+				Key:  "ca.crt",
+			},
+		},
+	}
+	err = f.K8sClient.Create(context.Background(), ms)
+	assert.NilError(t, err)
+	f.AssertStatefulsetReady("prometheus-self-scrape-tls", e2eTestNamespace, framework.WithTimeout(5*time.Minute))(t)
+
+	stopChan := make(chan struct{})
+	defer close(stopChan)
+	if err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		err = f.StartServicePortForward(prometheusServiceName, e2eTestNamespace, "9090", stopChan)
+		return err == nil, nil
+	}); err != nil {
+		t.Fatal(fmt.Errorf("Failed to poll for port-forward: %w", err))
+	}
+
+	promClient, err := framework.NewTLSPrometheusClient("https://localhost:9090", promCerts[1], prometheusServiceName)
+	expectedResults := map[string]int{
+		"prometheus_build_info":   2, // scrapes from both endpoints
+		"alertmanager_build_info": 2,
+	}
+	if err != nil {
+		t.Fatal(fmt.Errorf("Failed to create prometheus client: %s", err))
+	}
+	if err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+		correct := 0
+		for query, value := range expectedResults {
+			result, err := promClient.Query(query)
+			if err != nil {
+				return false, nil
+			}
+
+			if len(result.Data.Result) == 0 {
+				return false, nil
+			}
+
+			if len(result.Data.Result) > value {
+				resultErr := fmt.Errorf("invalid result for query %s, got %d, want %d", query, len(result.Data.Result), value)
+				return true, resultErr
+			}
+
+			if len(result.Data.Result) != value {
+				return false, nil
+			}
+
+			correct++
+		}
+
+		return correct == len(expectedResults), nil
+	}); err != nil {
+		t.Fatal(fmt.Errorf("Could not query prometheus: %w", err))
+	}
+}
+
 // Update this json when a new Prometheus field is set by MonitoringStack
 const oboManagedFieldsJson = `
 {
@@ -699,6 +805,7 @@ const oboManagedFieldsJson = `
   "f:scrapeConfigNamespaceSelector": {},
   "f:scrapeConfigSelector": {},
   "f:scrapeInterval": {},
+  "f:secrets": {},
   "f:securityContext": {
     "f:fsGroup": {},
     "f:runAsNonRoot": {},
@@ -722,6 +829,14 @@ const oboManagedFieldsJson = `
     "f:resources": {}
   },
   "f:tsdb": {}
+  "f:web": {
+    "f:tlsConfig": {
+      "f:cert": {
+        "f:secret": {}
+      },
+      "f:client_ca": {},
+      "f:keySecret": {}
+  }
 }
 `