🌱 test/e2e,cmd/test: scrape metrics for test servers and e2e tests

.github/workflows/ci.yaml

@@ -68,14 +68,20 @@ jobs:
 
       - run: |-
           LOG_DIR=/tmp/e2e/shared-server/artifacts ARTIFACT_DIR=/tmp/e2e COUNT=2 SUITES=transparent-multi-cluster:requires-kind \
-          make test-e2e-shared
+          ./hack/run-with-prometheus.sh make test-e2e-shared
 
       - uses: cytopia/upload-artifact-retry-action@v0.1.7
         if: ${{ always() }}
         with:
           name: e2e-shared-server
           path: /tmp/e2e/**/artifacts/
 
+      - uses: cytopia/upload-artifact-retry-action@v0.1.7
+        if: ${{ always() }}
+        with:
+          name: e2e-shared-server-metrics
+          path: /tmp/e2e/**/metrics/
+
   e2e-sharded:
     name: e2e-sharded
     runs-on: ubuntu-latest
@@ -109,10 +115,16 @@ jobs:
 
     - run: |-
         LOG_DIR=/tmp/e2e/sharded/artifacts ARTIFACT_DIR=/tmp/e2e COUNT=2 SUITES=transparent-multi-cluster:requires-kind \
-        make test-e2e-sharded
+        ./hack/run-with-prometheus.sh make test-e2e-sharded
 
     - uses: cytopia/upload-artifact-retry-action@v0.1.7
       if: ${{ always() }}
       with:
         name: e2e-sharded
         path: /tmp/e2e/**/artifacts/
+
+    - uses: cytopia/upload-artifact-retry-action@v0.1.7
+      if: ${{ always() }}
+      with:
+        name: e2e-sharded-metrics
+        path: /tmp/e2e/**/metrics/

.gitignore

@@ -23,3 +23,7 @@ tmp
 
 kcp-docs-image
 docs/site
+
+.prometheus_data
+.prometheus-config.yaml
+.prometheus-pid

cmd/sharded-test-server/frontproxy.go

@@ -32,6 +32,7 @@ import (
 	"github.com/fatih/color"
 
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/klog/v2"
@@ -102,7 +103,10 @@ func startFrontProxy(
 	}
 
 	// write root shard kubeconfig
-	configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp-0/admin.kubeconfig")}, nil)
+	configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp-0/admin.kubeconfig")},
+		&clientcmd.ConfigOverrides{CurrentContext: "system:admin"},
+	)
 	raw, err := configLoader.RawConfig()
 	if err != nil {
 		return err
@@ -232,8 +236,21 @@ func startFrontProxy(
 		writer.StopOut()
 	}
 	fmt.Fprintf(successOut, "kcp-front-proxy is ready\n")
+	cfg, err := configLoader.ClientConfig()
+	if err != nil {
+		return err
+	}
+	return scrapeMetrics(ctx, cfg, workDirPath)
+}
 
-	return nil
+func scrapeMetrics(ctx context.Context, cfg *rest.Config, workDir string) error {
+	promUrl, set := os.LookupEnv("PROMETHEUS_URL")
+	if !set || promUrl == "" {
+		return nil
+	}
+	return framework.ScrapeMetrics(ctx, cfg, promUrl, workDir, "kcp-front-proxy", filepath.Join(workDir, ".kcp-front-proxy/apiserver.crt"), map[string]string{
+		"server": "kcp-front-proxy",
+	})
 }
 
 func writeAdminKubeConfig(hostIP string, workDirPath string) error {

cmd/sharded-test-server/main.go

@@ -234,8 +234,12 @@ func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numb
 
 	// Wait for shards to be ready
 	shardsErrCh := make(chan indexErrTuple)
-	for i, shard := range shards {
-		terminatedCh, err := shard.WaitForReady(ctx)
+	for i, s := range shards {
+		terminatedCh, err := s.WaitForReady(ctx)
+		if err != nil {
+			return err
+		}
+		err = shard.ScrapeMetrics(ctx, s, workDirPath)
 		if err != nil {
 			return err
 		}

cmd/test-server/kcp/shard.go

@@ -271,6 +271,31 @@ func (s *Shard) GatherMetrics(ctx context.Context) {
 	logger.Info("wrote metrics file", "path", metricsFile)
 }
 
+func ScrapeMetrics(ctx context.Context, s *Shard, workDir string) error {
+	logger := klog.FromContext(ctx).WithValues("shard", s.name)
+	promUrl, set := os.LookupEnv("PROMETHEUS_URL")
+	if !set || promUrl == "" {
+		logger.Info("PROMETHEUS_URL environment variable unset, skipping Prometheus scrape config generation")
+		return nil
+	}
+
+	logger.Info("scraping shard metrics using Prometheus", "prometheus_url", promUrl, "shard", s.name)
+	kubeconfigPath := filepath.Join(s.runtimeDir, "admin.kubeconfig")
+	configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath},
+		&clientcmd.ConfigOverrides{CurrentContext: "system:admin"},
+	)
+	config, err := configLoader.ClientConfig()
+	if err != nil {
+		logger.Error(err, "unable to collect metrics: error getting client config")
+		return err
+	}
+
+	return framework.ScrapeMetrics(ctx, config, promUrl, workDir, s.name, filepath.Join(s.runtimeDir, "apiserver.crt"), map[string]string{
+		"server": s.name,
+	})
+}
+
 // there doesn't seem to be any simple way to get a metav1.Status from the Go client, so we get
 // the content in a string-formatted error, unfortunately.
 func unreadyComponentsFromError(err error) sets.String {

cmd/test-server/main.go

@@ -99,7 +99,7 @@ func start(shardFlags []string, quiet bool) error {
 	}
 
 	logFilePath := flag.Lookup("log-file-path").Value.String()
-	shard := shard.NewShard(
+	s := shard.NewShard(
 		"kcp",
 		".kcp",
 		logFilePath,
@@ -108,11 +108,16 @@ func start(shardFlags []string, quiet bool) error {
 			"--client-ca-file", filepath.Join(".kcp", "client-ca.crt"),
 		),
 	)
-	if err := shard.Start(ctx, quiet); err != nil {
+	if err := s.Start(ctx, quiet); err != nil {
 		return err
 	}
 
-	errCh, err := shard.WaitForReady(ctx)
+	errCh, err := s.WaitForReady(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = shard.ScrapeMetrics(ctx, s, ".")
 	if err != nil {
 		return err
 	}
@@ -134,7 +139,7 @@ func start(shardFlags []string, quiet bool) error {
 	metricsCtx, metricsCancel := context.WithTimeout(ctx, wait.ForeverTestTimeout)
 	defer metricsCancel()
 
-	shard.GatherMetrics(metricsCtx)
+	s.GatherMetrics(metricsCtx)
 
 	return nil
 }

go.mod

@@ -36,6 +36,7 @@ require (
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
 	google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd
 	gopkg.in/square/go-jose.v2 v2.2.2
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.24.3
 	k8s.io/apiextensions-apiserver v0.24.3
 	k8s.io/apimachinery v0.24.3
@@ -170,7 +171,6 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/cloud-provider v0.0.0 // indirect
 	k8s.io/component-helpers v0.0.0 // indirect
 	k8s.io/controller-manager v0.0.0 // indirect

hack/run-with-prometheus.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The KCP Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+PROMETHEUS_VER=2.42.0
+ARCH=$(go env GOARCH)
+OS=$(go env GOOS)
+if ! command -v hack/tools/prometheus 1> /dev/null 2>&1; then
+  echo "Downloading Prometheus v${PROMETHEUS_VER}"
+  mkdir -p hack/tools
+  curl -L https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VER}/prometheus-${PROMETHEUS_VER}.${OS}-${ARCH}.tar.gz | tar -xz --strip-components 1 -C hack/tools prometheus-${PROMETHEUS_VER}.${OS}-${ARCH}/prometheus
+fi
+
+touch .prometheus-config.yaml
+./hack/tools/prometheus --storage.tsdb.path=".prometheus_data" --config.file=.prometheus-config.yaml --web.enable-lifecycle &
+PROM_PID=$!
+export PROMETHEUS_URL="http://localhost:9090"
+echo 'Waiting for Prometheus to be ready...'
+while ! curl -s http://localhost:9090/-/ready; do
+  sleep 1
+done
+echo 'Prometheus is ready!'
+
+set -o xtrace
+set +o errexit
+"${@}"
+EXIT_CODE=${?}
+set +o xtrace
+set -o errexit
+echo "Command terminated with ${EXIT_CODE}"
+
+if [ -n "${ARTIFACT_DIR:=}" ]; then
+  kill -TERM ${PROM_PID}
+  echo 'Waiting for Prometheus to shut down...'
+  while [ -f .prometheus_data/lock ]; do sleep 1; done
+  echo 'Prometheus shut down successfully!'
+  mkdir -p ${ARTIFACT_DIR}/metrics
+  tar cvzf ${ARTIFACT_DIR}/metrics/prometheus.tar -C .prometheus_data .
+fi
+
+exit "${EXIT_CODE}"