Benchmark dashboard (#349)

* benchmark: generate new benchmark dashboard

* benchmark: fix missing metrics

* dashboard: update the grafana example to point to the jsonnet dash

* clean up

* benchmark: prepend metrics with _benchmark and use gauge for progress

* benchmark: continue to display progress pane after execution is finished

Co-authored-by: Dmitry Filimonov <dmitry@pyroscope.io>

benchmark/README.md

@@ -32,6 +32,6 @@ Edit `run-parameters.env` file to change the parameters of the benchmark run.
 
 ## Browsing results
 
-To view results open [http://localhost:8080/d/65gjqY3Mk/main?orgId=1](http://localhost:8080/d/65gjqY3Mk/main?orgId=1).
+To view results open [http://localhost:8080/d/tsWRL6ReZQkirFirmyvnWX1akHXJeHT8I8emjGJo/main?orgId=1](http://localhost:8080/d/tsWRL6ReZQkirFirmyvnWX1akHXJeHT8I8emjGJo/main?orgId=1).
 
 You will also be able to see screenshots of the runs in `./runs` directory

benchmark/docker-compose.yml

@@ -40,6 +40,7 @@ services:
     volumes:
       - ./grafana-provisioning:/etc/grafana/provisioning
       - ./grafana/grafana.ini:/etc/grafana/grafana.ini
+      - ../monitoring/gen/benchmark-dashboard.json:/etc/grafana/provisioning/dashboards/benchmark-dashboard.json
     ports:
       - 8080:3000
 

benchmark/grafana-provisioning/datasources/prometheus.yml

@@ -1,6 +1,6 @@
 ---
 datasources:
-  - name: Prometheus
+  - name: prometheus
     # <string, required> datasource type. Required
     type: prometheus
     # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required

benchmark/main.go

@@ -51,7 +51,7 @@ func waitUntilEndpointReady(url string) {
 	}
 }
 
-func startClientThread(appName string, wg *sync.WaitGroup, appFixtures []*transporttrie.Trie, runProgress prometheus.Gauge) {
+func startClientThread(appName string, wg *sync.WaitGroup, appFixtures []*transporttrie.Trie, runProgress prometheus.Gauge, successfulUploads prometheus.Counter, uploadErrors prometheus.Counter) {
 	rc := remote.RemoteConfig{
 		UpstreamThreads:        1,
 		UpstreamAddress:        "http://pyroscope:4040",
@@ -69,17 +69,6 @@ func startClientThread(appName string, wg *sync.WaitGroup, appFixtures []*transp
 
 	st := threadStartTime
 
-	reg := prometheus.NewRegistry()
-
-	uploadErrors := promauto.With(reg).NewCounter(prometheus.CounterOpts{
-		Name: "pyroscope_upload_errors",
-		Help: "",
-	})
-	successfulUploads := promauto.With(reg).NewCounter(prometheus.CounterOpts{
-		Name: "pyroscope_successful_uploads",
-		Help: "",
-	})
-
 	for i := 0; i < requestsCount; i++ {
 		t := appFixtures[i%len(appFixtures)]
 
@@ -205,16 +194,25 @@ func main() {
 
 	logrus.Info("waiting for other services to load")
 
-	benchmark := promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "pyroscope_benchmark",
-		Help: "",
-	})
 	runProgress := promauto.NewGauge(prometheus.GaugeOpts{
-		Name: "pyroscope_run_progress",
-		Help: "",
+		Namespace: "pyroscope",
+		Subsystem: "benchmark",
+		Name:      "progress",
+		Help:      "",
 	})
 
-	benchmark.Set(0)
+	uploadErrors := promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "pyroscope",
+		Subsystem: "benchmark",
+		Name:      "upload_errors",
+		Help:      "",
+	})
+	successfulUploads := promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: "pyroscope",
+		Subsystem: "benchmark",
+		Name:      "successful_uploads",
+		Help:      "",
+	})
 
 	waitUntilEndpointReady("pyroscope:4040")
 	waitUntilEndpointReady("prometheus:9090")
@@ -239,7 +237,6 @@ func main() {
 	logrus.Info("done generating fixtures")
 
 	logrus.Info("starting sending requests")
-	benchmark.Set(1)
 	startTime := time.Now()
 	reportSummaryMetric("start-time", startTime.Format(timeFmt))
 	wg := sync.WaitGroup{}
@@ -249,12 +246,11 @@ func main() {
 	for i := 0; i < appsCount; i++ {
 		r.Read(appNameBuf)
 		for j := 0; j < clientsCount; j++ {
-			go startClientThread(hex.EncodeToString(appNameBuf), &wg, fixtures[i], runProgress)
+			go startClientThread(hex.EncodeToString(appNameBuf), &wg, fixtures[i], runProgress, successfulUploads, uploadErrors)
 		}
 	}
 	wg.Wait()
 	logrus.Info("done sending requests")
-	benchmark.Set(0)
 	reportSummaryMetric("stop-time", time.Now().Format(timeFmt))
 	reportSummaryMetric("duration", time.Since(startTime).String())
 

benchmark/start.sh

@@ -8,6 +8,7 @@ echo "building containers..."
 
 source ./run-parameters.env
 export PYROSCOPE_CPUS PYROSCOPE_MEMORY
+export DOCKER_BUILDKIT=1
 
 docker-compose build
 

examples/grafana-integration/docker-compose.yml

@@ -10,6 +10,7 @@ services:
     image: grafana/grafana:8.1.1
     volumes:
       - ./grafana-provisioning:/etc/grafana/provisioning
+      - ../../monitoring/gen/dashboard.json:/etc/grafana/provisioning/dashboards/dashboard.json
       - ./grafana/grafana.ini:/etc/grafana/grafana.ini
       - ./grafana/home.json:/default-dashboard.json
     environment:

monitoring/Makefile

@@ -1,5 +1,10 @@
+all: dashboard benchmark-dashboard
+
 dashboard:
-	jsonnet -J vendor dashboard.jsonnet | tee dashboard.json
+	jsonnet -J vendor dashboard.jsonnet | tee gen/dashboard.json
+
+benchmark-dashboard:
+	jsonnet -J vendor benchmark.jsonnet | tee gen/benchmark-dashboard.json
 
 .PHONY: init
 init:

monitoring/README.md

@@ -7,9 +7,9 @@
 make init
 ```
 
-3. (Re)Generate the dashboard
+3. (Re)Generate the dashboards
 ```
-make dashboard
+make
 ```
 
 # Development
@@ -18,11 +18,12 @@ make dashboard
 Run the [grafana-integration](../examples/grafana-integration) example docker-compose then copy the generated dashboard there:
 
 ```
-make dashboard && \
-  cp dashboard.json ../examples/grafana-integration/grafana-provisioning/dashboards/ && \
-  docker-compose -f ../examples/grafana-integration/docker-compose.yml up -d --force-recreate grafana
+make && docker-compose -f ../examples/grafana-integration/docker-compose.yml up -d --force-recreate grafana
 ```
 
+# Warnings
+* If you ever rename the dashboard path, don't forget to update the references (see all the docker-compose.yaml)
+
 # References
 
 * https://grafana.github.io/grafonnet-lib/api-docs/

monitoring/benchmark.jsonnet

@@ -0,0 +1,8 @@
+local config = import 'config.libsonnet';
+local dashboard = import './lib/dashboard.libsonnet';
+
+(config + dashboard + {
+  _config+:: {
+    benchmark: true,
+  }
+}).dashboard

monitoring/config.libsonnet

@@ -0,0 +1,7 @@
+{
+  _config+:: {
+    selector: 'instance="$instance"',
+    // whether to add additional benchmark fields or not
+    benchmark: false,
+  }
+}