From 87047e0abb1638b2911f17775c2db35f08e4695a Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Thu, 13 Jun 2024 12:19:56 +0200 Subject: [PATCH 1/6] Add metric to track discovered services --- pkg/internal/discover/attacher.go | 2 ++ pkg/internal/imetrics/imetrics.go | 6 ++++++ pkg/internal/imetrics/iprom.go | 16 +++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/internal/discover/attacher.go b/pkg/internal/discover/attacher.go index 8ed29959d..6c95ebd51 100644 --- a/pkg/internal/discover/attacher.go +++ b/pkg/internal/discover/attacher.go @@ -95,6 +95,7 @@ func (ta *TraceAttacher) getTracer(ie *Instrumentable) (*ebpf.ProcessTracer, boo ta.log.Debug(".done") return nil, false } + ta.Metrics.DiscoverService(ie.FileInfo.ExecutableName()) ta.log.Info("instrumenting process", "cmd", ie.FileInfo.CmdExePath, "pid", ie.FileInfo.Pid) // builds a tracer for that executable @@ -205,6 +206,7 @@ func (ta *TraceAttacher) notifyProcessDeletion(ie *Instrumentable) { // notifying the tracer to block any trace from that PID // to avoid that a new process reusing this PID could send traces // unless explicitly allowed + ta.Metrics.UndiscoverService(ie.FileInfo.ExecutableName()) tracer.BlockPID(uint32(ie.FileInfo.Pid), ie.FileInfo.Ns) // if there are no more trace instances for a Go program, we need to notify that diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index eb4e5a49a..313ec7ce4 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -28,6 +28,10 @@ type Reporter interface { OTELTraceExportError(err error) // PrometheusRequest is invoked every time the Prometheus exporter is invoked, for a given port and path PrometheusRequest(port, path string) + // DiscoverService is invoked every time a new service is discovered by the eBPF tracer + DiscoverService(service string) + // UndiscoverService is invoked every time a new service is removed from the discovered services by the eBPF tracer + UndiscoverService(service string) } // NoopReporter is a metrics Reporter that just does nothing @@ -40,3 +44,5 @@ func (n NoopReporter) OTELMetricExportError(_ error) {} func (n NoopReporter) OTELTraceExport(_ int) {} func (n NoopReporter) OTELTraceExportError(_ error) {} func (n NoopReporter) PrometheusRequest(_, _ string) {} +func (n NoopReporter) DiscoverService(_ string) {} +func (n NoopReporter) UndiscoverService(_ string) {} diff --git a/pkg/internal/imetrics/iprom.go b/pkg/internal/imetrics/iprom.go index ed4f8d54f..c25a33045 100644 --- a/pkg/internal/imetrics/iprom.go +++ b/pkg/internal/imetrics/iprom.go @@ -28,6 +28,7 @@ type PrometheusReporter struct { otelTraceExports prometheus.Counter otelTraceExportErrs *prometheus.CounterVec prometheusRequests *prometheus.CounterVec + discoveredServices *prometheus.GaugeVec } func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusManager) *PrometheusReporter { @@ -61,6 +62,10 @@ func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusM Name: "prometheus_http_requests", Help: "requests towards the Prometheus Scrape endpoint", }, []string{"port", "path"}), + discoveredServices: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "beyla_discovered_services", + Help: "discovered services by the eBPF tracer", + }, []string{"service"}), } manager.Register(cfg.Port, cfg.Path, pr.tracerFlushes, @@ -68,7 +73,8 @@ func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusM pr.otelMetricExportErrs, pr.otelTraceExports, pr.otelTraceExportErrs, - pr.prometheusRequests) + pr.prometheusRequests, + pr.discoveredServices) return pr } @@ -100,3 +106,11 @@ func (p *PrometheusReporter) OTELTraceExportError(err error) { func (p *PrometheusReporter) PrometheusRequest(port, path string) { p.prometheusRequests.WithLabelValues(port, path).Inc() } + +func (p *PrometheusReporter) DiscoverService(service string) { + p.discoveredServices.WithLabelValues(service).Set(1) +} + +func (p *PrometheusReporter) UndiscoverService(service string) { + p.discoveredServices.WithLabelValues(service).Set(0) +} From f7cd272ae26ab1098c95ba219b4a3a85974bfcd6 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Thu, 13 Jun 2024 12:22:15 +0200 Subject: [PATCH 2/6] update docs --- docs/sources/metrics.md | 1 + pkg/internal/imetrics/iprom.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sources/metrics.md b/docs/sources/metrics.md index 5f0d15aa1..9e0b876e2 100644 --- a/docs/sources/metrics.md +++ b/docs/sources/metrics.md @@ -109,3 +109,4 @@ Beyla can be [configured to report internal metrics]({{< relref "./configure/opt | `otel_trace_exports` | Counter | Length of the trace batches submitted to the remote OTEL collector | | `otel_trace_export_errors` | CounterVec | Error count on each failed OTEL trace export, by error type | | `prometheus_http_requests` | CounterVec | Number of requests towards the Prometheus Scrape endpoint, faceted by HTTP port and path | +| `beyla_discovered_services` | GaugeVec | Discovered services by Beyla, with process name | diff --git a/pkg/internal/imetrics/iprom.go b/pkg/internal/imetrics/iprom.go index c25a33045..93ec2eb20 100644 --- a/pkg/internal/imetrics/iprom.go +++ b/pkg/internal/imetrics/iprom.go @@ -64,7 +64,7 @@ func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusM }, []string{"port", "path"}), discoveredServices: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "beyla_discovered_services", - Help: "discovered services by the eBPF tracer", + Help: "discovered services by Beyla", }, []string{"service"}), } manager.Register(cfg.Port, cfg.Path, From 85592a447ff939074f5d7cba9ba6cf45625cb242 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Fri, 14 Jun 2024 14:50:08 +0200 Subject: [PATCH 3/6] improve logic of register instrumented processes --- docs/sources/metrics.md | 2 +- pkg/internal/discover/attacher.go | 5 +++-- pkg/internal/imetrics/imetrics.go | 12 +++++------ pkg/internal/imetrics/iprom.go | 34 +++++++++++++++---------------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/docs/sources/metrics.md b/docs/sources/metrics.md index 9e0b876e2..87f89e4d0 100644 --- a/docs/sources/metrics.md +++ b/docs/sources/metrics.md @@ -109,4 +109,4 @@ Beyla can be [configured to report internal metrics]({{< relref "./configure/opt | `otel_trace_exports` | Counter | Length of the trace batches submitted to the remote OTEL collector | | `otel_trace_export_errors` | CounterVec | Error count on each failed OTEL trace export, by error type | | `prometheus_http_requests` | CounterVec | Number of requests towards the Prometheus Scrape endpoint, faceted by HTTP port and path | -| `beyla_discovered_services` | GaugeVec | Discovered services by Beyla, with process name | +| `beyla_instrumented_processes` | GaugeVec | Instrumented processes by Beyla, with process name | diff --git a/pkg/internal/discover/attacher.go b/pkg/internal/discover/attacher.go index 6c95ebd51..a5256c849 100644 --- a/pkg/internal/discover/attacher.go +++ b/pkg/internal/discover/attacher.go @@ -89,14 +89,15 @@ func (ta *TraceAttacher) getTracer(ie *Instrumentable) (*ebpf.ProcessTracer, boo ie.FileInfo.Service.SDKLanguage = ie.Type // allowing the tracer to forward traces from the new PID and its children processes monitorPIDs(tracer, ie) + ta.Metrics.InstrumentProcess(ie.FileInfo.ExecutableName()) if tracer.Type == ebpf.Generic { monitorPIDs(ta.reusableTracer, ie) } ta.log.Debug(".done") return nil, false } - ta.Metrics.DiscoverService(ie.FileInfo.ExecutableName()) ta.log.Info("instrumenting process", "cmd", ie.FileInfo.CmdExePath, "pid", ie.FileInfo.Pid) + ta.Metrics.InstrumentProcess(ie.FileInfo.ExecutableName()) // builds a tracer for that executable var programs []ebpf.Tracer @@ -206,7 +207,7 @@ func (ta *TraceAttacher) notifyProcessDeletion(ie *Instrumentable) { // notifying the tracer to block any trace from that PID // to avoid that a new process reusing this PID could send traces // unless explicitly allowed - ta.Metrics.UndiscoverService(ie.FileInfo.ExecutableName()) + ta.Metrics.UninstrumentProcess(ie.FileInfo.ExecutableName()) tracer.BlockPID(uint32(ie.FileInfo.Pid), ie.FileInfo.Ns) // if there are no more trace instances for a Go program, we need to notify that diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index 313ec7ce4..9b2b1dab6 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -28,10 +28,10 @@ type Reporter interface { OTELTraceExportError(err error) // PrometheusRequest is invoked every time the Prometheus exporter is invoked, for a given port and path PrometheusRequest(port, path string) - // DiscoverService is invoked every time a new service is discovered by the eBPF tracer - DiscoverService(service string) - // UndiscoverService is invoked every time a new service is removed from the discovered services by the eBPF tracer - UndiscoverService(service string) + // InstrumentProcess is invoked every time a new process is instrumented + InstrumentProcess(process_name string) + // UninstrumentProcess is invoked every time a process is removed from the instrumented processed + UninstrumentProcess(process_name string) } // NoopReporter is a metrics Reporter that just does nothing @@ -44,5 +44,5 @@ func (n NoopReporter) OTELMetricExportError(_ error) {} func (n NoopReporter) OTELTraceExport(_ int) {} func (n NoopReporter) OTELTraceExportError(_ error) {} func (n NoopReporter) PrometheusRequest(_, _ string) {} -func (n NoopReporter) DiscoverService(_ string) {} -func (n NoopReporter) UndiscoverService(_ string) {} +func (n NoopReporter) InstrumentProcess(_ string) {} +func (n NoopReporter) UninstrumentProcess(_ string) {} diff --git a/pkg/internal/imetrics/iprom.go b/pkg/internal/imetrics/iprom.go index 93ec2eb20..977baf043 100644 --- a/pkg/internal/imetrics/iprom.go +++ b/pkg/internal/imetrics/iprom.go @@ -21,14 +21,14 @@ type PrometheusConfig struct { // PrometheusReporter is an internal metrics Reporter that exports to Prometheus type PrometheusReporter struct { - connector *connector.PrometheusManager - tracerFlushes prometheus.Histogram - otelMetricExports prometheus.Counter - otelMetricExportErrs *prometheus.CounterVec - otelTraceExports prometheus.Counter - otelTraceExportErrs *prometheus.CounterVec - prometheusRequests *prometheus.CounterVec - discoveredServices *prometheus.GaugeVec + connector *connector.PrometheusManager + tracerFlushes prometheus.Histogram + otelMetricExports prometheus.Counter + otelMetricExportErrs *prometheus.CounterVec + otelTraceExports prometheus.Counter + otelTraceExportErrs *prometheus.CounterVec + prometheusRequests *prometheus.CounterVec + instrumentedProcesses *prometheus.GaugeVec } func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusManager) *PrometheusReporter { @@ -62,10 +62,10 @@ func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusM Name: "prometheus_http_requests", Help: "requests towards the Prometheus Scrape endpoint", }, []string{"port", "path"}), - discoveredServices: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "beyla_discovered_services", - Help: "discovered services by Beyla", - }, []string{"service"}), + instrumentedProcesses: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "beyla_instrumented_processes", + Help: "instrumented processes by Beyla", + }, []string{"process_name"}), } manager.Register(cfg.Port, cfg.Path, pr.tracerFlushes, @@ -74,7 +74,7 @@ func NewPrometheusReporter(cfg *PrometheusConfig, manager *connector.PrometheusM pr.otelTraceExports, pr.otelTraceExportErrs, pr.prometheusRequests, - pr.discoveredServices) + pr.instrumentedProcesses) return pr } @@ -107,10 +107,10 @@ func (p *PrometheusReporter) PrometheusRequest(port, path string) { p.prometheusRequests.WithLabelValues(port, path).Inc() } -func (p *PrometheusReporter) DiscoverService(service string) { - p.discoveredServices.WithLabelValues(service).Set(1) +func (p *PrometheusReporter) InstrumentProcess(process_name string) { + p.instrumentedProcesses.WithLabelValues(process_name).Inc() } -func (p *PrometheusReporter) UndiscoverService(service string) { - p.discoveredServices.WithLabelValues(service).Set(0) +func (p *PrometheusReporter) UninstrumentProcess(process_name string) { + p.instrumentedProcesses.WithLabelValues(process_name).Dec() } From d0c54a9618c64555b7b7e35e9207cdef63e09632 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Fri, 14 Jun 2024 15:03:52 +0200 Subject: [PATCH 4/6] make linter happy --- pkg/internal/imetrics/imetrics.go | 4 ++-- pkg/internal/imetrics/iprom.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/internal/imetrics/imetrics.go b/pkg/internal/imetrics/imetrics.go index 9b2b1dab6..2917683ca 100644 --- a/pkg/internal/imetrics/imetrics.go +++ b/pkg/internal/imetrics/imetrics.go @@ -29,9 +29,9 @@ type Reporter interface { // PrometheusRequest is invoked every time the Prometheus exporter is invoked, for a given port and path PrometheusRequest(port, path string) // InstrumentProcess is invoked every time a new process is instrumented - InstrumentProcess(process_name string) + InstrumentProcess(processName string) // UninstrumentProcess is invoked every time a process is removed from the instrumented processed - UninstrumentProcess(process_name string) + UninstrumentProcess(processName string) } // NoopReporter is a metrics Reporter that just does nothing diff --git a/pkg/internal/imetrics/iprom.go b/pkg/internal/imetrics/iprom.go index 977baf043..0ab056bec 100644 --- a/pkg/internal/imetrics/iprom.go +++ b/pkg/internal/imetrics/iprom.go @@ -107,10 +107,10 @@ func (p *PrometheusReporter) PrometheusRequest(port, path string) { p.prometheusRequests.WithLabelValues(port, path).Inc() } -func (p *PrometheusReporter) InstrumentProcess(process_name string) { - p.instrumentedProcesses.WithLabelValues(process_name).Inc() +func (p *PrometheusReporter) InstrumentProcess(processName string) { + p.instrumentedProcesses.WithLabelValues(processName).Inc() } -func (p *PrometheusReporter) UninstrumentProcess(process_name string) { - p.instrumentedProcesses.WithLabelValues(process_name).Dec() +func (p *PrometheusReporter) UninstrumentProcess(processName string) { + p.instrumentedProcesses.WithLabelValues(processName).Dec() } From e7678e129136a988b8818926e6129203cd520c2b Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 17 Jun 2024 16:00:31 +0200 Subject: [PATCH 5/6] Add integration test --- Makefile | 2 +- .../integration/configs/prometheus-config.yml | 5 +++ test/integration/docker-compose-multiexec.yml | 5 +++ test/integration/multiprocess_test.go | 33 +++++++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d08c7b5c6..e1bb31499 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ cleanup-integration-test: run-integration-test: @echo "### Running integration tests" go clean -testcache - go test -p 1 -failfast -v -timeout 60m -mod vendor -a ./test/integration/... --tags=integration + go test -p 1 -failfast -v -timeout 60m -mod vendor -a ./test/integration/... --tags=integration -run ^TestMultiProcess$ .PHONY: run-integration-test-k8s run-integration-test-k8s: diff --git a/test/integration/configs/prometheus-config.yml b/test/integration/configs/prometheus-config.yml index cc4215ccc..b7fef72ef 100644 --- a/test/integration/configs/prometheus-config.yml +++ b/test/integration/configs/prometheus-config.yml @@ -11,3 +11,8 @@ scrape_configs: static_configs: - targets: - 'otelcol:8888' + - job_name: autoinstrumenter-collector + honor_labels: true + static_configs: + - targets: + - '172.17.0.1:8999' diff --git a/test/integration/docker-compose-multiexec.yml b/test/integration/docker-compose-multiexec.yml index f876ebdf5..b7f7dc6c2 100644 --- a/test/integration/docker-compose-multiexec.yml +++ b/test/integration/docker-compose-multiexec.yml @@ -134,6 +134,10 @@ services: BEYLA_METRICS_REPORT_TARGET: "true" BEYLA_METRICS_REPORT_PEER: "true" BEYLA_HOSTNAME: "beyla" + BEYLA_INTERNAL_METRICS_PROMETHEUS_PORT: 8999 + BEYLA_INTERNAL_METRICS_PROMETHEUS_PATH: /metrics + ports: + - "8999:8999" # OpenTelemetry Collector otelcol: @@ -166,6 +170,7 @@ services: - --storage.tsdb.path=/prometheus - --web.enable-lifecycle - --web.route-prefix=/ + - --log.level=debug volumes: - ./configs/:/etc/prometheus ports: diff --git a/test/integration/multiprocess_test.go b/test/integration/multiprocess_test.go index 61e512210..44f695ef5 100644 --- a/test/integration/multiprocess_test.go +++ b/test/integration/multiprocess_test.go @@ -3,8 +3,10 @@ package integration import ( + "fmt" "net/http" "path" + "strconv" "testing" "time" @@ -96,6 +98,10 @@ func TestMultiProcess(t *testing.T) { }) } + t.Run("Instrumented processes metric", func(t *testing.T) { + checkInstrumentedProcessesMetric(t) + }) + t.Run("BPF pinning folders mounted", func(t *testing.T) { // 1 beyla pinned map folder for all processes testBPFPinningMounted(t) @@ -130,3 +136,30 @@ func checkReportedOnlyOnce(t *testing.T, baseURL, serviceName string) { }, test.Interval(1000*time.Millisecond)) } + +func checkInstrumentedProcessesMetric(t *testing.T) { + pq := prom.Client{HostPort: prometheusHostPort} + test.Eventually(t, testTimeout, func(t require.TestingT) { + // we expected to have this in Prometheus at this point + processes := map[string]int{ + "python3.11": 10, + "greetings": 2, + "java": 1, + "node": 2, + "ruby": 2, + "duped_service": 1, + "testserver": 2, + "rename1": 1, + } + + for processName, expectedCount := range processes { + results, err := pq.Query(fmt.Sprintf(`beyla_instrumented_processes{process_name="%s"}`, processName)) + require.NoError(t, err) + value, err := strconv.Atoi(results[0].Value[1].(string)) + require.NoError(t, err) + assert.Equal(t, expectedCount, value) + } + + }, test.Interval(1000*time.Millisecond)) + +} From a16601440db61f6df87572a274b2c94164ba06c4 Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Mon, 17 Jun 2024 16:00:56 +0200 Subject: [PATCH 6/6] rever change in Make --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e1bb31499..d08c7b5c6 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ cleanup-integration-test: run-integration-test: @echo "### Running integration tests" go clean -testcache - go test -p 1 -failfast -v -timeout 60m -mod vendor -a ./test/integration/... --tags=integration -run ^TestMultiProcess$ + go test -p 1 -failfast -v -timeout 60m -mod vendor -a ./test/integration/... --tags=integration .PHONY: run-integration-test-k8s run-integration-test-k8s: