From aced4f760e0839ffbd88c205be541898107ac230 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Wed, 4 Sep 2024 15:50:03 +0200 Subject: [PATCH] fix instance and job in prometheus, and document honor_labels --- docs/sources/configure/options.md | 12 +++++++++--- pkg/export/attributes/attr_defs.go | 6 ++++-- pkg/export/attributes/names/attrs.go | 14 +++++++------- pkg/export/prom/prom.go | 12 ++---------- pkg/internal/infraolly/process/status.go | 4 +++- pkg/internal/request/span_getters.go | 4 +++- pkg/internal/svc/svc.go | 4 ++++ 7 files changed, 32 insertions(+), 24 deletions(-) diff --git a/docs/sources/configure/options.md b/docs/sources/configure/options.md index c8d4c4e74..78a52d34a 100644 --- a/docs/sources/configure/options.md +++ b/docs/sources/configure/options.md @@ -300,7 +300,7 @@ precedence: - If Kubernetes is enabled: 1. The name of the Deployment that runs the instrumented process, if any. - 2. The name of the ReplicaSet that runs the instrumented process, if any. + 2. The name of the ReplicaSet/DaemonSet/StatefulSet that runs the instrumented process, if any. 3. The name of the Pod that runs the instrumented process. - If kubernetes is not enabled: 1. The name of the process executable file. @@ -308,8 +308,8 @@ precedence: If multiple processes match the service selection criteria described below, the metrics and traces for all the instances might share the same service name; for example, when multiple instrumented processes run under the same Deployment, -or have the same executable name. In that case, the reported `instance.id` (OTEL) or -`target_instance` (Prometheus) would allow differentiating the different instances +or have the same executable name. In that case, the reported `instance` attribute +would allow differentiating the different instances of the service. | YAML | Environment variable | Type | Default | @@ -1215,6 +1215,12 @@ API key of your Grafana Cloud account. ## Prometheus HTTP endpoint +> ℹ️ The Prometheus scraper might override the values of the `instance` and `job` labels. +> To keep the original values as set by Beyla, make sure to configure the +> Prometheus scraper to set the `honor_labels` option to `true`. +> +> ([How to set `honor_labels` in Grafana Alloy](/docs/alloy/latest/reference/components/prometheus/prometheus.scrape/)). + YAML section `prometheus_export`. This component opens an HTTP endpoint in the auto-instrumentation tool diff --git a/pkg/export/attributes/attr_defs.go b/pkg/export/attributes/attr_defs.go index 5c6f31f74..b93f3b96e 100644 --- a/pkg/export/attributes/attr_defs.go +++ b/pkg/export/attributes/attr_defs.go @@ -43,7 +43,8 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup { var prometheusAttributes = AttrReportGroup{ Disabled: !promEnabled, Attributes: map[attr.Name]Default{ - attr.TargetInstance: true, + attr.Instance: true, + attr.Job: true, attr.ServiceNamespace: true, }, } @@ -166,11 +167,12 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup { var promProcessAttributes = AttrReportGroup{ Disabled: !promEnabled, Attributes: map[attr.Name]Default{ + attr.Instance: true, + attr.Job: true, attr.ProcCommand: true, attr.ProcOwner: true, attr.ProcParentPid: true, attr.ProcPid: true, - attr.TargetInstance: true, attr.ProcCommandLine: false, attr.ProcCommandArgs: false, attr.ProcExecName: false, diff --git a/pkg/export/attributes/names/attrs.go b/pkg/export/attributes/names/attrs.go index 4351fc899..55eac8bc3 100644 --- a/pkg/export/attributes/names/attrs.go +++ b/pkg/export/attributes/names/attrs.go @@ -124,13 +124,13 @@ const ( // other beyla-specific attributes const ( - // TargetInstance is a Prometheus-only attribute. - // It will expose the process hostname-pid (or K8s Pod). - // It is advised for users that to use relabeling rules to - // override the "instance" attribute with "target" in the - // Prometheus server. This would be similar to the "multi target pattern": - // https://prometheus.io/docs/guides/multi-target-exporter/ - TargetInstance = Name("target.instance") + // Instance and Job are only explicitly used in the Prometheus + // exporter, as the OpenTelemetry SDK already sets them implicitly. + // It is advised for users to configure their Prometheus scraper with + // the `honor_labels` option set to true, to avoid overwriting the + // instance attribute with the target attribute. + Instance = Name("instance") + Job = Name("job") // ServiceName and ServiceNamespace are going to be used only on Prometheus // as metric attributes. The OTEL exporter already uses them as Resource diff --git a/pkg/export/prom/prom.go b/pkg/export/prom/prom.go index c0f103ccb..4644ba51d 100644 --- a/pkg/export/prom/prom.go +++ b/pkg/export/prom/prom.go @@ -701,10 +701,6 @@ func labelNamesSpans() []string { } func (r *metricsReporter) labelValuesSpans(span *request.Span) []string { - job := span.ServiceID.Name - if span.ServiceID.Namespace != "" { - job = span.ServiceID.Namespace + "/" + job - } return []string{ span.ServiceID.Name, span.ServiceID.Namespace, @@ -712,7 +708,7 @@ func (r *metricsReporter) labelValuesSpans(span *request.Span) []string { strconv.Itoa(int(request.SpanStatusCode(span))), span.ServiceGraphKind(), string(span.ServiceID.UID), // app instance ID - job, + span.ServiceID.Job(), "beyla", } } @@ -728,17 +724,13 @@ func labelNamesTargetInfo(kubeEnabled bool) []string { } func (r *metricsReporter) labelValuesTargetInfo(service svc.ID) []string { - job := service.Name - if service.Namespace != "" { - job = service.Namespace + "/" + job - } values := []string{ r.hostID, service.HostName, service.Name, service.Namespace, string(service.UID), // app instance ID - job, + service.Job(), service.SDKLanguage.String(), "beyla", "beyla", diff --git a/pkg/internal/infraolly/process/status.go b/pkg/internal/infraolly/process/status.go index 76773fc9d..e50135d20 100644 --- a/pkg/internal/infraolly/process/status.go +++ b/pkg/internal/infraolly/process/status.go @@ -119,8 +119,10 @@ func PromGetters(name attr.Name) (attributes.Getter[*Status, string], bool) { case attr.ProcCPUMode, attr.ProcDiskIODir, attr.ProcNetIODir: // the attributes are handled explicitly by the prometheus exporter, but we need to // ignore them to avoid that the default case tries to report them from service metadata - case attr.TargetInstance: + case attr.Instance: g = func(s *Status) string { return string(s.ID.UID) } + case attr.Job: + g = func(s *Status) string { return s.ID.Service.Job() } default: g = func(s *Status) string { return s.ID.Service.Metadata[name] } } diff --git a/pkg/internal/request/span_getters.go b/pkg/internal/request/span_getters.go index dc5a8f02d..eaaa92d65 100644 --- a/pkg/internal/request/span_getters.go +++ b/pkg/internal/request/span_getters.go @@ -180,8 +180,10 @@ func SpanPromGetters(attrName attr.Name) (attributes.Getter[*Span, string], bool getter = func(s *Span) string { return string(s.ServiceID.UID) } // resource metadata values below. Unlike OTEL, they are included here because they // belong to the metric, instead of the Resource - case attr.TargetInstance: + case attr.Instance: getter = func(s *Span) string { return string(s.ServiceID.UID) } + case attr.Job: + getter = func(s *Span) string { return s.ServiceID.Job() } case attr.ServiceName: getter = func(s *Span) string { return s.ServiceID.Name } case attr.ServiceNamespace: diff --git a/pkg/internal/svc/svc.go b/pkg/internal/svc/svc.go index 1713e43a9..b9cd55c33 100644 --- a/pkg/internal/svc/svc.go +++ b/pkg/internal/svc/svc.go @@ -86,6 +86,10 @@ func (i *ID) GetUID() UID { } func (i *ID) String() string { + return i.Job() +} + +func (i *ID) Job() string { if i.Namespace != "" { return i.Namespace + "/" + i.Name }