Fix instance and job in prometheus, and document honor_labels

docs/sources/configure/options.md

@@ -300,16 +300,16 @@ precedence:
 
 - If Kubernetes is enabled:
  1. The name of the Deployment that runs the instrumented process, if any.
- 2. The name of the ReplicaSet that runs the instrumented process, if any.
+ 2. The name of the ReplicaSet/DaemonSet/StatefulSet that runs the instrumented process, if any.
  3. The name of the Pod that runs the instrumented process.
 - If kubernetes is not enabled:
  1. The name of the process executable file.
 
 If multiple processes match the service selection criteria described below,
 the metrics and traces for all the instances might share the same service name;
 for example, when multiple instrumented processes run under the same Deployment,
-or have the same executable name. In that case, the reported `instance.id` (OTEL) or
-`target_instance` (Prometheus) would allow differentiating the different instances
+or have the same executable name. In that case, the reported `instance` attribute
+would allow differentiating the different instances
 of the service.
 
 | YAML | Environment variable | Type | Default |
@@ -1215,6 +1215,12 @@ API key of your Grafana Cloud account.
 
 ## Prometheus HTTP endpoint
 
+> ℹ️ The Prometheus scraper might override the values of the `instance` and `job` labels.
+> To keep the original values as set by Beyla, make sure to configure the
+> Prometheus scraper to set the `honor_labels` option to `true`.
+> 
+> ([How to set `honor_labels` in Grafana Alloy](/docs/alloy/latest/reference/components/prometheus/prometheus.scrape/)).
+
 YAML section `prometheus_export`.
 
 This component opens an HTTP endpoint in the auto-instrumentation tool

pkg/export/attributes/attr_defs.go

@@ -43,7 +43,8 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup {
  var prometheusAttributes = AttrReportGroup{
  Disabled: !promEnabled,
  Attributes: map[attr.Name]Default{
- attr.TargetInstance: true,
+ attr.Instance: true,
+ attr.Job: true,
  attr.ServiceNamespace: true,
  },
  }
@@ -166,11 +167,12 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup {
  var promProcessAttributes = AttrReportGroup{
  Disabled: !promEnabled,
  Attributes: map[attr.Name]Default{
+ attr.Instance: true,
+ attr.Job: true,
  attr.ProcCommand: true,
  attr.ProcOwner: true,
  attr.ProcParentPid: true,
  attr.ProcPid: true,
- attr.TargetInstance: true,
  attr.ProcCommandLine: false,
  attr.ProcCommandArgs: false,
  attr.ProcExecName: false,

pkg/export/attributes/names/attrs.go

@@ -124,13 +124,13 @@ const (
 
 // other beyla-specific attributes
 const (
- // TargetInstance is a Prometheus-only attribute.
- // It will expose the process hostname-pid (or K8s Pod).
- // It is advised for users that to use relabeling rules to
- // override the "instance" attribute with "target" in the
- // Prometheus server. This would be similar to the "multi target pattern":
- // https://prometheus.io/docs/guides/multi-target-exporter/
- TargetInstance = Name("target.instance")
+ // Instance and Job are only explicitly used in the Prometheus
+ // exporter, as the OpenTelemetry SDK already sets them implicitly.
+ // It is advised for users to configure their Prometheus scraper with
+ // the `honor_labels` option set to true, to avoid overwriting the
+ // instance attribute with the target attribute.
+ Instance = Name("instance")
+ Job = Name("job")
 
  // ServiceName and ServiceNamespace are going to be used only on Prometheus
  // as metric attributes. The OTEL exporter already uses them as Resource

pkg/export/prom/prom.go

@@ -701,18 +701,14 @@ func labelNamesSpans() []string {
 }
 
 func (r *metricsReporter) labelValuesSpans(span *request.Span) []string {
- job := span.ServiceID.Name
- if span.ServiceID.Namespace != "" {
- job = span.ServiceID.Namespace + "/" + job
- }
  return []string{
  span.ServiceID.Name,
  span.ServiceID.Namespace,
  span.TraceName(),
  strconv.Itoa(int(request.SpanStatusCode(span))),
  span.ServiceGraphKind(),
  string(span.ServiceID.UID), // app instance ID
- job,
+ span.ServiceID.Job(),
  "beyla",
  }
 }
@@ -728,17 +724,13 @@ func labelNamesTargetInfo(kubeEnabled bool) []string {
 }
 
 func (r *metricsReporter) labelValuesTargetInfo(service svc.ID) []string {
- job := service.Name
- if service.Namespace != "" {
- job = service.Namespace + "/" + job
- }
  values := []string{
  r.hostID,
  service.HostName,
  service.Name,
  service.Namespace,
  string(service.UID), // app instance ID
- job,
+ service.Job(),
  service.SDKLanguage.String(),
  "beyla",
  "beyla",

pkg/internal/infraolly/process/status.go

@@ -119,8 +119,10 @@ func PromGetters(name attr.Name) (attributes.Getter[*Status, string], bool) {
  case attr.ProcCPUMode, attr.ProcDiskIODir, attr.ProcNetIODir:
  // the attributes are handled explicitly by the prometheus exporter, but we need to
  // ignore them to avoid that the default case tries to report them from service metadata
- case attr.TargetInstance:
+ case attr.Instance:
  g = func(s *Status) string { return string(s.ID.UID) }
+ case attr.Job:
+ g = func(s *Status) string { return s.ID.Service.Job() }
  default:
  g = func(s *Status) string { return s.ID.Service.Metadata[name] }
  }

pkg/internal/request/span_getters.go

@@ -180,8 +180,10 @@ func SpanPromGetters(attrName attr.Name) (attributes.Getter[*Span, string], bool
  getter = func(s *Span) string { return string(s.ServiceID.UID) }
  // resource metadata values below. Unlike OTEL, they are included here because they
  // belong to the metric, instead of the Resource
- case attr.TargetInstance:
+ case attr.Instance:
  getter = func(s *Span) string { return string(s.ServiceID.UID) }
+ case attr.Job:
+ getter = func(s *Span) string { return s.ServiceID.Job() }
  case attr.ServiceName:
  getter = func(s *Span) string { return s.ServiceID.Name }
  case attr.ServiceNamespace:

pkg/internal/svc/svc.go

@@ -86,6 +86,10 @@ func (i *ID) GetUID() UID {
 }
 
 func (i *ID) String() string {
+ return i.Job()
+}
+
+func (i *ID) Job() string {
  if i.Namespace != "" {
  return i.Namespace + "/" + i.Name
  }