"Alloy / OpenTelemetry" dashboard has unexpected blank panels

[rfratto]

What's wrong?

Multiple panels on the "Alloy / OpenTelemetry" dashboard are blank; potentially related to #711.

Steps to reproduce

1. Run Alloy with a main commit build, sending OpenTelemetry data to Prometheus or Mimir.
2. Install the Alloy Jsonnet mixin.
3. Observe the "Alloy / OpenTelemetry" dashboard has multiple blank panels.

System information

Linux

Software version

v1.2.0-devel+270e3dbbb

Configuration

otelcol.auth.headers "otlp_gateway" {
	header {
        key   = "X-Scope-OrgID"
        value = "REDACTED"
    }
}

otelcol.receiver.jaeger "default" {
	debug_metrics {
		disable_high_cardinality_metrics = true
	}

  protocols {
    grpc {}
    thrift_http {}
    thrift_binary {}
    thrift_compact {}
  }

  output {
		traces  = [otelcol.processor.k8sattributes.default.input]
  }
}

otelcol.receiver.otlp "default" {
	debug_metrics {
		disable_high_cardinality_metrics = true
	}
  
	grpc { }

	http { }

	output {
		metrics = [otelcol.processor.k8sattributes.default.input]
		logs    = [otelcol.processor.k8sattributes.default.input]
		traces  = [otelcol.processor.k8sattributes.default.input]
	}
}

otelcol.processor.k8sattributes "default" {
	extract {
		metadata = ["k8s.pod.name", "k8s.namespace.name", "k8s.node.name", "k8s.container.name"]
	}

	output {
		metrics = [otelcol.processor.batch.default.input]
		logs    = [otelcol.processor.batch.default.input]
		traces  = [otelcol.processor.discovery.default.input, otelcol.connector.host_info.default.input]
	}
}

discovery.kubernetes "pods" {
	role = "pod"
}

//TODO: Figure out how discovery.kubernetes is able to discover the pod container.
// otelcol.processor.k8sattributes is not able to discover it, because all containers on
// same pod share the same IP address.
//
//TODO: It feels wasteful to run both otelcol.processor.k8sattributes and otelcol.processor.discovery.
// Can we run only one of them? Either k8sattributes needs to be able to find the container names by itself,
// or otelcol.processor.discovery should start to follow OTel semantic conventions.
discovery.relabel "pods" {
	rule {
		action        = "replace"
		source_labels = ["__meta_kubernetes_pod_container_name"]
		target_label  = "container"
	}

	rule {
		action        = "drop"
		regex         = ""
		source_labels = ["__meta_kubernetes_pod_container_port_number"]
	}

	rule {
		action        = "replace"
		source_labels = ["__meta_kubernetes_pod_ip"]
		target_label  = "__address__"
	}
	targets = discovery.kubernetes.pods.targets
}

// otelcol.connector.host_info extracts host information from traces and exports it as metrics.
// This is used for billing in app o11y's host based pricing model.
otelcol.connector.host_info "default" {
	metrics_flush_interval = "60s"
	host_identifiers = ["k8s.node.name"]
	output {
		metrics = [otelcol.processor.batch.default.input]
	}
}

// otelcol.processor.discovery goes after otelcol.processor.k8sattributes,
// because k8sattributes is what we want to keep in the long term.
// otelcol.processor.discovery should not add any attributes which were already added
// by otelcol.processor.k8sattributes.
otelcol.processor.discovery "default" {
	operation_type = "insert"
	output {
		traces = [otelcol.processor.transform.default.input]
	}
	targets = discovery.relabel.pods.output
}

otelcol.processor.transform "default" {
	 error_mode = "ignore"

	trace_statements {
		context = "span"
		statements = [
			"set(attributes[\"otel.library.name\"], instrumentation_scope.attributes[\"otel.library.name\"])",
			"set(attributes[\"otel.scope.name\"], instrumentation_scope.attributes[\"otel.scope.name\"])",
	
			"set(resource.attributes[\"k8s.cluster.name\"], \"" + env("CLUSTER_NAME") + "\")",
		]
	}

	output {
		traces = [otelcol.processor.batch.default.input]
	}
}

otelcol.processor.batch "default" {
	output {
		metrics = [otelcol.exporter.otlphttp.otlp_gateway.input]
		logs    = [otelcol.exporter.otlphttp.otlp_gateway.input]
		traces  = [otelcol.exporter.otlphttp.otlp_gateway.input]
	}
}

otelcol.exporter.otlphttp "otlp_gateway" {
	client {
		auth     = otelcol.auth.headers.otlp_gateway.handler
		endpoint = "REDACTED/otlp"
	}
}

local.file "sampling" {
	filename = "/var/sampling-strategies/jaeger-sampling.json"
}

otelcol.extension.jaeger_remote_sampling "default" {
	http {}
	source {
		content = local.file.sampling.content
	}
}

Logs

N/A

[rfratto]

cc @ptodev Can you help me look into this once you're back from PTO?

[ptodev]

Apologies for this - I'd seen this breaking change in the OTel changelog and wasn't too sure if it's going to have an impact for us. I meant to test it later, but didn't get around to it - I should have left a comment on the PR.

I opened a PR to fix this now.