Add prometheus native histogram support to otel.receiver.prometheus

CHANGELOG.md

@@ -15,6 +15,10 @@ Main (unreleased)
 - Update Public preview `remotecfg` to use `alloy-remote-config` instead of `agent-remote-config`. The
   API has been updated to use the term `collector` over `agent`. (@erikbaranowski)
 
+### Enhancements
+
+- (_Public preview_) Add native histogram support to `otelcol.receiver.prometheus`. (@wildum)
+
 ### Bugfixes
 
 - Fixed an issue with `prometheus.scrape` in which targets that move from one

internal/cmd/integration-tests/tests/scrape-prom-metrics/config.alloy

@@ -2,7 +2,7 @@ prometheus.scrape "scrape_prom_metrics" {
   targets = [
     {"__address__" = "localhost:9001"},
   ]
-  forward_to = [prometheus.remote_write.scrape_prom_metrics.receiver]
+  forward_to = [prometheus.remote_write.scrape_prom_metrics.receiver, otelcol.receiver.prometheus.scrape_prom_metrics_to_otlp.receiver]
   scrape_classic_histograms = true
   enable_protobuf_negotiation = true
   scrape_interval = "1s"
@@ -24,3 +24,31 @@ prometheus.remote_write "scrape_prom_metrics" {
     test_name = "scrape_prom_metrics",
   }
 }
+
+otelcol.receiver.prometheus "scrape_prom_metrics_to_otlp" {
+  output {
+    metrics = [otelcol.processor.attributes.scrape_prom_metrics_to_otlp.input]
+  }
+}
+
+otelcol.processor.attributes "scrape_prom_metrics_to_otlp" {
+  action {
+    key = "test_name"
+    value = "scrape_prom_metrics_to_otlp"
+    action = "insert"
+  }
+
+  output {
+    metrics = [otelcol.exporter.otlphttp.scrape_prom_metrics_to_otlp.input]
+  }
+}
+
+otelcol.exporter.otlphttp "scrape_prom_metrics_to_otlp" {
+  client {
+    endpoint = "http://localhost:9009/otlp"
+    tls {
+      insecure             = true
+      insecure_skip_verify = true
+    }
+  }
+}

internal/cmd/integration-tests/tests/scrape-prom-metrics/scrape_prom_metrics_to_otlp_test.go

@@ -0,0 +1,13 @@
+//go:build !windows
+
+package main
+
+import (
+	"testing"
+
+	"github.com/grafana/alloy/internal/cmd/integration-tests/common"
+)
+
+func TestScrapePromMetricsToOtlp(t *testing.T) {
+	common.MimirMetricsTest(t, common.PromDefaultMetrics, common.PromDefaultHistogramMetric, "scrape_prom_metrics_to_otlp")
+}

internal/cmd/integration-tests/utils.go

@@ -56,7 +56,7 @@ func runSingleTest(testDir string, port int) {
 	dirName := filepath.Base(testDir)
 
 	var alloyLogBuffer bytes.Buffer
-	cmd := exec.Command(alloyBinaryPath, "run", "config.alloy", "--server.http.listen-addr", fmt.Sprintf("0.0.0.0:%d", port))
+	cmd := exec.Command(alloyBinaryPath, "run", "config.alloy", "--server.http.listen-addr", fmt.Sprintf("0.0.0.0:%d", port), "--stability.level", "experimental")
 	cmd.Dir = testDir
 	cmd.Stdout = &alloyLogBuffer
 	cmd.Stderr = &alloyLogBuffer

internal/component/otelcol/receiver/prometheus/internal/appendable.go

@@ -16,14 +16,14 @@ import (
 )
 
 // appendable translates Prometheus scraping diffs into OpenTelemetry format.
-
 type appendable struct {
-	sink                 consumer.Metrics
-	metricAdjuster       MetricsAdjuster
-	useStartTimeMetric   bool
-	trimSuffixes         bool
-	startTimeMetricRegex *regexp.Regexp
-	externalLabels       labels.Labels
+	sink                   consumer.Metrics
+	metricAdjuster         MetricsAdjuster
+	useStartTimeMetric     bool
+	enableNativeHistograms bool
+	trimSuffixes           bool
+	startTimeMetricRegex   *regexp.Regexp
+	externalLabels         labels.Labels
 
 	settings receiver.CreateSettings
 	obsrecv  *receiverhelper.ObsReport
@@ -37,6 +37,7 @@ func NewAppendable(
 	useStartTimeMetric bool,
 	startTimeMetricRegex *regexp.Regexp,
 	useCreatedMetric bool,
+	enableNativeHistograms bool,
 	externalLabels labels.Labels,
 	trimSuffixes bool) (storage.Appendable, error) {
 
@@ -53,17 +54,18 @@ func NewAppendable(
 	}
 
 	return &appendable{
-		sink:                 sink,
-		settings:             set,
-		metricAdjuster:       metricAdjuster,
-		useStartTimeMetric:   useStartTimeMetric,
-		startTimeMetricRegex: startTimeMetricRegex,
-		externalLabels:       externalLabels,
-		obsrecv:              obsrecv,
-		trimSuffixes:         trimSuffixes,
+		sink:                   sink,
+		settings:               set,
+		metricAdjuster:         metricAdjuster,
+		useStartTimeMetric:     useStartTimeMetric,
+		enableNativeHistograms: enableNativeHistograms,
+		startTimeMetricRegex:   startTimeMetricRegex,
+		externalLabels:         externalLabels,
+		obsrecv:                obsrecv,
+		trimSuffixes:           trimSuffixes,
 	}, nil
 }
 
 func (o *appendable) Appender(ctx context.Context) storage.Appender {
-	return newTransaction(ctx, o.metricAdjuster, o.sink, o.externalLabels, o.settings, o.obsrecv, o.trimSuffixes)
+	return newTransaction(ctx, o.metricAdjuster, o.sink, o.externalLabels, o.settings, o.obsrecv, o.trimSuffixes, o.enableNativeHistograms)
 }

internal/component/otelcol/receiver/prometheus/internal/doc.go

@@ -1,5 +1,5 @@
 // Package internal is a near copy of
-// https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.87.0/receiver/prometheusreceiver/internal
+// https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/90603afc2fe0c44c9398822e4afa3a4e045f4524/receiver/prometheusreceiver/internal
 // A copy was made because the upstream package is internal. If it is ever made
 // public, our copy can be removed.
 //

internal/component/otelcol/receiver/prometheus/internal/logger.go

@@ -30,10 +30,10 @@ type zapToGokitLogAdapter struct {
 type logData struct {
 	level       level.Value
 	msg         string
-	otherFields []interface{}
+	otherFields []any
 }
 
-func (w *zapToGokitLogAdapter) Log(keyvals ...interface{}) error {
+func (w *zapToGokitLogAdapter) Log(keyvals ...any) error {
 	// expecting key value pairs, the number of items need to be even
 	if len(keyvals)%2 == 0 {
 		// Extract log level and message and log them using corresponding zap function
@@ -47,7 +47,7 @@ func (w *zapToGokitLogAdapter) Log(keyvals ...interface{}) error {
 	return nil
 }
 
-func extractLogData(keyvals []interface{}) logData {
+func extractLogData(keyvals []any) logData {
 	ld := logData{
 		level: level.InfoValue(), // default
 	}
@@ -78,7 +78,7 @@ func extractLogData(keyvals []interface{}) logData {
 }
 
 // check if a given key-value pair represents go-kit log message and return it
-func matchLogMessage(key interface{}, val interface{}) (string, bool) {
+func matchLogMessage(key any, val any) (string, bool) {
 	if strKey, ok := key.(string); !ok || strKey != msgKey {
 		return "", false
 	}
@@ -91,7 +91,7 @@ func matchLogMessage(key interface{}, val interface{}) (string, bool) {
 }
 
 // check if a given key-value pair represents go-kit log level and return it
-func matchLogLevel(key interface{}, val interface{}) (level.Value, bool) {
+func matchLogLevel(key any, val any) (level.Value, bool) {
 	strKey, ok := key.(string)
 	if !ok || strKey != levelKey {
 		return nil, false
@@ -107,7 +107,7 @@ func matchLogLevel(key interface{}, val interface{}) (level.Value, bool) {
 //revive:disable:error-return
 
 // check if a given key-value pair represents an error and return it
-func matchError(key interface{}, val interface{}) (error, bool) {
+func matchError(key any, val any) (error, bool) {
 	strKey, ok := key.(string)
 	if !ok || strKey != errKey {
 		return nil, false
@@ -123,7 +123,7 @@ func matchError(key interface{}, val interface{}) (error, bool) {
 //revive:enable:error-return
 
 // find a matching zap logging function to be used for a given level
-func levelToFunc(logger *zap.SugaredLogger, lvl level.Value) func(string, ...interface{}) {
+func levelToFunc(logger *zap.SugaredLogger, lvl level.Value) func(string, ...any) {
 	switch lvl {
 	case level.DebugValue():
 		return logger.Debugw

internal/component/otelcol/receiver/prometheus/internal/logger_test.go

@@ -20,13 +20,13 @@ import (
 func TestLog(t *testing.T) {
 	tcs := []struct {
 		name        string
-		input       []interface{}
+		input       []any
 		wantLevel   zapcore.Level
 		wantMessage string
 	}{
 		{
 			name: "Starting provider",
-			input: []interface{}{
+			input: []any{
 				"level",
 				level.DebugValue(),
 				"msg",
@@ -41,7 +41,7 @@ func TestLog(t *testing.T) {
 		},
 		{
 			name: "Scrape failed",
-			input: []interface{}{
+			input: []any{
 				"level",
 				level.ErrorValue(),
 				"scrape_pool",
@@ -84,10 +84,10 @@ func TestLog(t *testing.T) {
 func TestExtractLogData(t *testing.T) {
 	tcs := []struct {
 		name        string
-		input       []interface{}
+		input       []any
 		wantLevel   level.Value
 		wantMessage string
-		wantOutput  []interface{}
+		wantOutput  []any
 	}{
 		{
 			name:        "nil fields",
@@ -98,14 +98,14 @@ func TestExtractLogData(t *testing.T) {
 		},
 		{
 			name:        "empty fields",
-			input:       []interface{}{},
+			input:       []any{},
 			wantLevel:   level.InfoValue(), // Default
 			wantMessage: "",
 			wantOutput:  nil,
 		},
 		{
 			name: "info level",
-			input: []interface{}{
+			input: []any{
 				"level",
 				level.InfoValue(),
 			},
@@ -115,7 +115,7 @@ func TestExtractLogData(t *testing.T) {
 		},
 		{
 			name: "warn level",
-			input: []interface{}{
+			input: []any{
 				"level",
 				level.WarnValue(),
 			},
@@ -125,7 +125,7 @@ func TestExtractLogData(t *testing.T) {
 		},
 		{
 			name: "error level",
-			input: []interface{}{
+			input: []any{
 				"level",
 				level.ErrorValue(),
 			},
@@ -135,7 +135,7 @@ func TestExtractLogData(t *testing.T) {
 		},
 		{
 			name: "debug level + extra fields",
-			input: []interface{}{
+			input: []any{
 				"timestamp",
 				1596604719,
 				"level",
@@ -145,32 +145,32 @@ func TestExtractLogData(t *testing.T) {
 			},
 			wantLevel:   level.DebugValue(),
 			wantMessage: "http client error",
-			wantOutput: []interface{}{
+			wantOutput: []any{
 				"timestamp", 1596604719,
 			},
 		},
 		{
 			name: "missing level field",
-			input: []interface{}{
+			input: []any{
 				"timestamp",
 				1596604719,
 				"msg",
 				"http client error",
 			},
 			wantLevel:   level.InfoValue(), // Default
 			wantMessage: "http client error",
-			wantOutput: []interface{}{
+			wantOutput: []any{
 				"timestamp", 1596604719,
 			},
 		},
 		{
 			name: "invalid level type",
-			input: []interface{}{
+			input: []any{
 				"level",
 				"warn", // String is not recognized
 			},
 			wantLevel: level.InfoValue(), // Default
-			wantOutput: []interface{}{
+			wantOutput: []any{
 				"level", "warn", // Field is preserved
 			},
 		},