"wake up" internal prometheus scrapper metrics (up / scrape_xxxx) (#3116

)

* Try to make internal prometheus scrapper metrics working

* Alternative method, working directly on metricFamily

* Try to fix unittests

* Try to fix prometheus exporter tests

* Try to fix prometheus exporter tests

* Remove unwanted test about internal metrics

* Try to fix unittests

* Try to fix unittests

* Try to fix unittests

* receiver/prometheus: fix e2e tests

Signed-off-by: Anthony J Mirabella <a9@aneurysm9.com>

* receiver/prometheus: add e2e tests for label keys and values

Signed-off-by: Anthony J Mirabella <a9@aneurysm9.com>

* receiver/prometheus: remove unused test metric spec

Signed-off-by: Anthony J Mirabella <a9@aneurysm9.com>

* Add useful logs

* Fix useless units and comments

* Fix too verbose log and bad test

* Fix prom exporter test if scrap duration is instant

* Fix lint

* Update exporter/prometheusexporter/end_to_end_test.go

Co-authored-by: Anthony Mirabella <a9@aneurysm9.com>

* Try to fix exporter test

* Remove unwanted file

* fix tests after rebasing

* Fix lint

Co-authored-by: Anthony J Mirabella <a9@aneurysm9.com>
Co-authored-by: David Ashpole <dashpole@google.com>

exporter/prometheusexporter/end_to_end_test.go

@@ -152,6 +152,21 @@ func TestEndToEndSummarySupport(t *testing.T) {
 		`test_jvm_memory_pool_bytes_used.pool="G1 Old Gen". 4.385408e.06.*`,
 		`test_jvm_memory_pool_bytes_used.pool="G1 Survivor Space". 8.388608e.06.*`,
 		`test_jvm_memory_pool_bytes_used.pool="Metaspace". 2.6218176e.07.*`,
+		`. HELP test_scrape_duration_seconds Duration of the scrape`,
+		`. TYPE test_scrape_duration_seconds gauge`,
+		`test_scrape_duration_seconds [0-9.e-]+ [0-9]+`,
+		`. HELP test_scrape_samples_post_metric_relabeling The number of samples remaining after metric relabeling was applied`,
+		`. TYPE test_scrape_samples_post_metric_relabeling gauge`,
+		`test_scrape_samples_post_metric_relabeling 13 .*`,
+		`. HELP test_scrape_samples_scraped The number of samples the target exposed`,
+		`. TYPE test_scrape_samples_scraped gauge`,
+		`test_scrape_samples_scraped 13 .*`,
+		`. HELP test_scrape_series_added The approximate number of new series in this scrape`,
+		`. TYPE test_scrape_series_added gauge`,
+		`test_scrape_series_added 13 .*`,
+		`. HELP test_up The scraping was successful`,
+		`. TYPE test_up gauge`,
+		`test_up 1 .*`,
 	}
 
 	// 5.5: Perform a complete line by line prefix verification to ensure we extract back the inputs

receiver/prometheusreceiver/internal/metricfamily.go

@@ -15,13 +15,15 @@
 package internal
 
 import (
+	"fmt"
 	"sort"
 	"strings"
 
 	metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/textparse"
 	"github.com/prometheus/prometheus/scrape"
+	"go.uber.org/zap"
 	"google.golang.org/protobuf/types/known/timestamppb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 )
@@ -46,7 +48,7 @@ type metricFamily struct {
 	groups            map[string]*metricGroup
 }
 
-func newMetricFamily(metricName string, mc MetadataCache) MetricFamily {
+func newMetricFamily(metricName string, mc MetadataCache, logger *zap.Logger) MetricFamily {
 	familyName := normalizeMetricName(metricName)
 
 	// lookup metadata based on familyName
@@ -62,11 +64,17 @@ func newMetricFamily(metricName string, mc MetadataCache) MetricFamily {
 			metadata.Metric = familyName
 			metadata.Type = textparse.MetricTypeUnknown
 		}
+	} else if !ok && isInternalMetric(metricName) {
+		metadata = defineInternalMetric(metricName, metadata, logger)
+	}
+	ocaMetricType := convToOCAMetricType(metadata.Type)
+	if ocaMetricType == metricspb.MetricDescriptor_UNSPECIFIED {
+		logger.Debug(fmt.Sprintf("Invalid metric : %s %+v", metricName, metadata))
 	}
 
 	return &metricFamily{
 		name:              familyName,
-		mtype:             convToOCAMetricType(metadata.Type),
+		mtype:             ocaMetricType,
 		mc:                mc,
 		droppedTimeseries: 0,
 		labelKeys:         make(map[string]bool),
@@ -77,6 +85,35 @@ func newMetricFamily(metricName string, mc MetadataCache) MetricFamily {
 	}
 }
 
+// Define manually the metadata of prometheus scrapper internal metrics
+func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, logger *zap.Logger) scrape.MetricMetadata {
+	if metadata.Metric != "" && metadata.Type != "" && metadata.Help != "" {
+		logger.Debug("Internal metric seems already fully defined")
+		return metadata
+	}
+	metadata.Metric = metricName
+
+	switch metricName {
+	case scrapeUpMetricName:
+		metadata.Type = textparse.MetricTypeGauge
+		metadata.Help = "The scraping was successful"
+	case "scrape_duration_seconds":
+		metadata.Unit = "seconds"
+		metadata.Type = textparse.MetricTypeGauge
+		metadata.Help = "Duration of the scrape"
+	case "scrape_samples_scraped":
+		metadata.Type = textparse.MetricTypeGauge
+		metadata.Help = "The number of samples the target exposed"
+	case "scrape_series_added":
+		metadata.Type = textparse.MetricTypeGauge
+		metadata.Help = "The approximate number of new series in this scrape"
+	case "scrape_samples_post_metric_relabeling":
+		metadata.Type = textparse.MetricTypeGauge
+		metadata.Help = "The number of samples remaining after metric relabeling was applied"
+	}
+	return metadata
+}
+
 func (mf *metricFamily) IsSameFamily(metricName string) bool {
 	// trim known suffix if necessary
 	familyName := normalizeMetricName(metricName)

receiver/prometheusreceiver/internal/metricsbuilder.go

@@ -114,7 +114,6 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error
 	case isInternalMetric(metricName):
 		b.hasInternalMetric = true
 		lm := ls.Map()
-		delete(lm, model.MetricNameLabel)
 		// See https://www.prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series
 		// up: 1 if the instance is healthy, i.e. reachable, or 0 if the scrape failed.
 		if metricName == scrapeUpMetricName && v != 1.0 {
@@ -129,7 +128,6 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error
 					zap.String("target_labels", fmt.Sprintf("%v", lm)))
 			}
 		}
-		return nil
 	case b.useStartTimeMetric && b.matchStartTimeMetric(metricName):
 		b.startTime = v
 	}
@@ -143,9 +141,9 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error
 		if m != nil {
 			b.metrics = append(b.metrics, m)
 		}
-		b.currentMf = newMetricFamily(metricName, b.mc)
+		b.currentMf = newMetricFamily(metricName, b.mc, b.logger)
 	} else if b.currentMf == nil {
-		b.currentMf = newMetricFamily(metricName, b.mc)
+		b.currentMf = newMetricFamily(metricName, b.mc, b.logger)
 	}
 
 	return b.currentMf.Add(metricName, ls, t, v)

receiver/prometheusreceiver/internal/metricsbuilder_test.go

@@ -1198,34 +1198,6 @@ func Test_metricBuilder_summary(t *testing.T) {
 	runBuilderTests(t, tests)
 }
 
-func Test_metricBuilder_skipped(t *testing.T) {
-	tests := []buildTestData{
-		{
-			name: "skip-internal-metrics",
-			inputs: []*testScrapedPage{
-				{
-					pts: []*testDataPoint{
-						createDataPoint("scrape_foo", 1),
-						createDataPoint("up", 1.0),
-					},
-				},
-				{
-					pts: []*testDataPoint{
-						createDataPoint("scrape_foo", 2),
-						createDataPoint("up", 2.0),
-					},
-				},
-			},
-			wants: [][]*metricspb.Metric{
-				{},
-				{},
-			},
-		},
-	}
-
-	runBuilderTests(t, tests)
-}
-
 func Test_metricBuilder_baddata(t *testing.T) {
 	t.Run("empty-metric-name", func(t *testing.T) {
 		mc := newMockMetadataCache(testMetadata)

receiver/prometheusreceiver/internal/otlp_metricfamily_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/prometheus/prometheus/pkg/textparse"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap"
 )
 
 type byLookupMetadataCache map[string]scrape.MetricMetadata
@@ -89,7 +90,7 @@ func TestIsCumulativeEquivalence(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			mf := newMetricFamily(tt.name, mc).(*metricFamily)
+			mf := newMetricFamily(tt.name, mc, zap.NewNop()).(*metricFamily)
 			mfp := newMetricFamilyPdata(tt.name, mc).(*metricFamilyPdata)
 			assert.Equal(t, mf.isCumulativeType(), mfp.isCumulativeTypePdata(), "mismatch in isCumulative")
 			assert.Equal(t, mf.isCumulativeType(), tt.want, "isCumulative does not match for regular metricFamily")