Merge pull request #810 from grafana/prometheus-ff398062c

Sync upstream Prometheus to ff39806

config/config.go

@@ -1427,10 +1427,13 @@ func getGoGCEnv() int {
 type translationStrategyOption string
 
 var (
-	// NoUTF8EscapingWithSuffixes will keep UTF-8 characters as they are, units and type suffixes will still be added.
+	// NoUTF8EscapingWithSuffixes will accept metric/label names as they are.
+	// Unit and type suffixes may be added to metric names, according to certain rules.
 	NoUTF8EscapingWithSuffixes translationStrategyOption = "NoUTF8EscapingWithSuffixes"
 	// UnderscoreEscapingWithSuffixes is the default option for translating OTLP to Prometheus.
-	// This option will translate all UTF-8 characters to underscores, while adding units and type suffixes.
+	// This option will translate metric name characters that are not alphanumerics/underscores/colons to underscores,
+	// and label name characters that are not alphanumerics/underscores to underscores.
+	// Unit and type suffixes may be appended to metric names, according to certain rules.
 	UnderscoreEscapingWithSuffixes translationStrategyOption = "UnderscoreEscapingWithSuffixes"
 )
 

docs/configuration/configuration.md

@@ -179,8 +179,8 @@ otlp:
   # - "UnderscoreEscapingWithSuffixes" refers to commonly agreed normalization used
   #   by OpenTelemetry in https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/translator/prometheus
   # - "NoUTF8EscapingWithSuffixes" is a mode that relies on UTF-8 support in Prometheus.
-  #   It preserves all special characters like dots, but it still add required suffixes
-  #   for units and _total like in UnderscoreEscapingWithSuffixes.
+  #   It preserves all special characters like dots, but still adds required metric name suffixes
+  #   for units and _total, as UnderscoreEscapingWithSuffixes does.
   [ translation_strategy: <string> | default = "UnderscoreEscapingWithSuffixes" ]
   # Enables adding "service.name", "service.namespace" and "service.instance.id"
   # resource attributes to the "target_info" metric, on top of converting

docs/feature_flags.md

@@ -15,7 +15,7 @@ They may be enabled by default in future versions.
 
 `--enable-feature=exemplar-storage`
 
-[OpenMetrics](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces.
+[OpenMetrics](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars) introduces the ability for scrape targets to add exemplars to certain metrics. Exemplars are references to data outside of the MetricSet. A common use case are IDs of program traces.
 
 Exemplar storage is implemented as a fixed size circular buffer that stores exemplars in memory for all series. Enabling this feature will enable the storage of exemplars scraped by Prometheus. The config file block [storage](configuration/configuration.md#configuration-file)/[exemplars](configuration/configuration.md#exemplars) can be used to control the size of circular buffer by # of exemplars. An exemplar with just a `trace_id=<jaeger-trace-id>` uses roughly 100 bytes of memory via the in-memory exemplar storage. If the exemplar storage is enabled, we will also append the exemplars to WAL for local persistence (for WAL duration).
 

documentation/examples/prometheus-otlp.yml

@@ -22,7 +22,7 @@ otlp:
     - k8s.pod.name
     - k8s.replicaset.name
     - k8s.statefulset.name
-  # Ingest OTLP data keeping UTF-8 characters in metric/label names.
+  # Ingest OTLP data keeping all characters in metric/label names.
   translation_strategy: NoUTF8EscapingWithSuffixes
 
 storage:

model/exemplar/exemplar.go

@@ -18,7 +18,7 @@ import "github.com/prometheus/prometheus/model/labels"
 // ExemplarMaxLabelSetLength is defined by OpenMetrics: "The combined length of
 // the label names and values of an Exemplar's LabelSet MUST NOT exceed 128
 // UTF-8 characters."
-// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
+// https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars
 const ExemplarMaxLabelSetLength = 128
 
 // Exemplar is additional information associated with a time series.

model/labels/labels.go

@@ -19,6 +19,7 @@ import (
 	"bytes"
 	"slices"
 	"strings"
+	"unsafe"
 
 	"github.com/cespare/xxhash/v2"
 )
@@ -488,3 +489,8 @@ func (b *ScratchBuilder) Labels() Labels {
 func (b *ScratchBuilder) Overwrite(ls *Labels) {
 	*ls = append((*ls)[:0], b.add...)
 }
+
+// SizeOfLabels returns the approximate space required for n copies of a label.
+func SizeOfLabels(name, value string, n uint64) uint64 {
+	return (uint64(len(name)) + uint64(unsafe.Sizeof(name)) + uint64(len(value)) + uint64(unsafe.Sizeof(value))) * n
+}

model/labels/labels_dedupelabels.go

@@ -815,3 +815,8 @@ func (b *ScratchBuilder) Overwrite(ls *Labels) {
 	ls.syms = b.syms.nameTable
 	ls.data = yoloString(b.overwriteBuffer)
 }
+
+// SizeOfLabels returns the approximate space required for n copies of a label.
+func SizeOfLabels(name, value string, n uint64) uint64 {
+	return uint64(len(name)+len(value)) + n*4 // Assuming most symbol-table entries are 2 bytes long.
+}

model/labels/labels_stringlabels.go

@@ -691,3 +691,8 @@ func NewScratchBuilderWithSymbolTable(_ *SymbolTable, n int) ScratchBuilder {
 func (b *ScratchBuilder) SetSymbolTable(_ *SymbolTable) {
 	// no-op
 }
+
+// SizeOfLabels returns the approximate space required for n copies of a label.
+func SizeOfLabels(name, value string, n uint64) uint64 {
+	return uint64(labelSize(&Label{Name: name, Value: value})) * n
+}

model/textparse/openmetricsparse.go

@@ -337,7 +337,7 @@ func (p *OpenMetricsParser) CreatedTimestamp() *int64 {
 		}
 
 		// All timestamps in OpenMetrics are Unix Epoch in seconds. Convert to milliseconds.
-		// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#timestamps
+		// https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#timestamps
 		ct := int64(p.val * 1000.0)
 		p.setCTParseValues(ct, currHash, currName, true)
 		return &ct

scrape/manager_test.go

@@ -836,7 +836,7 @@ scrape_configs:
 								require.Len(t, createdSeriesSamples, 1)
 								// Conversion taken from common/expfmt.writeOpenMetricsFloat.
 								// We don't check the ct timestamp as explicit ts was not implemented in expfmt.Encoder,
-								// but exists in OM https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#:~:text=An%20example%20with%20a%20Metric%20with%20no%20labels%2C%20and%20a%20MetricPoint%20with%20a%20timestamp%20and%20a%20created
+								// but exists in OM https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#:~:text=An%20example%20with%20a%20Metric%20with%20no%20labels%2C%20and%20a%20MetricPoint%20with%20a%20timestamp%20and%20a%20created
 								// We can implement this, but we want to potentially get rid of OM 1.0 CT lines
 								require.Equal(t, float64(timestamppb.New(ctTs).AsTime().UnixNano())/1e9, createdSeriesSamples[0].f)
 							} else {

storage/remote/otlptranslator/prometheus/normalize_name.go

@@ -30,7 +30,7 @@ import (
 // OTLP metrics use the c/s notation as specified at https://ucum.org/ucum.html
 // (See also https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/semantic_conventions/README.md#instrument-units)
 // Prometheus best practices for units: https://prometheus.io/docs/practices/naming/#base-units
-// OpenMetrics specification for units: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#units-and-base-units
+// OpenMetrics specification for units: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#units-and-base-units
 var unitMap = map[string]string{
 	// Time
 	"d":   "days",

storage/remote/otlptranslator/prometheusremotewrite/helper.go

@@ -51,7 +51,7 @@ const (
 	createdSuffix = "_created"
 	// maxExemplarRunes is the maximum number of UTF-8 exemplar characters
 	// according to the prometheus specification
-	// https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
+	// https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars
 	maxExemplarRunes = 128
 	// Trace and Span id keys are defined as part of the spec:
 	// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification%2Fmetrics%2Fdatamodel.md#exemplars-2

tsdb/docs/format/wal.md

@@ -126,7 +126,7 @@ The first row stores the starting id and the starting timestamp.
 Series reference and timestamp are encoded as deltas w.r.t the first exemplar.
 The first exemplar record begins at the second row.
 
-See: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#exemplars
+See: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#exemplars
 
 ```
 ┌──────────────────────────────────────────────────────────────────┐

tsdb/head.go

@@ -1079,7 +1079,7 @@ func (h *Head) PostingsCardinalityStats(statsByLabelName string, limit int) *ind
 		return h.cardinalityCache
 	}
 	h.cardinalityCacheKey = cacheKey
-	h.cardinalityCache = h.postings.Stats(statsByLabelName, limit)
+	h.cardinalityCache = h.postings.Stats(statsByLabelName, limit, labels.SizeOfLabels)
 	h.lastPostingsStatsCall = time.Duration(time.Now().Unix()) * time.Second
 
 	return h.cardinalityCache

tsdb/index/postings.go

@@ -72,7 +72,7 @@ type MemPostings struct {
 	// lvs holds the label values for each label name.
 	// lvs[name] is essentially an unsorted append-only list of all keys in m[name]
 	// mtx must be held when interacting with lvs.
-	// Since it's append-only, it is safe to the label values slice after releasing the lock.
+	// Since it's append-only, it is safe to read the label values slice after releasing the lock.
 	lvs map[string][]string
 
 	ordered bool
@@ -190,7 +190,8 @@ type PostingsStats struct {
 }
 
 // Stats calculates the cardinality statistics from postings.
-func (p *MemPostings) Stats(label string, limit int) *PostingsStats {
+// Caller can pass in a function which computes the space required for n series with a given label.
+func (p *MemPostings) Stats(label string, limit int, labelSizeFunc func(string, string, uint64) uint64) *PostingsStats {
 	var size uint64
 	p.mtx.RLock()
 
@@ -218,7 +219,7 @@ func (p *MemPostings) Stats(label string, limit int) *PostingsStats {
 			}
 			seriesCnt := uint64(len(values))
 			labelValuePairs.push(Stat{Name: n + "=" + name, Count: seriesCnt})
-			size += uint64(len(name)) * seriesCnt
+			size += labelSizeFunc(n, name, seriesCnt)
 		}
 		labelValueLength.push(Stat{Name: n, Count: size})
 	}

tsdb/index/postings_test.go

@@ -939,7 +939,7 @@ func BenchmarkPostings_Stats(b *testing.B) {
 	}
 	b.ResetTimer()
 	for n := 0; n < b.N; n++ {
-		p.Stats("__name__", 10)
+		p.Stats("__name__", 10, labels.SizeOfLabels)
 	}
 }
 
@@ -954,7 +954,8 @@ func TestMemPostingsStats(t *testing.T) {
 	p.Add(2, labels.FromStrings("label", "value1"))
 
 	// call the Stats method to calculate the cardinality statistics
-	stats := p.Stats("label", 10)
+	// passing a fake calculation so we get the same result regardless of compilation -tags.
+	stats := p.Stats("label", 10, func(name, value string, n uint64) uint64 { return uint64(len(name)+len(value)) * n })
 
 	// assert that the expected statistics were calculated
 	require.Equal(t, uint64(2), stats.CardinalityMetricsStats[0].Count)
@@ -963,7 +964,7 @@ func TestMemPostingsStats(t *testing.T) {
 	require.Equal(t, uint64(3), stats.CardinalityLabelStats[0].Count)
 	require.Equal(t, "label", stats.CardinalityLabelStats[0].Name)
 
-	require.Equal(t, uint64(24), stats.LabelValueStats[0].Count)
+	require.Equal(t, uint64(44), stats.LabelValueStats[0].Count)
 	require.Equal(t, "label", stats.LabelValueStats[0].Name)
 
 	require.Equal(t, uint64(2), stats.LabelValuePairsStats[0].Count)

tsdb/querier.go

@@ -270,7 +270,7 @@ func PostingsForMatchers(ctx context.Context, ix IndexPostingsReader, ms ...*lab
 			its = append(its, it)
 		case m.Type == labels.MatchNotRegexp && m.Value == ".+":
 			// .+ regexp matches any non-empty string: get postings for all label values and remove them.
-			its = append(notIts, ix.PostingsForAllLabelValues(ctx, m.Name))
+			notIts = append(notIts, ix.PostingsForAllLabelValues(ctx, m.Name))
 
 		case labelMustBeSet[m.Name]:
 			// If this matcher must be non-empty, we can be smarter.

tsdb/querier_test.go

@@ -3086,6 +3086,15 @@ func TestPostingsForMatchers(t *testing.T) {
 			matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "n", "1"), labels.MustNewMatcher(labels.MatchNotRegexp, "i", "^.*$")},
 			exp:      []labels.Labels{},
 		},
+		// Test shortcut i!~".+"
+		{
+			matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "n", ".*"), labels.MustNewMatcher(labels.MatchNotRegexp, "i", ".+")},
+			exp: []labels.Labels{
+				labels.FromStrings("n", "1"),
+				labels.FromStrings("n", "2"),
+				labels.FromStrings("n", "2.5"),
+			},
+		},
 	}
 
 	ir, err := h.Index()