From 73b584c77ba9da821bb128fb001016483399a736 Mon Sep 17 00:00:00 2001 From: Justin Lei Date: Fri, 20 Sep 2024 13:50:18 -0700 Subject: [PATCH] Update mimir-prometheus (#9358) * Update mimir-prometheus * Run make generate-otlp --- go.mod | 2 +- go.sum | 4 ++-- pkg/distributor/otlp/helper_generated.go | 21 +++++++++++++++++ pkg/distributor/otlp/histograms_generated.go | 5 ++++ .../prometheusremotewrite/helper.go | 21 +++++++++++++++++ .../prometheusremotewrite/histograms.go | 5 ++++ .../prometheus/prometheus/tsdb/db.go | 8 +++++-- .../prometheus/tsdb/ooo_head_read.go | 23 ++++++++++--------- vendor/modules.txt | 4 ++-- 9 files changed, 75 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 2f329639639..983e5b21404 100644 --- a/go.mod +++ b/go.mod @@ -279,7 +279,7 @@ require ( ) // Using a fork of Prometheus with Mimir-specific changes. -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240917104844-e4312d119ad6 +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240920170818-87bc81f0005c // client_golang v1.20.3 has some data races in histogram exemplars. // Stick to v1.19.1 until they are fixed. diff --git a/go.sum b/go.sum index 56cb152924f..a1a41ba92fd 100644 --- a/go.sum +++ b/go.sum @@ -1265,8 +1265,8 @@ github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56 h1:X8IKQ0wu40wp github.com/grafana/gomemcache v0.0.0-20240229205252-cd6a66d6fb56/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/grafana/mimir-prometheus v0.0.0-20240917104844-e4312d119ad6 h1:ytXRuYayyA9+LNmmeWmXeC/eDeyRM4YNVDntdJkYC40= -github.com/grafana/mimir-prometheus v0.0.0-20240917104844-e4312d119ad6/go.mod h1:oyDm7JaLUh+QGuGkC7iXC8IyTUq5rlh1ba2CRm9DlVg= +github.com/grafana/mimir-prometheus v0.0.0-20240920170818-87bc81f0005c h1:+Bs4f9RDbRpBAzq50oCnbkc9yZNFkKZl53uchHvCwmc= +github.com/grafana/mimir-prometheus v0.0.0-20240920170818-87bc81f0005c/go.mod h1:oyDm7JaLUh+QGuGkC7iXC8IyTUq5rlh1ba2CRm9DlVg= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/prometheus-alertmanager v0.25.1-0.20240906122908-60521fae6a42 h1:01cldIKyxTrraTYroDWNBItBWWs9Bh07I4rSOakLilo= diff --git a/pkg/distributor/otlp/helper_generated.go b/pkg/distributor/otlp/helper_generated.go index c51d38a0168..3ca6c4dfa12 100644 --- a/pkg/distributor/otlp/helper_generated.go +++ b/pkg/distributor/otlp/helper_generated.go @@ -608,6 +608,27 @@ func (c *MimirConverter) handleStartTime(startTs, ts int64, labels []mimirpb.Lab c.addSample(&mimirpb.Sample{TimestampMs: startTs}, labels) } +// handleHistogramStartTime similar to the method above but for native histograms.. +func (c *MimirConverter) handleHistogramStartTime(startTs, sampleTs int64, ts *mimirpb.TimeSeries, settings Settings) { + if !settings.EnableCreatedTimestampZeroIngestion { + return + } + // We want to ignore the write in three cases. + // - We've seen samples with the start timestamp set to epoch meaning it wasn't set by the sender so we skip those. + // - If StartTimestamp equals Timestamp ist means we don't know at which time the metric restarted according to the spec. + // - StartTimestamp can never be greater than the sample timestamp. + if startTs <= 0 || startTs == sampleTs || startTs > sampleTs { + return + } + + // The difference between the start and the actual timestamp is more than a reasonable time, so we skip this sample. + if sampleTs-startTs > validIntervalForStartTimestamps { + return + } + + ts.Histograms = append(ts.Histograms, mimirpb.Histogram{Timestamp: startTs}) +} + // addResourceTargetInfo converts the resource to the target info metric. func addResourceTargetInfo(resource pcommon.Resource, settings Settings, timestamp pcommon.Timestamp, converter *MimirConverter) { if settings.DisableTargetInfo || timestamp == 0 { diff --git a/pkg/distributor/otlp/histograms_generated.go b/pkg/distributor/otlp/histograms_generated.go index 8d01c769e16..6029b57ac4d 100644 --- a/pkg/distributor/otlp/histograms_generated.go +++ b/pkg/distributor/otlp/histograms_generated.go @@ -46,6 +46,9 @@ func (c *MimirConverter) addExponentialHistogramDataPoints(ctx context.Context, } pt := dataPoints.At(x) + timestamp := convertTimeStamp(pt.Timestamp()) + startTimestampNs := pt.StartTimestamp() + startTimestampMs := convertTimeStamp(startTimestampNs) histogram, ws, err := exponentialToNativeHistogram(pt) annots.Merge(ws) @@ -63,6 +66,8 @@ func (c *MimirConverter) addExponentialHistogramDataPoints(ctx context.Context, promName, ) ts, _ := c.getOrCreateTimeSeries(lbls) + + c.handleHistogramStartTime(startTimestampMs, timestamp, ts, settings) ts.Histograms = append(ts.Histograms, histogram) exemplars, err := getPromExemplars[pmetric.ExponentialHistogramDataPoint](ctx, &c.everyN, pt) diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go index 116ded55f2a..abec70ff763 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/helper.go @@ -606,6 +606,27 @@ func (c *PrometheusConverter) handleStartTime(startTs, ts int64, labels []prompb c.addSample(&prompb.Sample{Timestamp: startTs}, labels) } +// handleHistogramStartTime similar to the method above but for native histograms.. +func (c *PrometheusConverter) handleHistogramStartTime(startTs, sampleTs int64, ts *prompb.TimeSeries, settings Settings) { + if !settings.EnableCreatedTimestampZeroIngestion { + return + } + // We want to ignore the write in three cases. + // - We've seen samples with the start timestamp set to epoch meaning it wasn't set by the sender so we skip those. + // - If StartTimestamp equals Timestamp ist means we don't know at which time the metric restarted according to the spec. + // - StartTimestamp can never be greater than the sample timestamp. + if startTs <= 0 || startTs == sampleTs || startTs > sampleTs { + return + } + + // The difference between the start and the actual timestamp is more than a reasonable time, so we skip this sample. + if sampleTs-startTs > validIntervalForStartTimestamps { + return + } + + ts.Histograms = append(ts.Histograms, prompb.Histogram{Timestamp: startTs}) +} + // addResourceTargetInfo converts the resource to the target info metric. func addResourceTargetInfo(resource pcommon.Resource, settings Settings, timestamp pcommon.Timestamp, converter *PrometheusConverter) { if settings.DisableTargetInfo || timestamp == 0 { diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go index b615c744d91..306d04530e4 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/otlptranslator/prometheusremotewrite/histograms.go @@ -43,6 +43,9 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Cont } pt := dataPoints.At(x) + timestamp := convertTimeStamp(pt.Timestamp()) + startTimestampNs := pt.StartTimestamp() + startTimestampMs := convertTimeStamp(startTimestampNs) histogram, ws, err := exponentialToNativeHistogram(pt) annots.Merge(ws) @@ -60,6 +63,8 @@ func (c *PrometheusConverter) addExponentialHistogramDataPoints(ctx context.Cont promName, ) ts, _ := c.getOrCreateTimeSeries(lbls) + + c.handleHistogramStartTime(startTimestampMs, timestamp, ts, settings) ts.Histograms = append(ts.Histograms, histogram) exemplars, err := getPromExemplars[pmetric.ExponentialHistogramDataPoint](ctx, &c.everyN, pt) diff --git a/vendor/github.com/prometheus/prometheus/tsdb/db.go b/vendor/github.com/prometheus/prometheus/tsdb/db.go index 7cbb78ca986..70cf68deeae 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/db.go @@ -2132,6 +2132,7 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) { overlapsOOO := overlapsClosedInterval(mint, maxt, db.head.MinOOOTime(), db.head.MaxOOOTime()) var headQuerier storage.Querier + inoMint := mint if maxt >= db.head.MinTime() || overlapsOOO { rh := NewRangeHead(db.head, mint, maxt) var err error @@ -2156,13 +2157,14 @@ func (db *DB) Querier(mint, maxt int64) (_ storage.Querier, err error) { if err != nil { return nil, fmt.Errorf("open block querier for head while getting new querier %s: %w", rh, err) } + inoMint = newMint } } if overlapsOOO { // We need to fetch from in-order and out-of-order chunks: wrap the headQuerier. isoState := db.head.oooIso.TrackReadAfter(db.lastGarbageCollectedMmapRef) - headQuerier = NewHeadAndOOOQuerier(mint, maxt, db.head, isoState, headQuerier) + headQuerier = NewHeadAndOOOQuerier(inoMint, mint, maxt, db.head, isoState, headQuerier) } if headQuerier != nil { @@ -2208,6 +2210,7 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer overlapsOOO := overlapsClosedInterval(mint, maxt, db.head.MinOOOTime(), db.head.MaxOOOTime()) var headQuerier storage.ChunkQuerier + inoMint := mint if maxt >= db.head.MinTime() || overlapsOOO { rh := NewRangeHead(db.head, mint, maxt) headQuerier, err = db.blockChunkQuerierFunc(rh, mint, maxt) @@ -2231,13 +2234,14 @@ func (db *DB) blockChunkQuerierForRange(mint, maxt int64) (_ []storage.ChunkQuer if err != nil { return nil, fmt.Errorf("open querier for head while getting new querier %s: %w", rh, err) } + inoMint = newMint } } if overlapsOOO { // We need to fetch from in-order and out-of-order chunks: wrap the headQuerier. isoState := db.head.oooIso.TrackReadAfter(db.lastGarbageCollectedMmapRef) - headQuerier = NewHeadAndOOOChunkQuerier(mint, maxt, db.head, isoState, headQuerier) + headQuerier = NewHeadAndOOOChunkQuerier(inoMint, mint, maxt, db.head, isoState, headQuerier) } if headQuerier != nil { diff --git a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go index 94cbf462918..81528b80bb7 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/ooo_head_read.go @@ -35,6 +35,7 @@ var _ IndexReader = &HeadAndOOOIndexReader{} type HeadAndOOOIndexReader struct { *headIndexReader // A reference to the headIndexReader so we can reuse as many interface implementation as possible. + inoMint int64 lastGarbageCollectedMmapRef chunks.ChunkDiskMapperRef } @@ -49,13 +50,13 @@ func (o mergedOOOChunks) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator return storage.ChainSampleIteratorFromIterables(iterator, o.chunkIterables) } -func NewHeadAndOOOIndexReader(head *Head, mint, maxt int64, lastGarbageCollectedMmapRef chunks.ChunkDiskMapperRef) *HeadAndOOOIndexReader { +func NewHeadAndOOOIndexReader(head *Head, inoMint, mint, maxt int64, lastGarbageCollectedMmapRef chunks.ChunkDiskMapperRef) *HeadAndOOOIndexReader { hr := &headIndexReader{ head: head, mint: mint, maxt: maxt, } - return &HeadAndOOOIndexReader{hr, lastGarbageCollectedMmapRef} + return &HeadAndOOOIndexReader{hr, inoMint, lastGarbageCollectedMmapRef} } func (oh *HeadAndOOOIndexReader) Series(ref storage.SeriesRef, builder *labels.ScratchBuilder, chks *[]chunks.Meta) error { @@ -76,9 +77,9 @@ func (oh *HeadAndOOOIndexReader) Series(ref storage.SeriesRef, builder *labels.S *chks = (*chks)[:0] if s.ooo != nil { - return getOOOSeriesChunks(s, oh.mint, oh.maxt, oh.lastGarbageCollectedMmapRef, 0, true, chks) + return getOOOSeriesChunks(s, oh.mint, oh.maxt, oh.lastGarbageCollectedMmapRef, 0, true, oh.inoMint, chks) } - *chks = appendSeriesChunks(s, oh.mint, oh.maxt, *chks) + *chks = appendSeriesChunks(s, oh.inoMint, oh.maxt, *chks) return nil } @@ -87,7 +88,7 @@ func (oh *HeadAndOOOIndexReader) Series(ref storage.SeriesRef, builder *labels.S // // maxMmapRef tells upto what max m-map chunk that we can consider. If it is non-0, then // the oooHeadChunk will not be considered. -func getOOOSeriesChunks(s *memSeries, mint, maxt int64, lastGarbageCollectedMmapRef, maxMmapRef chunks.ChunkDiskMapperRef, includeInOrder bool, chks *[]chunks.Meta) error { +func getOOOSeriesChunks(s *memSeries, mint, maxt int64, lastGarbageCollectedMmapRef, maxMmapRef chunks.ChunkDiskMapperRef, includeInOrder bool, inoMint int64, chks *[]chunks.Meta) error { tmpChks := make([]chunks.Meta, 0, len(s.ooo.oooMmappedChunks)) addChunk := func(minT, maxT int64, ref chunks.ChunkRef, chunk chunkenc.Chunk) { @@ -128,7 +129,7 @@ func getOOOSeriesChunks(s *memSeries, mint, maxt int64, lastGarbageCollectedMmap } if includeInOrder { - tmpChks = appendSeriesChunks(s, mint, maxt, tmpChks) + tmpChks = appendSeriesChunks(s, inoMint, maxt, tmpChks) } // There is nothing to do if we did not collect any chunk. @@ -492,7 +493,7 @@ func (ir *OOOCompactionHeadIndexReader) Series(ref storage.SeriesRef, builder *l return nil } - return getOOOSeriesChunks(s, ir.ch.mint, ir.ch.maxt, 0, ir.ch.lastMmapRef, false, chks) + return getOOOSeriesChunks(s, ir.ch.mint, ir.ch.maxt, 0, ir.ch.lastMmapRef, false, 0, chks) } func (ir *OOOCompactionHeadIndexReader) SortedLabelValues(_ context.Context, name string, matchers ...*labels.Matcher) ([]string, error) { @@ -532,7 +533,7 @@ type HeadAndOOOQuerier struct { querier storage.Querier // Used for LabelNames, LabelValues, but may be nil if head was truncated in the mean time, in which case we ignore it and not close it in the end. } -func NewHeadAndOOOQuerier(mint, maxt int64, head *Head, oooIsoState *oooIsolationState, querier storage.Querier) storage.Querier { +func NewHeadAndOOOQuerier(inoMint, mint, maxt int64, head *Head, oooIsoState *oooIsolationState, querier storage.Querier) storage.Querier { cr := &headChunkReader{ head: head, mint: mint, @@ -543,7 +544,7 @@ func NewHeadAndOOOQuerier(mint, maxt int64, head *Head, oooIsoState *oooIsolatio mint: mint, maxt: maxt, head: head, - index: NewHeadAndOOOIndexReader(head, mint, maxt, oooIsoState.minRef), + index: NewHeadAndOOOIndexReader(head, inoMint, mint, maxt, oooIsoState.minRef), chunkr: NewHeadAndOOOChunkReader(head, mint, maxt, cr, oooIsoState, 0), querier: querier, } @@ -584,7 +585,7 @@ type HeadAndOOOChunkQuerier struct { querier storage.ChunkQuerier } -func NewHeadAndOOOChunkQuerier(mint, maxt int64, head *Head, oooIsoState *oooIsolationState, querier storage.ChunkQuerier) storage.ChunkQuerier { +func NewHeadAndOOOChunkQuerier(inoMint, mint, maxt int64, head *Head, oooIsoState *oooIsolationState, querier storage.ChunkQuerier) storage.ChunkQuerier { cr := &headChunkReader{ head: head, mint: mint, @@ -595,7 +596,7 @@ func NewHeadAndOOOChunkQuerier(mint, maxt int64, head *Head, oooIsoState *oooIso mint: mint, maxt: maxt, head: head, - index: NewHeadAndOOOIndexReader(head, mint, maxt, oooIsoState.minRef), + index: NewHeadAndOOOIndexReader(head, inoMint, mint, maxt, oooIsoState.minRef), chunkr: NewHeadAndOOOChunkReader(head, mint, maxt, cr, oooIsoState, 0), querier: querier, } diff --git a/vendor/modules.txt b/vendor/modules.txt index 31aca2a49d4..cc71cb2110e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1001,7 +1001,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20240917104844-e4312d119ad6 +# github.com/prometheus/prometheus v1.99.0 => github.com/grafana/mimir-prometheus v0.0.0-20240920170818-87bc81f0005c ## explicit; go 1.22.0 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -1653,7 +1653,7 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk sigs.k8s.io/yaml sigs.k8s.io/yaml/goyaml.v2 sigs.k8s.io/yaml/goyaml.v3 -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240917104844-e4312d119ad6 +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20240920170818-87bc81f0005c # github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.19.1 # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094