From 5c6d4ae3ca88d6efa9424f8c456f121512482519 Mon Sep 17 00:00:00 2001 From: Owen Diehl Date: Tue, 4 Jun 2024 11:55:41 -0700 Subject: [PATCH] metric integration Signed-off-by: Owen Diehl --- pkg/storage/bloom/v1/bloom_tokenizer.go | 22 +++++++++++++++++----- pkg/storage/bloom/v1/metrics.go | 21 ++++++--------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/pkg/storage/bloom/v1/bloom_tokenizer.go b/pkg/storage/bloom/v1/bloom_tokenizer.go index 06e70b428436..7d2ba41b7f49 100644 --- a/pkg/storage/bloom/v1/bloom_tokenizer.go +++ b/pkg/storage/bloom/v1/bloom_tokenizer.go @@ -137,10 +137,7 @@ func (bt *BloomTokenizer) Populate( // If a bloom is full, the chunk wasn't completely added // so we'll submit this bloom, start a new one, and continue indexing if full { - ch <- &BloomCreation{ - Bloom: bloom, - SourceBytesAdded: bytesAdded, - } + bt.sendBloom(ch, bloom, bytesAdded) // start a new bloom + reset bytesAdded counter bytesAdded = 0 @@ -158,11 +155,26 @@ func (bt *BloomTokenizer) Populate( } // Send the last bloom + bt.sendBloom(ch, bloom, bytesAdded) + close(ch) +} + +func (bt *BloomTokenizer) sendBloom( + ch chan<- *BloomCreation, + bloom *Bloom, + bytesAdded int, +) { + fillRatio := bloom.ScalableBloomFilter.FillRatio() + bt.metrics.hammingWeightRatio.Observe(fillRatio) + bt.metrics.estimatedCount.Observe( + float64(estimatedCount(bloom.ScalableBloomFilter.Capacity(), fillRatio)), + ) + bt.metrics.bloomSize.Observe(float64(bloom.ScalableBloomFilter.Capacity() / eightBits)) + bt.metrics.bloomsTotal.Inc() ch <- &BloomCreation{ Bloom: bloom, SourceBytesAdded: bytesAdded, } - close(ch) } // addChunkToBloom adds the tokens from the given chunk to the given bloom. diff --git a/pkg/storage/bloom/v1/metrics.go b/pkg/storage/bloom/v1/metrics.go index 26c4542a008f..cb94373185d8 100644 --- a/pkg/storage/bloom/v1/metrics.go +++ b/pkg/storage/bloom/v1/metrics.go @@ -9,11 +9,10 @@ import ( type Metrics struct { // writes - bloomsTotal *prometheus.CounterVec // number of blooms created - sbfCreationTime *prometheus.CounterVec // time spent creating sbfs - bloomSize prometheus.Histogram // size of the bloom filter in bytes - hammingWeightRatio prometheus.Histogram // ratio of the hamming weight of the bloom filter to the number of bits in the bloom filter - estimatedCount prometheus.Histogram // estimated number of elements in the bloom filter + bloomsTotal prometheus.Counter // number of blooms created + bloomSize prometheus.Histogram // size of the bloom filter in bytes + hammingWeightRatio prometheus.Histogram // ratio of the hamming weight of the bloom filter to the number of bits in the bloom filter + estimatedCount prometheus.Histogram // estimated number of elements in the bloom filter chunksIndexed *prometheus.CounterVec chunksPerSeries prometheus.Histogram blockSeriesIterated prometheus.Counter @@ -51,9 +50,6 @@ const ( skipReasonErr = "err" skipReasonOOB = "out_of_bounds" - bloomCreationTypeIndexed = "indexed" - bloomCreationTypeSkipped = "skipped" - recorderRequested = "requested" recorderFound = "found" recorderSkipped = "skipped" @@ -63,16 +59,11 @@ const ( func NewMetrics(r prometheus.Registerer) *Metrics { return &Metrics{ - bloomsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ + bloomsTotal: promauto.With(r).NewCounter(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "blooms_created_total", Help: "Number of blooms created", - }, []string{"type"}), - sbfCreationTime: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: constants.Loki, - Name: "bloom_creation_time_total", - Help: "Time spent creating scalable bloom filters", - }, []string{"type"}), + }), bloomSize: promauto.With(r).NewHistogram(prometheus.HistogramOpts{ Namespace: constants.Loki, Name: "bloom_size",