From acc0b815829b2de72cdc96a61acc1b6d0b180a1c Mon Sep 17 00:00:00 2001 From: Tyler Reid Date: Mon, 4 Oct 2021 16:55:23 -0500 Subject: [PATCH 1/4] Move lifecycler metrics to a new class from global Signed-off-by: Tyler Reid --- ring/lifecycler.go | 39 ++++++++++--------------------------- ring/lifecycler_metrics.go | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 29 deletions(-) create mode 100644 ring/lifecycler_metrics.go diff --git a/ring/lifecycler.go b/ring/lifecycler.go index aeb811eba..9ea3fa9db 100644 --- a/ring/lifecycler.go +++ b/ring/lifecycler.go @@ -14,7 +14,6 @@ import ( "github.com/pkg/errors" perrors "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/atomic" "github.com/grafana/dskit/flagext" @@ -23,26 +22,6 @@ import ( dstime "github.com/grafana/dskit/time" ) -var ( - consulHeartbeats = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "member_consul_heartbeats_total", - Help: "The total number of heartbeats sent to consul.", - }, []string{"name"}) - tokensOwned = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "member_ring_tokens_owned", - Help: "The number of tokens owned in the ring.", - }, []string{"name"}) - tokensToOwn = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "member_ring_tokens_to_own", - Help: "The number of tokens to own in the ring.", - }, []string{"name"}) - shutdownDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Name: "shutdown_duration_seconds", - Help: "Duration (in seconds) of shutdown procedure (ie transfer or flush).", - Buckets: prometheus.ExponentialBuckets(10, 2, 8), // Biggest bucket is 10*2^(9-1) = 2560, or 42 mins. - }, []string{"op", "status", "name"}) -) - // LifecyclerConfig is the config to build a Lifecycler. type LifecyclerConfig struct { RingConfig Config `yaml:"ring"` @@ -145,7 +124,8 @@ type Lifecycler struct { healthyInstancesCount int zonesCount int - logger log.Logger + lifecyclerMetrics *LifecyclerMetrics + logger log.Logger } // NewLifecycler creates new Lifecycler. It must be started via StartAsync. @@ -191,10 +171,11 @@ func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, ringNa Zone: zone, actorChan: make(chan func()), state: PENDING, + lifecyclerMetrics: NewLifecyclerMetrics(ringName, reg), logger: logger, } - tokensToOwn.WithLabelValues(l.RingName).Set(float64(cfg.NumTokens)) + l.lifecyclerMetrics.tokensToOwn.Set(float64(cfg.NumTokens)) l.BasicService = services. NewBasicService(nil, l.loop, l.stopping). @@ -322,7 +303,7 @@ func (i *Lifecycler) getTokens() Tokens { } func (i *Lifecycler) setTokens(tokens Tokens) { - tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) + i.lifecyclerMetrics.tokensOwned.Set(float64(len(tokens))) i.stateMtx.Lock() defer i.stateMtx.Unlock() @@ -473,7 +454,7 @@ func (i *Lifecycler) loop(ctx context.Context) error { } case <-heartbeatTickerChan: - consulHeartbeats.WithLabelValues(i.RingName).Inc() + i.lifecyclerMetrics.consulHeartbeats.Inc() if err := i.updateConsul(context.Background()); err != nil { level.Error(i.logger).Log("msg", "failed to write to the KV store, sleeping", "ring", i.RingName, "err", err) } @@ -520,7 +501,7 @@ heartbeatLoop: for { select { case <-heartbeatTickerChan: - consulHeartbeats.WithLabelValues(i.RingName).Inc() + i.lifecyclerMetrics.consulHeartbeats.Inc() if err := i.updateConsul(context.Background()); err != nil { level.Error(i.logger).Log("msg", "failed to write to the KV store, sleeping", "ring", i.RingName, "err", err) } @@ -851,17 +832,17 @@ func (i *Lifecycler) processShutdown(ctx context.Context) { level.Info(i.logger).Log("msg", "transfers are disabled") } else { level.Error(i.logger).Log("msg", "failed to transfer chunks to another instance", "ring", i.RingName, "err", err) - shutdownDuration.WithLabelValues("transfer", "fail", i.RingName).Observe(time.Since(transferStart).Seconds()) + i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "fail", i.RingName).Observe(time.Since(transferStart).Seconds()) } } else { flushRequired = false - shutdownDuration.WithLabelValues("transfer", "success", i.RingName).Observe(time.Since(transferStart).Seconds()) + i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "success", i.RingName).Observe(time.Since(transferStart).Seconds()) } if flushRequired { flushStart := time.Now() i.flushTransferer.Flush() - shutdownDuration.WithLabelValues("flush", "success", i.RingName).Observe(time.Since(flushStart).Seconds()) + i.lifecyclerMetrics.shutdownDuration.WithLabelValues("flush", "success", i.RingName).Observe(time.Since(flushStart).Seconds()) } // Sleep so the shutdownDuration metric can be collected. diff --git a/ring/lifecycler_metrics.go b/ring/lifecycler_metrics.go new file mode 100644 index 000000000..d24483e3d --- /dev/null +++ b/ring/lifecycler_metrics.go @@ -0,0 +1,40 @@ +package ring + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +type LifecyclerMetrics struct { + consulHeartbeats prometheus.Counter + tokensOwned prometheus.Gauge + tokensToOwn prometheus.Gauge + shutdownDuration *prometheus.HistogramVec +} + +func NewLifecyclerMetrics(ringName string, reg prometheus.Registerer) *LifecyclerMetrics { + return &LifecyclerMetrics{ + consulHeartbeats: promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Name: "member_consul_heartbeats_total", + Help: "The total number of heartbeats sent to consul.", + ConstLabels: prometheus.Labels{"name": ringName}, + }), + tokensOwned: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "member_ring_tokens_owned", + Help: "The number of tokens owned in the ring.", + ConstLabels: prometheus.Labels{"name": ringName}, + }), + tokensToOwn: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "member_ring_tokens_to_own", + Help: "The number of tokens to own in the ring.", + ConstLabels: prometheus.Labels{"name": ringName}, + }), + shutdownDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ + Name: "shutdown_duration_seconds", + Help: "Duration (in seconds) of shutdown procedure (ie transfer or flush).", + Buckets: prometheus.ExponentialBuckets(10, 2, 8), // Biggest bucket is 10*2^(9-1) = 2560, or 42 mins. + ConstLabels: prometheus.Labels{"name": ringName}, + }, []string{"op", "status", "name"}), + } + +} From 80acac39018cf4bf3ed0e8d773880e007ca2cd3b Mon Sep 17 00:00:00 2001 From: Tyler Reid Date: Tue, 5 Oct 2021 16:11:21 -0500 Subject: [PATCH 2/4] Refactor global metrics in ring to use a gauge and a ticker to update the gauges --- ring/lifecycler_metrics.go | 14 ++-- ring/ring.go | 139 ++++++++++++++++--------------------- 2 files changed, 65 insertions(+), 88 deletions(-) diff --git a/ring/lifecycler_metrics.go b/ring/lifecycler_metrics.go index d24483e3d..978b2a641 100644 --- a/ring/lifecycler_metrics.go +++ b/ring/lifecycler_metrics.go @@ -20,19 +20,19 @@ func NewLifecyclerMetrics(ringName string, reg prometheus.Registerer) *Lifecycle ConstLabels: prometheus.Labels{"name": ringName}, }), tokensOwned: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "member_ring_tokens_owned", - Help: "The number of tokens owned in the ring.", + Name: "member_ring_tokens_owned", + Help: "The number of tokens owned in the ring.", ConstLabels: prometheus.Labels{"name": ringName}, }), tokensToOwn: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "member_ring_tokens_to_own", - Help: "The number of tokens to own in the ring.", + Name: "member_ring_tokens_to_own", + Help: "The number of tokens to own in the ring.", ConstLabels: prometheus.Labels{"name": ringName}, }), shutdownDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ - Name: "shutdown_duration_seconds", - Help: "Duration (in seconds) of shutdown procedure (ie transfer or flush).", - Buckets: prometheus.ExponentialBuckets(10, 2, 8), // Biggest bucket is 10*2^(9-1) = 2560, or 42 mins. + Name: "shutdown_duration_seconds", + Help: "Duration (in seconds) of shutdown procedure (ie transfer or flush).", + Buckets: prometheus.ExponentialBuckets(10, 2, 8), // Biggest bucket is 10*2^(9-1) = 2560, or 42 mins. ConstLabels: prometheus.Labels{"name": ringName}, }, []string{"op", "status", "name"}), } diff --git a/ring/ring.go b/ring/ring.go index 4579f87c8..999589dc7 100644 --- a/ring/ring.go +++ b/ring/ring.go @@ -15,6 +15,7 @@ import ( "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/grafana/dskit/kv" shardUtil "github.com/grafana/dskit/ring/shard" @@ -47,7 +48,6 @@ const ( // ReadRing represents the read interface to the ring. type ReadRing interface { - prometheus.Collector // Get returns n (or more) instances which form the replicas for the given key. // bufDescs, bufHosts and bufZones are slices to be overwritten for the return value @@ -193,11 +193,12 @@ type Ring struct { // If set to nil, no caching is done (used by tests, and subrings). shuffledSubringCache map[subringCacheKey]*Ring - memberOwnershipDesc *prometheus.Desc - numMembersDesc *prometheus.Desc - totalTokensDesc *prometheus.Desc - numTokensDesc *prometheus.Desc - oldestTimestampDesc *prometheus.Desc + memberOwnershipDesc *prometheus.GaugeVec + numMembersDesc *prometheus.GaugeVec + totalTokensDesc prometheus.Gauge + numTokensDesc *prometheus.GaugeVec + oldestTimestampDesc *prometheus.GaugeVec + metricsUpdateTicker *time.Ticker logger log.Logger } @@ -221,10 +222,10 @@ func New(cfg Config, name, key string, logger log.Logger, reg prometheus.Registe return nil, err } - return NewWithStoreClientAndStrategy(cfg, name, key, store, NewDefaultReplicationStrategy(), logger) + return NewWithStoreClientAndStrategy(cfg, name, key, store, NewDefaultReplicationStrategy(), reg, logger) } -func NewWithStoreClientAndStrategy(cfg Config, name, key string, store kv.Client, strategy ReplicationStrategy, logger log.Logger) (*Ring, error) { +func NewWithStoreClientAndStrategy(cfg Config, name, key string, store kv.Client, strategy ReplicationStrategy, reg prometheus.Registerer, logger log.Logger) (*Ring, error) { if cfg.ReplicationFactor <= 0 { return nil, fmt.Errorf("ReplicationFactor must be greater than zero: %d", cfg.ReplicationFactor) } @@ -236,40 +237,34 @@ func NewWithStoreClientAndStrategy(cfg Config, name, key string, store kv.Client strategy: strategy, ringDesc: &Desc{}, shuffledSubringCache: map[subringCacheKey]*Ring{}, - memberOwnershipDesc: prometheus.NewDesc( - "ring_member_ownership_percent", - "The percent ownership of the ring by member", - []string{"member"}, - map[string]string{"name": name}, - ), - numMembersDesc: prometheus.NewDesc( - "ring_members", - "Number of members in the ring", - []string{"state"}, - map[string]string{"name": name}, - ), - totalTokensDesc: prometheus.NewDesc( - "ring_tokens_total", - "Number of tokens in the ring", - nil, - map[string]string{"name": name}, - ), - numTokensDesc: prometheus.NewDesc( - "ring_tokens_owned", - "The number of tokens in the ring owned by the member", - []string{"member"}, - map[string]string{"name": name}, - ), - oldestTimestampDesc: prometheus.NewDesc( - "ring_oldest_member_timestamp", - "Timestamp of the oldest member in the ring.", - []string{"state"}, - map[string]string{"name": name}, - ), + memberOwnershipDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "ring_member_ownership_percent", + Help: "The percent ownership of the ring by member", + ConstLabels: map[string]string{"name": name}}, + []string{"member"}), + numMembersDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "ring_members", + Help: "Number of members in the ring", + ConstLabels: map[string]string{"name": name}}, + []string{"state"}), + totalTokensDesc: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "ring_tokens_total", + Help: "Number of tokens in the ring", + ConstLabels: map[string]string{"name": name}}), + numTokensDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "ring_tokens_owned", + Help: "The number of tokens in the ring owned by the member", + ConstLabels: map[string]string{"name": name}}, + []string{"member"}), + oldestTimestampDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Name: "ring_oldest_member_timestamp", + Help: "Timestamp of the oldest member in the ring.", + ConstLabels: map[string]string{"name": name}}, + []string{"state"}), logger: logger, } - r.Service = services.NewBasicService(r.starting, r.loop, nil).WithName(fmt.Sprintf("%s ring client", name)) + r.Service = services.NewBasicService(r.starting, r.loop, r.stopping).WithName(fmt.Sprintf("%s ring client", name)) return r, nil } @@ -287,6 +282,14 @@ func (r *Ring) starting(ctx context.Context) error { } r.updateRingState(value.(*Desc)) + + // Start metrics update ticker, and give it a function to update the ring metrics. + r.metricsUpdateTicker = time.NewTicker(10 * time.Second) + go func() { + for range r.metricsUpdateTicker.C { + r.updateRingMetrics() + } + }() return nil } @@ -303,6 +306,14 @@ func (r *Ring) loop(ctx context.Context) error { return nil } +func (r *Ring) stopping(_ error) error { + // Stop Metrics ticker. + if r.metricsUpdateTicker != nil { + r.metricsUpdateTicker.Stop() + } + return nil +} + func (r *Ring) updateRingState(ringDesc *Desc) { r.mtx.RLock() prevRing := r.ringDesc @@ -523,15 +534,6 @@ func (r *Ring) GetReplicationSetForOperation(op Operation) (ReplicationSet, erro }, nil } -// Describe implements prometheus.Collector. -func (r *Ring) Describe(ch chan<- *prometheus.Desc) { - ch <- r.memberOwnershipDesc - ch <- r.numMembersDesc - ch <- r.totalTokensDesc - ch <- r.oldestTimestampDesc - ch <- r.numTokensDesc -} - // countTokens returns the number of tokens and tokens within the range for each instance. // The ring read lock must be already taken when calling this function. func (r *Ring) countTokens() (map[string]uint32, map[string]uint32) { @@ -563,25 +565,15 @@ func (r *Ring) countTokens() (map[string]uint32, map[string]uint32) { return numTokens, owned } -// Collect implements prometheus.Collector. -func (r *Ring) Collect(ch chan<- prometheus.Metric) { +// UpdateMetrics updates ring metrics, called by a Ticker in the ring at a set 10 second interval. +func (r *Ring) updateRingMetrics() { r.mtx.RLock() defer r.mtx.RUnlock() numTokens, ownedRange := r.countTokens() for id, totalOwned := range ownedRange { - ch <- prometheus.MustNewConstMetric( - r.memberOwnershipDesc, - prometheus.GaugeValue, - float64(totalOwned)/float64(math.MaxUint32), - id, - ) - ch <- prometheus.MustNewConstMetric( - r.numTokensDesc, - prometheus.GaugeValue, - float64(numTokens[id]), - id, - ) + r.memberOwnershipDesc.WithLabelValues(id).Set(float64(totalOwned) / float64(math.MaxUint32)) + r.numMembersDesc.WithLabelValues(id).Set(float64(numTokens[id])) } numByState := map[string]int{} @@ -605,27 +597,12 @@ func (r *Ring) Collect(ch chan<- prometheus.Metric) { } for state, count := range numByState { - ch <- prometheus.MustNewConstMetric( - r.numMembersDesc, - prometheus.GaugeValue, - float64(count), - state, - ) + r.numMembersDesc.WithLabelValues(state).Set(float64(count)) } for state, timestamp := range oldestTimestampByState { - ch <- prometheus.MustNewConstMetric( - r.oldestTimestampDesc, - prometheus.GaugeValue, - float64(timestamp), - state, - ) - } - - ch <- prometheus.MustNewConstMetric( - r.totalTokensDesc, - prometheus.GaugeValue, - float64(len(r.ringTokens)), - ) + r.oldestTimestampDesc.WithLabelValues(state).Set(float64(timestamp)) + } + r.totalTokensDesc.Set(float64(len(r.ringTokens))) } // ShuffleShard returns a subring for the provided identifier (eg. a tenant ID) From 6ca16e2a50f2c21a2431c005125f95afcb5ee701 Mon Sep 17 00:00:00 2001 From: Tyler Reid Date: Wed, 6 Oct 2021 12:08:30 -0500 Subject: [PATCH 3/4] Review cleanup --- ring/lifecycler.go | 6 +++--- ring/lifecycler_metrics.go | 2 +- ring/ring.go | 34 +++++++++++++++++----------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ring/lifecycler.go b/ring/lifecycler.go index 9ea3fa9db..51bfe0f15 100644 --- a/ring/lifecycler.go +++ b/ring/lifecycler.go @@ -832,17 +832,17 @@ func (i *Lifecycler) processShutdown(ctx context.Context) { level.Info(i.logger).Log("msg", "transfers are disabled") } else { level.Error(i.logger).Log("msg", "failed to transfer chunks to another instance", "ring", i.RingName, "err", err) - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "fail", i.RingName).Observe(time.Since(transferStart).Seconds()) + i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "fail").Observe(time.Since(transferStart).Seconds()) } } else { flushRequired = false - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "success", i.RingName).Observe(time.Since(transferStart).Seconds()) + i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "success").Observe(time.Since(transferStart).Seconds()) } if flushRequired { flushStart := time.Now() i.flushTransferer.Flush() - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("flush", "success", i.RingName).Observe(time.Since(flushStart).Seconds()) + i.lifecyclerMetrics.shutdownDuration.WithLabelValues("flush", "success").Observe(time.Since(flushStart).Seconds()) } // Sleep so the shutdownDuration metric can be collected. diff --git a/ring/lifecycler_metrics.go b/ring/lifecycler_metrics.go index 978b2a641..422a564c1 100644 --- a/ring/lifecycler_metrics.go +++ b/ring/lifecycler_metrics.go @@ -34,7 +34,7 @@ func NewLifecyclerMetrics(ringName string, reg prometheus.Registerer) *Lifecycle Help: "Duration (in seconds) of shutdown procedure (ie transfer or flush).", Buckets: prometheus.ExponentialBuckets(10, 2, 8), // Biggest bucket is 10*2^(9-1) = 2560, or 42 mins. ConstLabels: prometheus.Labels{"name": ringName}, - }, []string{"op", "status", "name"}), + }, []string{"op", "status"}), } } diff --git a/ring/ring.go b/ring/ring.go index 999589dc7..f1ad503c2 100644 --- a/ring/ring.go +++ b/ring/ring.go @@ -193,12 +193,12 @@ type Ring struct { // If set to nil, no caching is done (used by tests, and subrings). shuffledSubringCache map[subringCacheKey]*Ring - memberOwnershipDesc *prometheus.GaugeVec - numMembersDesc *prometheus.GaugeVec - totalTokensDesc prometheus.Gauge - numTokensDesc *prometheus.GaugeVec - oldestTimestampDesc *prometheus.GaugeVec - metricsUpdateTicker *time.Ticker + memberOwnershipGaugeVec *prometheus.GaugeVec + numMembersGaugeVec *prometheus.GaugeVec + totalTokensGauge prometheus.Gauge + numTokensGaugeVec *prometheus.GaugeVec + oldestTimestampGaugeVec *prometheus.GaugeVec + metricsUpdateTicker *time.Ticker logger log.Logger } @@ -237,26 +237,26 @@ func NewWithStoreClientAndStrategy(cfg Config, name, key string, store kv.Client strategy: strategy, ringDesc: &Desc{}, shuffledSubringCache: map[subringCacheKey]*Ring{}, - memberOwnershipDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + memberOwnershipGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "ring_member_ownership_percent", Help: "The percent ownership of the ring by member", ConstLabels: map[string]string{"name": name}}, []string{"member"}), - numMembersDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + numMembersGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "ring_members", Help: "Number of members in the ring", ConstLabels: map[string]string{"name": name}}, []string{"state"}), - totalTokensDesc: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + totalTokensGauge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ Name: "ring_tokens_total", Help: "Number of tokens in the ring", ConstLabels: map[string]string{"name": name}}), - numTokensDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + numTokensGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "ring_tokens_owned", Help: "The number of tokens in the ring owned by the member", ConstLabels: map[string]string{"name": name}}, []string{"member"}), - oldestTimestampDesc: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + oldestTimestampGaugeVec: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ Name: "ring_oldest_member_timestamp", Help: "Timestamp of the oldest member in the ring.", ConstLabels: map[string]string{"name": name}}, @@ -565,15 +565,15 @@ func (r *Ring) countTokens() (map[string]uint32, map[string]uint32) { return numTokens, owned } -// UpdateMetrics updates ring metrics, called by a Ticker in the ring at a set 10 second interval. +// updateRingMetrics updates ring metrics. func (r *Ring) updateRingMetrics() { r.mtx.RLock() defer r.mtx.RUnlock() numTokens, ownedRange := r.countTokens() for id, totalOwned := range ownedRange { - r.memberOwnershipDesc.WithLabelValues(id).Set(float64(totalOwned) / float64(math.MaxUint32)) - r.numMembersDesc.WithLabelValues(id).Set(float64(numTokens[id])) + r.memberOwnershipGaugeVec.WithLabelValues(id).Set(float64(totalOwned) / float64(math.MaxUint32)) + r.numMembersGaugeVec.WithLabelValues(id).Set(float64(numTokens[id])) } numByState := map[string]int{} @@ -597,12 +597,12 @@ func (r *Ring) updateRingMetrics() { } for state, count := range numByState { - r.numMembersDesc.WithLabelValues(state).Set(float64(count)) + r.numMembersGaugeVec.WithLabelValues(state).Set(float64(count)) } for state, timestamp := range oldestTimestampByState { - r.oldestTimestampDesc.WithLabelValues(state).Set(float64(timestamp)) + r.oldestTimestampGaugeVec.WithLabelValues(state).Set(float64(timestamp)) } - r.totalTokensDesc.Set(float64(len(r.ringTokens))) + r.totalTokensGauge.Set(float64(len(r.ringTokens))) } // ShuffleShard returns a subring for the provided identifier (eg. a tenant ID) From 3c98e4adef4fc1b7f5cb83c3f101a75bf83d229d Mon Sep 17 00:00:00 2001 From: Tyler Reid Date: Thu, 7 Oct 2021 09:14:26 -0500 Subject: [PATCH 4/4] Fix to use numTokensGauge correctly instead of numMembersGauge --- ring/ring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ring/ring.go b/ring/ring.go index f1ad503c2..c5a23dc49 100644 --- a/ring/ring.go +++ b/ring/ring.go @@ -573,7 +573,7 @@ func (r *Ring) updateRingMetrics() { numTokens, ownedRange := r.countTokens() for id, totalOwned := range ownedRange { r.memberOwnershipGaugeVec.WithLabelValues(id).Set(float64(totalOwned) / float64(math.MaxUint32)) - r.numMembersGaugeVec.WithLabelValues(id).Set(float64(numTokens[id])) + r.numTokensGaugeVec.WithLabelValues(id).Set(float64(numTokens[id])) } numByState := map[string]int{}