From aa53859f54f09ed7579c82bd556cfb4df601f777 Mon Sep 17 00:00:00 2001 From: Wonki Kim Date: Tue, 30 May 2023 16:41:43 +0900 Subject: [PATCH] Fix double-counting bug in http_request_duration metric (#6399) * fix double-counting bug in http_request_duration metric Signed-off-by: 4orty * Update Changelog Signed-off-by: 4orty --------- Signed-off-by: 4orty --- CHANGELOG.md | 1 + pkg/extprom/http/instrument_server.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f0c57d30bc..1b84e3f8671 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6330](https://github.com/thanos-io/thanos/pull/6330) Store: Fix inconsistent error for series limits. - [#6342](https://github.com/thanos-io/thanos/pull/6342) Cache/Redis: Upgrade `rueidis` to v1.0.2 to to improve error handling while shrinking a redis cluster. - [#6325](https://github.com/thanos-io/thanos/pull/6325) Store: return gRPC resource exhausted error for byte limiter. +- [#6399](https://github.com/thanos-io/thanos/pull/6399) *: Fix double-counting bug in http_request_duration metric ### Changed - [#6168](https://github.com/thanos-io/thanos/pull/6168) Receiver: Make ketama hashring fail early when configured with number of nodes lower than the replication factor. diff --git a/pkg/extprom/http/instrument_server.go b/pkg/extprom/http/instrument_server.go index 6c0e6262674..745a5c8af7e 100644 --- a/pkg/extprom/http/instrument_server.go +++ b/pkg/extprom/http/instrument_server.go @@ -76,7 +76,7 @@ func httpInstrumentationHandler(baseLabels prometheus.Labels, metrics *defaultMe requestLabels := prometheus.Labels{"code": wd.Status(), "method": strings.ToLower(r.Method)} observer := metrics.requestDuration.MustCurryWith(baseLabels).With(requestLabels) - observer.Observe(time.Since(now).Seconds()) + requestDuration := time.Since(now).Seconds() // If we find a tracingID we'll expose it as Exemplar. var ( @@ -104,11 +104,13 @@ func httpInstrumentationHandler(baseLabels prometheus.Labels, metrics *defaultMe if traceID != "" { observer.(prometheus.ExemplarObserver).ObserveWithExemplar( - time.Since(now).Seconds(), + requestDuration, prometheus.Labels{ "traceID": traceID, }, ) + } else { + observer.Observe(requestDuration) } }), ),