From c388813a1051bb6c5a98aca0ed441a79145b3011 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 31 Jan 2025 14:19:40 +0900 Subject: [PATCH] Add a too_many_tenants label to cortex_rejected_queries_total metric Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + pkg/frontend/transport/handler.go | 10 ++++++++-- pkg/frontend/transport/handler_test.go | 7 ++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7744fdb638..3de4ff5f2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master / unreleased * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 +* [ENHANCEMENT] Query Frontend: Add a `too_many_tenants` reason label value to `cortex_rejected_queries_total` metric to track the rejected query count due to the # of tenant limits. #6569 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517 ## 1.19.0 in progress diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index b703778ca0..d5ed3f353a 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -45,6 +45,7 @@ var ( ) const ( + reasonTooManyTenants = "too_many_tenants" reasonRequestBodySizeExceeded = "request_body_size_exceeded" reasonResponseBodySizeExceeded = "response_body_size_exceeded" reasonTooManyRequests = "too_many_requests" @@ -191,16 +192,21 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + userID := tenant.JoinTenantIDs(tenantIDs) + if f.tenantFederationCfg.Enabled { maxTenant := f.tenantFederationCfg.MaxTenant if maxTenant > 0 && len(tenantIDs) > maxTenant { + source := tripperware.GetSource(r.Header.Get("User-Agent")) + if f.cfg.QueryStatsEnabled { + // when query stats enabled, track too many tenants reason + f.rejectedQueries.WithLabelValues(reasonTooManyTenants, source, userID).Inc() + } http.Error(w, fmt.Errorf(errTooManyTenants, maxTenant, len(tenantIDs)).Error(), http.StatusBadRequest) return } } - userID := tenant.JoinTenantIDs(tenantIDs) - // Initialise the stats in the context and make sure it's propagated // down the request chain. if f.cfg.QueryStatsEnabled { diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index 92e0b59fd4..10a844ee8d 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -589,7 +589,7 @@ func Test_TenantFederation_MaxTenant(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - handler := NewHandler(HandlerConfig{}, test.cfg, roundTripper, log.NewNopLogger(), nil) + handler := NewHandler(HandlerConfig{QueryStatsEnabled: true}, test.cfg, roundTripper, log.NewNopLogger(), nil) handlerWithAuth := middleware.Merge(middleware.AuthenticateUser).Wrap(handler) req := httptest.NewRequest("GET", "http://fake", nil) @@ -604,6 +604,11 @@ func Test_TenantFederation_MaxTenant(t *testing.T) { if test.expectedErrMsg != "" { require.Contains(t, string(body), test.expectedErrMsg) + + if strings.Contains(test.expectedErrMsg, "too many tenants") { + v := promtest.ToFloat64(handler.rejectedQueries.WithLabelValues(reasonTooManyTenants, tripperware.SourceAPI, test.orgId)) + assert.Equal(t, float64(1), v) + } } }) }