Add a too_many_tenants label to cortex_rejected_queries_total metric

Signed-off-by: SungJin1212 <tjdwls1201@gmail.com>

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## master / unreleased
 
 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526
+* [ENHANCEMENT] Query Frontend: Add a `too_many_tenants` reason label value to `cortex_rejected_queries_total` metric to track the rejected query count due to the # of tenant limits. #6569 
 * [BUGFIX] Ingester: Avoid error or early throttling when READONLY ingesters are present in the ring #6517
 
 ## 1.19.0 in progress

pkg/frontend/transport/handler.go

@@ -45,6 +45,7 @@ var (
 )
 
 const (
+	reasonTooManyTenants           = "too_many_tenants"
 	reasonRequestBodySizeExceeded  = "request_body_size_exceeded"
 	reasonResponseBodySizeExceeded = "response_body_size_exceeded"
 	reasonTooManyRequests          = "too_many_requests"
@@ -191,16 +192,21 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	userID := tenant.JoinTenantIDs(tenantIDs)
+
 	if f.tenantFederationCfg.Enabled {
 		maxTenant := f.tenantFederationCfg.MaxTenant
 		if maxTenant > 0 && len(tenantIDs) > maxTenant {
+			source := tripperware.GetSource(r.Header.Get("User-Agent"))
+			if f.cfg.QueryStatsEnabled {
+				// when query stats enabled, track too many tenants reason
+				f.rejectedQueries.WithLabelValues(reasonTooManyTenants, source, userID).Inc()
+			}
 			http.Error(w, fmt.Errorf(errTooManyTenants, maxTenant, len(tenantIDs)).Error(), http.StatusBadRequest)
 			return
 		}
 	}
 
-	userID := tenant.JoinTenantIDs(tenantIDs)
-
 	// Initialise the stats in the context and make sure it's propagated
 	// down the request chain.
 	if f.cfg.QueryStatsEnabled {

pkg/frontend/transport/handler_test.go

@@ -589,7 +589,7 @@ func Test_TenantFederation_MaxTenant(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			handler := NewHandler(HandlerConfig{}, test.cfg, roundTripper, log.NewNopLogger(), nil)
+			handler := NewHandler(HandlerConfig{QueryStatsEnabled: true}, test.cfg, roundTripper, log.NewNopLogger(), nil)
 			handlerWithAuth := middleware.Merge(middleware.AuthenticateUser).Wrap(handler)
 
 			req := httptest.NewRequest("GET", "http://fake", nil)
@@ -604,6 +604,11 @@ func Test_TenantFederation_MaxTenant(t *testing.T) {
 
 			if test.expectedErrMsg != "" {
 				require.Contains(t, string(body), test.expectedErrMsg)
+
+				if strings.Contains(test.expectedErrMsg, "too many tenants") {
+					v := promtest.ToFloat64(handler.rejectedQueries.WithLabelValues(reasonTooManyTenants, tripperware.SourceAPI, test.orgId))
+					assert.Equal(t, float64(1), v)
+				}
 			}
 		})
 	}