Add sharded queries metrics

CHANGELOG.md

@@ -16,6 +16,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Added
 * [#5654](https://github.com/thanos-io/thanos/pull/5654) Query: add `--grpc-compression` flag that controls the compression used in gRPC client. With the flag it is now possible to compress the traffic between Query and StoreAPI nodes - you get lower network usage in exchange for a bit higher CPU/RAM usage.
+- [#5650](https://github.com/thanos-io/thanos/pull/5650) Query Frontend: Add sharded queries metrics.
 
 ### Changed
 

pkg/queryfrontend/roundtrip.go

@@ -225,7 +225,7 @@ func newQueryRangeTripperware(
 	if numShards > 0 {
 		queryRangeMiddleware = append(
 			queryRangeMiddleware,
-			PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec),
+			PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec, reg),
 		)
 	}
 
@@ -313,7 +313,7 @@ func newInstantQueryTripperware(
 		instantQueryMiddlewares = append(
 			instantQueryMiddlewares,
 			queryrange.InstrumentMiddleware("sharding", m),
-			PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec),
+			PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec, reg),
 		)
 	}
 

pkg/queryfrontend/shard_query.go

@@ -9,21 +9,34 @@ package queryfrontend
 import (
 	"context"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+
 	"github.com/thanos-io/thanos/internal/cortex/querier/queryrange"
 
 	"github.com/thanos-io/thanos/pkg/querysharding"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 )
 
 // PromQLShardingMiddleware creates a new Middleware that shards PromQL aggregations using grouping labels.
-func PromQLShardingMiddleware(queryAnalyzer *querysharding.QueryAnalyzer, numShards int, limits queryrange.Limits, merger queryrange.Merger) queryrange.Middleware {
+func PromQLShardingMiddleware(queryAnalyzer *querysharding.QueryAnalyzer, numShards int, limits queryrange.Limits, merger queryrange.Merger, registerer prometheus.Registerer) queryrange.Middleware {
 	return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
+		queriesTotal := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
+			Namespace: "thanos",
+			Name:      "frontend_sharding_middleware_queries_total",
+			Help:      "Total number of queries analyzed by the sharding middleware",
+		}, []string{"shardable"})
+
+		queriesTotal.WithLabelValues("true")
+		queriesTotal.WithLabelValues("false")
+
 		return querySharder{
 			next:          next,
 			limits:        limits,
 			queryAnalyzer: queryAnalyzer,
 			numShards:     numShards,
 			merger:        merger,
+			queriesTotal:  queriesTotal,
 		}
 	})
 }
@@ -35,6 +48,9 @@ type querySharder struct {
 	queryAnalyzer *querysharding.QueryAnalyzer
 	numShards     int
 	merger        queryrange.Merger
+
+	// Metrics
+	queriesTotal *prometheus.CounterVec
 }
 
 func (s querySharder) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {
@@ -44,9 +60,11 @@ func (s querySharder) Do(ctx context.Context, r queryrange.Request) (queryrange.
 	}
 
 	if !analysis.IsShardable() {
+		s.queriesTotal.WithLabelValues("false").Inc()
 		return s.next.Do(ctx, r)
 	}
 
+	s.queriesTotal.WithLabelValues("true").Inc()
 	reqs := s.shardQuery(r, analysis)
 
 	reqResps, err := queryrange.DoRequests(ctx, s.next, reqs, s.limits)