Add sharded queries metrics

CHANGELOG.md

@@ -16,6 +16,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 ### Added
 * [#5654](https://github.com/thanos-io/thanos/pull/5654) Query: add `--grpc-compression` flag that controls the compression used in gRPC client. With the flag it is now possible to compress the traffic between Query and StoreAPI nodes - you get lower network usage in exchange for a bit higher CPU/RAM usage.
+- [#5650](https://github.com/thanos-io/thanos/pull/5650) Query Frontend: Add sharded queries metrics.
 
 ### Changed
 

pkg/queryfrontend/roundtrip.go

@@ -225,7 +225,7 @@ func newQueryRangeTripperware(
  if numShards > 0 {
  queryRangeMiddleware = append(
  queryRangeMiddleware,
- PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec),
+ PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec, reg),
  )
  }
 
@@ -313,7 +313,7 @@ func newInstantQueryTripperware(
  instantQueryMiddlewares = append(
  instantQueryMiddlewares,
  queryrange.InstrumentMiddleware("sharding", m),
- PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec),
+ PromQLShardingMiddleware(querysharding.NewQueryAnalyzer(), numShards, limits, codec, reg),
  )
  }
 

pkg/queryfrontend/shard_query.go

@@ -9,21 +9,34 @@ package queryfrontend
 import (
  "context"
 
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+
  "github.com/thanos-io/thanos/internal/cortex/querier/queryrange"
 
  "github.com/thanos-io/thanos/pkg/querysharding"
  "github.com/thanos-io/thanos/pkg/store/storepb"
 )
 
 // PromQLShardingMiddleware creates a new Middleware that shards PromQL aggregations using grouping labels.
-func PromQLShardingMiddleware(queryAnalyzer *querysharding.QueryAnalyzer, numShards int, limits queryrange.Limits, merger queryrange.Merger) queryrange.Middleware {
+func PromQLShardingMiddleware(queryAnalyzer *querysharding.QueryAnalyzer, numShards int, limits queryrange.Limits, merger queryrange.Merger, registerer prometheus.Registerer) queryrange.Middleware {
  return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
+ queriesTotal := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
+ Namespace: "thanos",
+ Name: "frontend_sharding_middleware_queries_total",
+ Help: "Total number of queries analyzed by the sharding middleware",
+ }, []string{"shardable"})
+
+ queriesTotal.WithLabelValues("true")
+ queriesTotal.WithLabelValues("false")
+
  return querySharder{
  next: next,
  limits: limits,
  queryAnalyzer: queryAnalyzer,
  numShards: numShards,
  merger: merger,
+ queriesTotal: queriesTotal,
  }
  })
 }
@@ -35,6 +48,9 @@ type querySharder struct {
  queryAnalyzer *querysharding.QueryAnalyzer
  numShards int
  merger queryrange.Merger
+
+ // Metrics
+ queriesTotal *prometheus.CounterVec
 }
 
 func (s querySharder) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {
@@ -44,9 +60,11 @@ func (s querySharder) Do(ctx context.Context, r queryrange.Request) (queryrange.
  }
 
  if !analysis.IsShardable() {
+ s.queriesTotal.WithLabelValues("false").Inc()
  return s.next.Do(ctx, r)
  }
 
+ s.queriesTotal.WithLabelValues("true").Inc()
  reqs := s.shardQuery(r, analysis)
 
  reqResps, err := queryrange.DoRequests(ctx, s.next, reqs, s.limits)