Adding new parameter for more dynamic horizontal query sharding

Signed-off-by: Pedro Tanaka <pedro.tanaka@shopify.com>

cmd/thanos/query_frontend.go

@@ -73,9 +73,14 @@ func registerQueryFrontend(app *extkingpin.App) {
 	cmd.Flag("query-range.request-downsampled", "Make additional query for downsampled data in case of empty or incomplete response to range request.").
 		Default("true").BoolVar(&cfg.QueryRangeConfig.RequestDownsampled)
 
-	cmd.Flag("query-range.split-interval", "Split query range requests by an interval and execute in parallel, it should be greater than 0 when query-range.response-cache-config is configured.").
+	cmd.Flag("query-range.split-interval", "Split query range requests by an interval and execute in parallel, it should be greater than 0 when query-range.response-cache-config is configured."+
+		"When used in conjunction with query-range.min-split-interval this becomes the upper boundary interval to split queries into.").
 		Default("24h").DurationVar(&cfg.QueryRangeConfig.SplitQueriesByInterval)
 
+	cmd.Flag("query-range.min-split-interval", "Split query range requests by at least this interval."+
+		"It also should be less than query-range.split-interval.").
+		Default("0").DurationVar(&cfg.QueryRangeConfig.MinQuerySplitInterval)
+
 	cmd.Flag("query-range.max-retries-per-request", "Maximum number of retries for a single query range request; beyond this, the downstream error is returned.").
 		Default("5").IntVar(&cfg.QueryRangeConfig.MaxRetries)
 

pkg/queryfrontend/config.go

@@ -219,6 +219,7 @@ type QueryRangeConfig struct {
 	AlignRangeWithStep     bool
 	RequestDownsampled     bool
 	SplitQueriesByInterval time.Duration
+	MinQuerySplitInterval  time.Duration
 	MaxRetries             int
 	Limits                 *cortexvalidation.Limits
 }

pkg/queryfrontend/roundtrip.go

@@ -179,11 +179,9 @@ func newQueryRangeTripperware(
 		)
 	}
 
-	queryIntervalFn := func(_ queryrange.Request) time.Duration {
-		return config.SplitQueriesByInterval
-	}
-
 	if config.SplitQueriesByInterval != 0 {
+		queryIntervalFn := dynamicIntervalFn(config)
+
 		queryRangeMiddleware = append(
 			queryRangeMiddleware,
 			queryrange.InstrumentMiddleware("split_by_interval", m),
@@ -237,6 +235,22 @@ func newQueryRangeTripperware(
 	}, nil
 }
 
+func dynamicIntervalFn(config QueryRangeConfig) queryrange.IntervalFn {
+	return func(r queryrange.Request) time.Duration {
+		if config.MinQuerySplitInterval == 0 && config.SplitQueriesByInterval != 0 {
+			return config.SplitQueriesByInterval
+		}
+
+		queryInterval := time.Duration(r.GetEnd()-r.GetStart()) * time.Millisecond
+		// if the query is multiple of max interval, we use the max interval.
+		if queryInterval/config.SplitQueriesByInterval >= 2 {
+			return config.SplitQueriesByInterval
+		}
+
+		return config.MinQuerySplitInterval
+	}
+}
+
 // newLabelsTripperware returns a Tripperware for labels and series requests
 // configured with middlewares of split by interval and retry.
 func newLabelsTripperware(

pkg/queryfrontend/roundtrip_test.go

@@ -241,12 +241,13 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {
 	labelsCodec := NewThanosLabelsCodec(true, 2*time.Hour)
 
 	for _, tc := range []struct {
-		name          string
-		splitInterval time.Duration
-		req           queryrange.Request
-		codec         queryrange.Codec
-		handlerFunc   func(bool) (*int, http.Handler)
-		expected      int
+		name             string
+		splitInterval    time.Duration
+		minSplitInterval time.Duration
+		req              queryrange.Request
+		codec            queryrange.Codec
+		handlerFunc      func(bool) (*int, http.Handler)
+		expected         int
 	}{
 		{
 			name:          "split interval == 0, disable split",
@@ -280,6 +281,24 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {
 			splitInterval: 1 * time.Hour,
 			expected:      2,
 		},
+		{
+			name:             "split to 4 requests, due to min interval",
+			req:              testRequest,
+			handlerFunc:      promqlResults,
+			codec:            queryRangeCodec,
+			splitInterval:    2 * time.Hour,
+			minSplitInterval: 30 * time.Minute,
+			expected:         4,
+		},
+		{
+			name:             "split to 2 requests, due to split interval",
+			req:              testRequest,
+			handlerFunc:      promqlResults,
+			codec:            queryRangeCodec,
+			splitInterval:    1 * time.Hour,
+			minSplitInterval: 30 * time.Minute,
+			expected:         2,
+		},
 		{
 			name:          "labels request won't be split",
 			req:           testLabelsRequest,
@@ -320,6 +339,7 @@ func TestRoundTripSplitIntervalMiddleware(t *testing.T) {
 					QueryRangeConfig: QueryRangeConfig{
 						Limits:                 defaultLimits,
 						SplitQueriesByInterval: tc.splitInterval,
+						MinQuerySplitInterval:  tc.minSplitInterval,
 					},
 					LabelsConfig: LabelsConfig{
 						Limits:                 defaultLimits,