diff --git a/CHANGELOG.md b/CHANGELOG.md index a33212266b..f5c278fd00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - `-alertmanager.receivers-firewall.block.private-addresses` renamed to `-alertmanager.receivers-firewall-block-private-addresses` * [CHANGE] Change default value of `-server.grpc.keepalive.min-time-between-pings` to `10s` and `-server.grpc.keepalive.ping-without-stream-allowed` to `true`. #4168 * [FEATURE] Querier: Added new `-querier.max-fetched-series-per-query` flag. When Cortex is running with blocks storage, the max series per query limit is enforced in the querier and applies to unique series received from ingesters and store-gateway (long-term storage). #4179 +* [FEATURE] Querier/Ruler: Added new `-querier.max-fetched-chunk-bytes-per-query` flag. When Cortex is running with blocks storage, the max chunk bytes limit is enforced in the querier and ruler and limits the size of all aggregated chunks returned from ingesters and storage as bytes for a query. #4216 * [FEATURE] Alertmanager: Added rate-limits to notifiers. Rate limits used by all integrations can be configured using `-alertmanager.notification-rate-limit`, while per-integration rate limits can be specified via `-alertmanager.notification-rate-limit-per-integration` parameter. Both shared and per-integration limits can be overwritten using overrides mechanism. These limits are applied on individual (per-tenant) alertmanagers. Rate-limited notifications are failed notifications. It is possible to monitor rate-limited notifications via new `cortex_alertmanager_notification_rate_limited_total` metric. #4135 #4163 * [ENHANCEMENT] Alertmanager: introduced new metrics to monitor operation when using `-alertmanager.sharding-enabled`: #4149 * `cortex_alertmanager_state_fetch_replica_state_total` diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 28b2222b41..95eef52b09 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4019,6 +4019,12 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # CLI flag: -querier.max-fetched-series-per-query [max_fetched_series_per_query: | default = 0] +# The maximum size of all chunks in bytes that a query can fetch from each +# ingester and storage. This limit is enforced in the querier and ruler only +# when running Cortex with blocks storage. 0 to disable. +# CLI flag: -querier.max-fetched-chunk-bytes-per-query +[max_fetched_chunk_bytes_per_query: | default = 0] + # Limit how long back data (series and metadata) can be queried, up until # duration ago. This limit is enforced in the query-frontend, querier # and ruler. If the requested time range is outside the allowed range, the diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index e3ddaa6718..fdb760e581 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -50,7 +50,6 @@ import ( var ( errFail = fmt.Errorf("Fail") emptyResponse = &cortexpb.WriteResponse{} - ctx = user.InjectOrgID(context.Background(), "user") ) func TestConfig_Validate(t *testing.T) { @@ -110,6 +109,7 @@ func TestDistributor_Push(t *testing.T) { lastSeenTimestamp := "cortex_distributor_latest_seen_sample_timestamp_seconds" distributorAppend := "cortex_distributor_ingester_appends_total" distributorAppendFailure := "cortex_distributor_ingester_append_failures_total" + ctx := user.InjectOrgID(context.Background(), "user") type samplesIn struct { num int @@ -380,6 +380,7 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { expectedError error } + ctx := user.InjectOrgID(context.Background(), "user") tests := map[string]struct { distributors int ingestionRateStrategy string @@ -469,12 +470,14 @@ func TestDistributor_PushIngestionRateLimiter(t *testing.T) { } func TestDistributor_PushInstanceLimits(t *testing.T) { + type testPush struct { samples int metadata int expectedError error } + ctx := user.InjectOrgID(context.Background(), "user") tests := map[string]struct { preInflight int preRateSamples int // initial rate before first push @@ -618,7 +621,7 @@ func TestDistributor_PushInstanceLimits(t *testing.T) { } func TestDistributor_PushHAInstances(t *testing.T) { - ctx = user.InjectOrgID(context.Background(), "user") + ctx := user.InjectOrgID(context.Background(), "user") for i, tc := range []struct { enableTracker bool @@ -720,6 +723,7 @@ func TestDistributor_PushHAInstances(t *testing.T) { func TestDistributor_PushQuery(t *testing.T) { const shuffleShardSize = 5 + ctx := user.InjectOrgID(context.Background(), "user") nameMatcher := mustEqualMatcher(model.MetricNameLabel, "foo") barMatcher := mustEqualMatcher("bar", "baz") @@ -895,6 +899,7 @@ func TestDistributor_PushQuery(t *testing.T) { func TestDistributor_QueryStream_ShouldReturnErrorIfMaxChunksPerQueryLimitIsReached(t *testing.T) { const maxChunksLimit = 30 // Chunks are duplicated due to replication factor. + ctx := user.InjectOrgID(context.Background(), "user") limits := &validation.Limits{} flagext.DefaultValues(limits) limits.MaxChunksPerQuery = maxChunksLimit @@ -949,9 +954,11 @@ func TestDistributor_QueryStream_ShouldReturnErrorIfMaxChunksPerQueryLimitIsReac func TestDistributor_QueryStream_ShouldReturnErrorIfMaxSeriesPerQueryLimitIsReached(t *testing.T) { const maxSeriesLimit = 10 + ctx := user.InjectOrgID(context.Background(), "user") limits := &validation.Limits{} flagext.DefaultValues(limits) - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(maxSeriesLimit)) + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(maxSeriesLimit, 0)) + // Prepare distributors. ds, _, r, _ := prepare(t, prepConfig{ numIngesters: 3, @@ -994,10 +1001,78 @@ func TestDistributor_QueryStream_ShouldReturnErrorIfMaxSeriesPerQueryLimitIsReac _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) require.Error(t, err) assert.Contains(t, err.Error(), "max number of series limit") + +} + +func TestDistributor_QueryStream_ShouldReturnErrorIfMaxChunkBytesPerQueryLimitIsReached(t *testing.T) { + const seriesToAdd = 10 + + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + + // Prepare distributors. + ds, _, r, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + shardByAllLabels: true, + limits: limits, + }) + defer stopAll(ds, r) + + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } + // Push a single series to allow us to calculate the chunk size to calculate the limit for the test. + writeReq := &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series"}}, 0, 0), + ) + writeRes, err := ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) + chunkSizeResponse, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + + // Use the resulting chunks size to calculate the limit as (series to add + our test series) * the response chunk size. + var responseChunkSize = chunkSizeResponse.ChunksSize() + var maxBytesLimit = (seriesToAdd) * responseChunkSize + + // Update the limiter with the calculated limits. + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, maxBytesLimit)) + + // Push a number of series below the max chunk bytes limit. Subtract one for the series added above. + writeReq = makeWriteRequest(0, seriesToAdd-1, 0) + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the number of chunk bytes is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) + assert.Len(t, queryRes.Chunkseries, seriesToAdd) + + // Push another series to exceed the chunk bytes limit once we'll query back all series. + writeReq = &cortexpb.WriteRequest{} + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]cortexpb.LabelAdapter{{Name: model.MetricNameLabel, Value: "another_series_1"}}, 0, 0), + ) + + writeRes, err = ds[0].Push(ctx, writeReq) + assert.Equal(t, &cortexpb.WriteResponse{}, writeRes) + assert.Nil(t, err) + + // Since the aggregated chunk size is exceeding the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + assert.Equal(t, err, validation.LimitError(fmt.Sprintf(limiter.ErrMaxChunkBytesHit, maxBytesLimit))) } func TestDistributor_Push_LabelRemoval(t *testing.T) { - ctx = user.InjectOrgID(context.Background(), "user") + ctx := user.InjectOrgID(context.Background(), "user") type testcase struct { inputSeries labels.Labels @@ -1086,6 +1161,7 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { } func TestDistributor_Push_ShouldGuaranteeShardingTokenConsistencyOverTheTime(t *testing.T) { + ctx := user.InjectOrgID(context.Background(), "user") tests := map[string]struct { inputSeries labels.Labels expectedSeries labels.Labels @@ -1161,8 +1237,6 @@ func TestDistributor_Push_ShouldGuaranteeShardingTokenConsistencyOverTheTime(t * limits.DropLabels = []string{"dropped"} limits.AcceptHASamples = true - ctx = user.InjectOrgID(context.Background(), "user") - for testName, testData := range tests { t.Run(testName, func(t *testing.T) { ds, ingesters, r, _ := prepare(t, prepConfig{ @@ -1198,6 +1272,8 @@ func TestDistributor_Push_LabelNameValidation(t *testing.T) { {Name: model.MetricNameLabel, Value: "foo"}, {Name: "999.illegal", Value: "baz"}, } + ctx := user.InjectOrgID(context.Background(), "user") + tests := map[string]struct { inputLabels labels.Labels skipLabelNameValidationCfg bool @@ -1245,7 +1321,7 @@ func TestDistributor_Push_LabelNameValidation(t *testing.T) { } func TestDistributor_Push_ExemplarValidation(t *testing.T) { - + ctx := user.InjectOrgID(context.Background(), "user") manyLabels := []string{model.MetricNameLabel, "test"} for i := 1; i < 31; i++ { manyLabels = append(manyLabels, fmt.Sprintf("name_%d", i), fmt.Sprintf("value_%d", i)) @@ -1292,7 +1368,6 @@ func TestDistributor_Push_ExemplarValidation(t *testing.T) { numDistributors: 1, shuffleShardSize: 1, }) - _, err := ds[0].Push(ctx, tc.req) if tc.errMsg != "" { fromError, _ := status.FromError(err) @@ -1308,6 +1383,7 @@ func BenchmarkDistributor_Push(b *testing.B) { const ( numSeriesPerRequest = 1000 ) + ctx := user.InjectOrgID(context.Background(), "user") tests := map[string]struct { prepareConfig func(limits *validation.Limits) @@ -1499,6 +1575,7 @@ func BenchmarkDistributor_Push(b *testing.B) { for testName, testData := range tests { b.Run(testName, func(b *testing.B) { + // Create an in-memory KV store for the ring with 1 ingester registered. kvStore := consul.NewInMemoryClient(ring.GetCodec()) err := kvStore.CAS(context.Background(), ring.IngesterRingKey, @@ -1573,6 +1650,7 @@ func BenchmarkDistributor_Push(b *testing.B) { } func TestSlowQueries(t *testing.T) { + ctx := user.InjectOrgID(context.Background(), "user") nameMatcher := mustEqualMatcher(model.MetricNameLabel, "foo") nIngesters := 3 for _, shardByAllLabels := range []bool{true, false} { @@ -2526,7 +2604,7 @@ func TestSortLabels(t *testing.T) { } func TestDistributor_Push_Relabel(t *testing.T) { - ctx = user.InjectOrgID(context.Background(), "user") + ctx := user.InjectOrgID(context.Background(), "user") type testcase struct { inputSeries labels.Labels diff --git a/pkg/distributor/ha_tracker_test.go b/pkg/distributor/ha_tracker_test.go index 2f1ebb9e35..dbfdc33bd8 100644 --- a/pkg/distributor/ha_tracker_test.go +++ b/pkg/distributor/ha_tracker_test.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/prometheus/pkg/timestamp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/weaveworks/common/user" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ring" @@ -660,7 +661,8 @@ func TestHATracker_MetricsCleanup(t *testing.T) { func TestCheckReplicaCleanup(t *testing.T) { replica := "r1" cluster := "c1" - user := "user" + userID := "user" + ctx := user.InjectOrgID(context.Background(), userID) reg := prometheus.NewPedanticRegistry() @@ -678,32 +680,32 @@ func TestCheckReplicaCleanup(t *testing.T) { now := time.Now() - err = c.checkReplica(context.Background(), user, cluster, replica, now) + err = c.checkReplica(context.Background(), userID, cluster, replica, now) assert.NoError(t, err) - checkReplicaTimestamp(t, time.Second, c, user, cluster, replica, now) + checkReplicaTimestamp(t, time.Second, c, userID, cluster, replica, now) // Replica is not marked for deletion yet. - checkReplicaDeletionState(t, time.Second, c, user, cluster, true, true, false) + checkReplicaDeletionState(t, time.Second, c, userID, cluster, true, true, false) // This will mark replica for deletion (with time.Now()) c.cleanupOldReplicas(ctx, now.Add(1*time.Second)) // Verify marking for deletion. - checkReplicaDeletionState(t, time.Second, c, user, cluster, false, true, true) + checkReplicaDeletionState(t, time.Second, c, userID, cluster, false, true, true) // This will "revive" the replica. now = time.Now() - err = c.checkReplica(context.Background(), user, cluster, replica, now) + err = c.checkReplica(context.Background(), userID, cluster, replica, now) assert.NoError(t, err) - checkReplicaTimestamp(t, time.Second, c, user, cluster, replica, now) // This also checks that entry is not marked for deletion. + checkReplicaTimestamp(t, time.Second, c, userID, cluster, replica, now) // This also checks that entry is not marked for deletion. // This will mark replica for deletion again (with new time.Now()) c.cleanupOldReplicas(ctx, now.Add(1*time.Second)) - checkReplicaDeletionState(t, time.Second, c, user, cluster, false, true, true) + checkReplicaDeletionState(t, time.Second, c, userID, cluster, false, true, true) // Delete entry marked for deletion completely. c.cleanupOldReplicas(ctx, time.Now().Add(5*time.Second)) - checkReplicaDeletionState(t, time.Second, c, user, cluster, false, false, false) + checkReplicaDeletionState(t, time.Second, c, userID, cluster, false, false, false) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` # HELP cortex_ha_tracker_replicas_cleanup_marked_for_deletion_total Number of elected replicas marked for deletion. diff --git a/pkg/distributor/query.go b/pkg/distributor/query.go index 5e36aec06f..e4c4455785 100644 --- a/pkg/distributor/query.go +++ b/pkg/distributor/query.go @@ -232,11 +232,17 @@ func (d *Distributor) queryIngesterStream(ctx context.Context, userID string, re return nil, validation.LimitError(fmt.Sprintf(errMaxChunksPerQueryLimit, util.LabelMatchersToString(matchers), chunksLimit)) } } + for _, series := range resp.Chunkseries { if limitErr := queryLimiter.AddSeries(series.Labels); limitErr != nil { return nil, limitErr } } + + if chunkBytesLimitErr := queryLimiter.AddChunkBytes(resp.ChunksSize()); chunkBytesLimitErr != nil { + return nil, chunkBytesLimitErr + } + for _, series := range resp.Timeseries { if limitErr := queryLimiter.AddSeries(series.Labels); limitErr != nil { return nil, limitErr diff --git a/pkg/ingester/client/custom.go b/pkg/ingester/client/custom.go index becaab3c02..e1c0af148c 100644 --- a/pkg/ingester/client/custom.go +++ b/pkg/ingester/client/custom.go @@ -12,3 +12,18 @@ func (m *QueryStreamResponse) ChunksCount() int { } return count } + +// ChunksSize returns the size of all chunks in the response. +func (m *QueryStreamResponse) ChunksSize() int { + if len(m.Chunkseries) == 0 { + return 0 + } + + size := 0 + for _, entry := range m.Chunkseries { + for _, chunk := range entry.Chunks { + size += chunk.Size() + } + } + return size +} diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index 6ed8030105..49b40b2cb6 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -626,6 +626,13 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( return validation.LimitError(fmt.Sprintf(errMaxChunksPerQueryLimit, util.LabelMatchersToString(matchers), maxChunksLimit)) } } + chunksSize := 0 + for _, c := range s.Chunks { + chunksSize += c.Size() + } + if chunkBytesLimitErr := queryLimiter.AddChunkBytes(chunksSize); chunkBytesLimitErr != nil { + return chunkBytesLimitErr + } } if w := resp.GetWarning(); w != "" { diff --git a/pkg/querier/blocks_store_queryable_test.go b/pkg/querier/blocks_store_queryable_test.go index d7bb456836..51380f9b5b 100644 --- a/pkg/querier/blocks_store_queryable_test.go +++ b/pkg/querier/blocks_store_queryable_test.go @@ -51,7 +51,7 @@ func TestBlocksStoreQuerier_Select(t *testing.T) { metricNameLabel = labels.Label{Name: labels.MetricName, Value: metricName} series1Label = labels.Label{Name: "series", Value: "1"} series2Label = labels.Label{Name: "series", Value: "2"} - noOpQueryLimiter = limiter.NewQueryLimiter(0) + noOpQueryLimiter = limiter.NewQueryLimiter(0, 0) ) type valueResult struct { @@ -507,9 +507,27 @@ func TestBlocksStoreQuerier_Select(t *testing.T) { }, }, limits: &blocksStoreLimitsMock{}, - queryLimiter: limiter.NewQueryLimiter(1), + queryLimiter: limiter.NewQueryLimiter(1, 0), expectedErr: validation.LimitError(fmt.Sprintf("The query hit the max number of series limit (limit: %d)", 1)), }, + "max chunk bytes per query limit hit while fetching chunks": { + finderResult: bucketindex.Blocks{ + {ID: block1}, + {ID: block2}, + }, + storeSetResponses: []interface{}{ + map[BlocksStoreClient][]ulid.ULID{ + &storeGatewayClientMock{remoteAddr: "1.1.1.1", mockedSeriesResponses: []*storepb.SeriesResponse{ + mockSeriesResponse(labels.Labels{metricNameLabel, series1Label}, minT, 1), + mockSeriesResponse(labels.Labels{metricNameLabel, series1Label}, minT+1, 2), + mockHintsResponse(block1, block2), + }}: {block1, block2}, + }, + }, + limits: &blocksStoreLimitsMock{maxChunksPerQuery: 1}, + queryLimiter: limiter.NewQueryLimiter(0, 8), + expectedErr: validation.LimitError(fmt.Sprintf(limiter.ErrMaxChunkBytesHit, 8)), + }, } for testName, testData := range tests { diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 3bd89a767f..1b15c10d00 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -224,7 +224,7 @@ func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, return nil, err } - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(limits.MaxFetchedSeriesPerQuery(userID))) + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(limits.MaxFetchedSeriesPerQuery(userID), limits.MaxFetchedChunkBytesPerQuery(userID))) mint, maxt, err = validateQueryTimeRange(ctx, userID, mint, maxt, limits, cfg.MaxQueryIntoFuture) if err == errEmptyTimeRange { diff --git a/pkg/util/limiter/query_limiter.go b/pkg/util/limiter/query_limiter.go index 9560c6c12e..2a14c90d3e 100644 --- a/pkg/util/limiter/query_limiter.go +++ b/pkg/util/limiter/query_limiter.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/prometheus/common/model" + "go.uber.org/atomic" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" @@ -15,25 +16,30 @@ import ( type queryLimiterCtxKey struct{} var ( - ctxKey = &queryLimiterCtxKey{} - errMaxSeriesHit = "The query hit the max number of series limit (limit: %d)" + ctxKey = &queryLimiterCtxKey{} + errMaxSeriesHit = "The query hit the max number of series limit (limit: %d)" + ErrMaxChunkBytesHit = "The query hit the aggregated chunks size limit (limit: %d bytes)" ) type QueryLimiter struct { uniqueSeriesMx sync.Mutex uniqueSeries map[model.Fingerprint]struct{} - maxSeriesPerQuery int + chunkBytesCount atomic.Int64 + + maxSeriesPerQuery int + maxChunkBytesPerQuery int } // NewQueryLimiter makes a new per-query limiter. Each query limiter // is configured using the `maxSeriesPerQuery` limit. -func NewQueryLimiter(maxSeriesPerQuery int) *QueryLimiter { +func NewQueryLimiter(maxSeriesPerQuery, maxChunkBytesPerQuery int) *QueryLimiter { return &QueryLimiter{ uniqueSeriesMx: sync.Mutex{}, uniqueSeries: map[model.Fingerprint]struct{}{}, - maxSeriesPerQuery: maxSeriesPerQuery, + maxSeriesPerQuery: maxSeriesPerQuery, + maxChunkBytesPerQuery: maxChunkBytesPerQuery, } } @@ -47,7 +53,7 @@ func QueryLimiterFromContextWithFallback(ctx context.Context) *QueryLimiter { ql, ok := ctx.Value(ctxKey).(*QueryLimiter) if !ok { // If there's no limiter return a new unlimited limiter as a fallback - ql = NewQueryLimiter(0) + ql = NewQueryLimiter(0, 0) } return ql } @@ -77,3 +83,14 @@ func (ql *QueryLimiter) uniqueSeriesCount() int { defer ql.uniqueSeriesMx.Unlock() return len(ql.uniqueSeries) } + +// AddChunkBytes adds the input chunk size in bytes and returns an error if the limit is reached. +func (ql *QueryLimiter) AddChunkBytes(chunkSizeInBytes int) error { + if ql.maxChunkBytesPerQuery == 0 { + return nil + } + if ql.chunkBytesCount.Add(int64(chunkSizeInBytes)) > int64(ql.maxChunkBytesPerQuery) { + return validation.LimitError(fmt.Sprintf(ErrMaxChunkBytesHit, ql.maxChunkBytesPerQuery)) + } + return nil +} diff --git a/pkg/util/limiter/query_limiter_test.go b/pkg/util/limiter/query_limiter_test.go index 2fafd6bac3..0456a804ee 100644 --- a/pkg/util/limiter/query_limiter_test.go +++ b/pkg/util/limiter/query_limiter_test.go @@ -25,7 +25,7 @@ func TestQueryLimiter_AddSeries_ShouldReturnNoErrorOnLimitNotExceeded(t *testing labels.MetricName: metricName + "_2", "series2": "1", }) - limiter = NewQueryLimiter(100) + limiter = NewQueryLimiter(100, 0) ) err := limiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(series1)) assert.NoError(t, err) @@ -53,7 +53,7 @@ func TestQueryLimiter_AddSeriers_ShouldReturnErrorOnLimitExceeded(t *testing.T) labels.MetricName: metricName + "_2", "series2": "1", }) - limiter = NewQueryLimiter(1) + limiter = NewQueryLimiter(1, 0) ) err := limiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(series1)) require.NoError(t, err) @@ -61,6 +61,15 @@ func TestQueryLimiter_AddSeriers_ShouldReturnErrorOnLimitExceeded(t *testing.T) require.Error(t, err) } +func TestQueryLimiter_AddChunkBytes(t *testing.T) { + var limiter = NewQueryLimiter(0, 100) + + err := limiter.AddChunkBytes(100) + require.NoError(t, err) + err = limiter.AddChunkBytes(1) + require.Error(t, err) +} + func BenchmarkQueryLimiter_AddSeries(b *testing.B) { const ( metricName = "test_metric" @@ -75,7 +84,7 @@ func BenchmarkQueryLimiter_AddSeries(b *testing.B) { } b.ResetTimer() - limiter := NewQueryLimiter(b.N + 1) + limiter := NewQueryLimiter(b.N+1, 0) for _, s := range series { err := limiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(s)) assert.NoError(b, err) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index a2117edfd6..a3d8a1fb80 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -72,15 +72,16 @@ type Limits struct { MaxGlobalMetadataPerMetric int `yaml:"max_global_metadata_per_metric" json:"max_global_metadata_per_metric"` // Querier enforced limits. - MaxChunksPerQueryFromStore int `yaml:"max_chunks_per_query" json:"max_chunks_per_query"` // TODO Remove in Cortex 1.12. - MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` - MaxFetchedSeriesPerQuery int `yaml:"max_fetched_series_per_query" json:"max_fetched_series_per_query"` - MaxQueryLookback model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"` - MaxQueryLength model.Duration `yaml:"max_query_length" json:"max_query_length"` - MaxQueryParallelism int `yaml:"max_query_parallelism" json:"max_query_parallelism"` - CardinalityLimit int `yaml:"cardinality_limit" json:"cardinality_limit"` - MaxCacheFreshness model.Duration `yaml:"max_cache_freshness" json:"max_cache_freshness"` - MaxQueriersPerTenant int `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"` + MaxChunksPerQueryFromStore int `yaml:"max_chunks_per_query" json:"max_chunks_per_query"` // TODO Remove in Cortex 1.12. + MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` + MaxFetchedSeriesPerQuery int `yaml:"max_fetched_series_per_query" json:"max_fetched_series_per_query"` + MaxFetchedChunkBytesPerQuery int `yaml:"max_fetched_chunk_bytes_per_query" json:"max_fetched_chunk_bytes_per_query"` + MaxQueryLookback model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"` + MaxQueryLength model.Duration `yaml:"max_query_length" json:"max_query_length"` + MaxQueryParallelism int `yaml:"max_query_parallelism" json:"max_query_parallelism"` + CardinalityLimit int `yaml:"cardinality_limit" json:"cardinality_limit"` + MaxCacheFreshness model.Duration `yaml:"max_cache_freshness" json:"max_cache_freshness"` + MaxQueriersPerTenant int `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"` // Ruler defaults and limits. RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"` @@ -147,6 +148,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxChunksPerQueryFromStore, "store.query-chunk-limit", 2e6, "Deprecated. Use -querier.max-fetched-chunks-per-query CLI flag and its respective YAML config option instead. Maximum number of chunks that can be fetched in a single query. This limit is enforced when fetching chunks from the long-term storage only. When running the Cortex chunks storage, this limit is enforced in the querier and ruler, while when running the Cortex blocks storage this limit is enforced in the querier, ruler and store-gateway. 0 to disable.") f.IntVar(&l.MaxChunksPerQuery, "querier.max-fetched-chunks-per-query", 0, "Maximum number of chunks that can be fetched in a single query from ingesters and long-term storage: the total number of actual fetched chunks could be 2x the limit, being independently applied when querying ingesters and long-term storage. This limit is enforced in the ingester (if chunks streaming is enabled), querier, ruler and store-gateway. Takes precedence over the deprecated -store.query-chunk-limit. 0 to disable.") f.IntVar(&l.MaxFetchedSeriesPerQuery, "querier.max-fetched-series-per-query", 0, "The maximum number of unique series for which a query can fetch samples from each ingesters and blocks storage. This limit is enforced in the querier only when running Cortex with blocks storage. 0 to disable") + f.IntVar(&l.MaxFetchedChunkBytesPerQuery, "querier.max-fetched-chunk-bytes-per-query", 0, "The maximum size of all chunks in bytes that a query can fetch from each ingester and storage. This limit is enforced in the querier and ruler only when running Cortex with blocks storage. 0 to disable.") f.Var(&l.MaxQueryLength, "store.max-query-length", "Limit the query time range (end - start time). This limit is enforced in the query-frontend (on the received query), in the querier (on the query possibly split by the query-frontend) and in the chunks storage. 0 to disable.") f.Var(&l.MaxQueryLookback, "querier.max-query-lookback", "Limit how long back data (series and metadata) can be queried, up until duration ago. This limit is enforced in the query-frontend, querier and ruler. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable.") f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 14, "Maximum number of split queries will be scheduled in parallel by the frontend.") @@ -394,6 +396,12 @@ func (o *Overrides) MaxFetchedSeriesPerQuery(userID string) int { return o.getOverridesForUser(userID).MaxFetchedSeriesPerQuery } +// MaxFetchedChunkBytesPerQuery returns the maximum number of bytes for chunks allowed per query when fetching +// chunks from ingesters and blocks storage. +func (o *Overrides) MaxFetchedChunkBytesPerQuery(userID string) int { + return o.getOverridesForUser(userID).MaxFetchedChunkBytesPerQuery +} + // MaxQueryLookback returns the max lookback period of queries. func (o *Overrides) MaxQueryLookback(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).MaxQueryLookback)