Skip to content

Commit 3a0f82f

Browse files
committed
kvserver: allowlist spanconfig updates to bypass range-size backpressure
This commit modifies `backpressurableSpans` to exclude the `system.span_configurations` table by splitting the original span into two ranges that create a gap around the `system.span_configurations` table. This allows spanconfig updates to bypass backpressure entirely, preventing the catch-22 deadlock. We only allowlist the `system.span_configurations` table as the table is self-referential: updates to it trigger spanconfig reconciliation that reads from the same table. When backpressure blocks these updates, protected timestamp and GC TTL updates cannot proceed, creating a deadlock where the system cannot update its own configuration. Other system tables lack this circular dependency, and are not exluded. Fixes: #146982 Release note: None
1 parent 5f0b662 commit 3a0f82f

File tree

3 files changed

+17
-10
lines changed

3 files changed

+17
-10
lines changed

pkg/keys/constants.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,10 @@ var (
374374
NamespaceTableMin = SystemSQLCodec.TablePrefix(NamespaceTableID)
375375
// NamespaceTableMax is the end key of system.namespace.
376376
NamespaceTableMax = SystemSQLCodec.TablePrefix(NamespaceTableID + 1)
377+
// SpanConfigTableDataMin is the start key of system.span_configurations.
378+
SpanConfigTableDataMin = SystemSQLCodec.TablePrefix(SpanConfigurationsTableID)
379+
// SpanConfigTableDataMax is the end key of system.span_configurations.
380+
SpanConfigTableDataMax = SystemSQLCodec.TablePrefix(SpanConfigurationsTableID + 1)
377381

378382
// 4. Non-system tenant SQL keys
379383
//

pkg/kv/kvserver/client_spanconfig_backpressure_test.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ func TestSpanConfigUpdatesBlockedByRangeSizeBackpressureOnDefaultRanges(t *testi
7373
return err
7474
}
7575
if conf.RangeMaxBytes != expRangeMaxBytes {
76-
return fmt.Errorf("expected RangeMaxBytes %d, got %d", expRangeMaxBytes, conf.RangeMaxBytes)
76+
return fmt.Errorf("expected RangeMaxBytes %d, got %d",
77+
expRangeMaxBytes, conf.RangeMaxBytes)
7778
}
7879
return nil
7980
})
@@ -215,8 +216,8 @@ func TestSpanConfigUpdatesBlockedByRangeSizeBackpressureOnDefaultRanges(t *testi
215216
}
216217

217218
// Assert that the operation failed due to backpressure.
218-
require.Error(t, err, `Expected span config writes to fail due to
219-
backpressure, but they succeeded`)
219+
require.NoError(t, err, `Expected span config writes to succeed due to
220+
allowlist; they should bypass backpressure`)
220221

221222
t.Logf("Completed %d direct KV writes\n", i)
222223

@@ -242,9 +243,9 @@ func TestSpanConfigUpdatesBlockedByRangeSizeBackpressureOnDefaultRanges(t *testi
242243
[]spanconfig.Target{testTarget}, []spanconfig.Record{smallSpancofnRecord},
243244
hlc.MinTimestamp, hlc.MaxTimestamp)
244245

245-
require.Error(t, smallSpancofnRecordWriteErr, `Expected smallSpancofnRecord
246-
write to fail`)
247-
t.Logf(`SUCCESS: smallSpancofnRecord write failed as expected;
248-
still getting backpressure\n`)
246+
require.NoError(t, smallSpancofnRecordWriteErr, `Expected smallSpancofnRecord
247+
write to succeed`)
248+
t.Logf(`SUCCESS: smallSpancofnRecord write succeeded;
249+
spanconfigs should bypass backpressure\n`)
249250

250251
}

pkg/kv/kvserver/replica_backpressure.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,9 +88,11 @@ var backpressureByteTolerance = settings.RegisterByteSizeSetting(
8888
// to be backpressured.
8989
var backpressurableSpans = []roachpb.Span{
9090
{Key: keys.TimeseriesPrefix, EndKey: keys.TimeseriesKeyMax},
91-
// Backpressure from the end of the system config forward instead of
92-
// over all table data to avoid backpressuring unsplittable ranges.
93-
{Key: keys.SystemConfigTableDataMax, EndKey: keys.TableDataMax},
91+
// Split the span to exclude the span_configurations table to avoid
92+
// catch-22 situations where protected timestamp updates or garbage
93+
// collection TTL updates are blocked by backpressure.
94+
{Key: keys.SystemConfigTableDataMax, EndKey: keys.SpanConfigTableDataMin},
95+
{Key: keys.SpanConfigTableDataMax, EndKey: keys.TableDataMax},
9496
}
9597

9698
// canBackpressureBatch returns whether the provided BatchRequest is eligible

0 commit comments

Comments
 (0)