Merge #47999

47999: keys: introduce tenant keyspaces, generate SQL keys through tenant-bound key generator r=nvanbenschoten a=nvanbenschoten

First step in addressing #47903.

This PR introduces a few new important concepts to the CockroachDB codebase:
- Tenants
- Tenant IDs
- Tenant keyspaces
- The "System" tenant

A TenantID is a unique ID associated with a tenant in a multi-tenant cluster. Each tenant is granted exclusive access to a portion of the keyspace and a collection of SQL tables in that keyspace which comprise a logical cluster. The "system" tenant is the system's internal tenant in a multi-tenant cluster and the only tenant in a single-tenant cluster.

The system tenant differs from all other tenants in three important ways:
1. the system tenant's keyspace is not prefixed with a tenant specifier.
2. the system tenant is created by default during cluster initialization.
2. the system tenant had the ability to create and destroy other tenants.

The PR then combines the concepts introduced in the previous commit with a new tenant-bound key generator structure called the `TenantIDKeyGen`. This key generator is bound to a TenantID on creation and is only capable of producing SQL keys for that tenant.

The change then removes all other forms of SQL key generation. Notably, it removes all free functions in `pkg/keys` that generate SQL keys and redirects all key generation in `pkg/sql/sqlbase/index_encoding.go` through `pkg/keys`. I audited all uses of `encoding.EncodeUvarintAscending` (and `DecodeUvarint64` for C++) in the codebase to get to a point where I feel relatively confident about this. Still, it will be nice to verify this further in the future when we test SQL tenant servers and reject all cross-tenant KV traffic.

With all SQL key generation flowing through the tenant-bound key generators, the change then introduces two global key generators:
- `SystemTenantKeyGen`: a SQL key generator for the system tenant
- `TODOTenantKeyGen`: equivalent to `SystemTenantKeyGen`, but should be used when it is unclear which tenant should be referenced by the surrounding context.

These two key generators are used appropriately. Most production code uses the `TODOTenantKeyGen` for now because it will later be replaced with a real tenant-bound key generator that hangs off of its SQL server. Most testing code uses the `SystemTenantKeyGen`. Once most of the obvious uses of `TODOTenantKeyGen` are addressed in the next PR, I might put it behind a function that takes an "issueNo" to make sure that all remaining uses of it are tracked in Github.

Co-authored-by: Nathan VanBenschoten <nvanbenschoten@gmail.com>

c-deps/libroach/encoding.h

@@ -121,6 +121,8 @@ WARN_UNUSED_RESULT bool IsInt(rocksdb::Slice* buf);
 // DecodeTablePrefix validates that the given key has a table prefix. On
 // completion, buf holds the remainder of the key (with the prefix removed) and
 // tbl stores the decoded descriptor ID of the table.
+//
+// TODO(nvanbenschoten): support tenant ID prefix.
 WARN_UNUSED_RESULT bool DecodeTablePrefix(rocksdb::Slice* buf, uint64_t* tbl);
 
 } // namespace cockroach

c-deps/libroach/row_counter.h

@@ -23,6 +23,8 @@ const int MaxReservedDescID = 49;
 // RowCounter counts how many distinct rows appear in the KVs that is is shown
 // via `Count`. Note: the `DataSize` field of the BulkOpSummary is *not*
 // populated by this and should be set separately.
+//
+// TODO(nvanbenschoten): support tenant ID prefix.
 struct RowCounter {
  RowCounter(cockroach::roachpb::BulkOpSummary* summary) : summary(summary) {}
  bool Count(const rocksdb::Slice& key);

pkg/ccl/backupccl/backup_test.go

@@ -967,11 +967,11 @@ func TestBackupRestoreCheckpointing(t *testing.T) {
  if err != nil {
  return err
  }
- low := keys.MakeTablePrefix(ip.backupTableID)
- high := keys.MakeTablePrefix(ip.backupTableID + 1)
+ low := keys.SystemSQLCodec.TablePrefix(ip.backupTableID)
+ high := keys.SystemSQLCodec.TablePrefix(ip.backupTableID + 1)
  if bytes.Compare(highWaterMark, low) <= 0 || bytes.Compare(highWaterMark, high) >= 0 {
  return errors.Errorf("expected high-water mark %v to be between %v and %v",
- highWaterMark, roachpb.Key(low), roachpb.Key(high))
+ highWaterMark, low, high)
  }
  return nil
  }
@@ -2467,7 +2467,7 @@ func TestRestoreAsOfSystemTimeGCBounds(t *testing.T) {
  gcr := roachpb.GCRequest{
  // Bogus span to make it a valid request.
  RequestHeader: roachpb.RequestHeader{
- Key: keys.MakeTablePrefix(keys.MinUserDescID),
+ Key: keys.SystemSQLCodec.TablePrefix(keys.MinUserDescID),
  EndKey: keys.MaxKey,
  },
  Threshold: tc.Server(0).Clock().Now(),

pkg/ccl/backupccl/restore_job.go

@@ -544,11 +544,11 @@ func rewriteBackupSpanKey(kr *storageccl.KeyRewriter, key roachpb.Key) (roachpb.
  // start of the table. That is, change a span start key from /Table/51/1 to
  // /Table/51. Otherwise a permanently empty span at /Table/51-/Table/51/1
  // will be created.
- if b, id, idx, err := sqlbase.DecodeTableIDIndexID(newKey); err != nil {
+ if b, id, idx, err := keys.TODOSQLCodec.DecodeIndexPrefix(newKey); err != nil {
  return nil, errors.NewAssertionErrorWithWrappedErrf(err,
  "could not rewrite span start key: %s", key)
  } else if idx == 1 && len(b) == 0 {
- newKey = keys.MakeTablePrefix(uint32(id))
+ newKey = keys.TODOSQLCodec.TablePrefix(id)
  }
  return newKey, nil
 }

pkg/ccl/backupccl/targets.go

@@ -15,7 +15,6 @@ import (
  "github.com/cockroachdb/cockroach/pkg/ccl/storageccl"
  "github.com/cockroachdb/cockroach/pkg/keys"
  "github.com/cockroachdb/cockroach/pkg/kv"
- "github.com/cockroachdb/cockroach/pkg/roachpb"
  "github.com/cockroachdb/cockroach/pkg/sql"
  "github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
  "github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
@@ -410,7 +409,7 @@ func getAllDescChanges(
  startTime, endTime hlc.Timestamp,
  priorIDs map[sqlbase.ID]sqlbase.ID,
 ) ([]BackupManifest_DescriptorRevision, error) {
- startKey := roachpb.Key(keys.MakeTablePrefix(keys.DescriptorTableID))
+ startKey := keys.TODOSQLCodec.TablePrefix(keys.DescriptorTableID)
  endKey := startKey.PrefixEnd()
 
  allRevs, err := storageccl.GetAllRevisions(ctx, db, startKey, endKey, startTime, endTime)
@@ -421,7 +420,7 @@ func getAllDescChanges(
  var res []BackupManifest_DescriptorRevision
 
  for _, revs := range allRevs {
- id, err := keys.DecodeDescMetadataID(revs.Key)
+ id, err := keys.TODOSQLCodec.DecodeDescMetadataID(revs.Key)
  if err != nil {
  return nil, err
  }
@@ -445,7 +444,7 @@ func getAllDescChanges(
 }
 
 func allSQLDescriptors(ctx context.Context, txn *kv.Txn) ([]sqlbase.Descriptor, error) {
- startKey := roachpb.Key(keys.MakeTablePrefix(keys.DescriptorTableID))
+ startKey := keys.TODOSQLCodec.TablePrefix(keys.DescriptorTableID)
  endKey := startKey.PrefixEnd()
  rows, err := txn.Scan(ctx, startKey, endKey, 0)
  if err != nil {

pkg/ccl/changefeedccl/changefeed.go

@@ -402,7 +402,7 @@ func makeSpansToProtect(targets jobspb.ChangefeedTargets) []roachpb.Span {
  // of table descriptors to version data.
  spansToProtect := make([]roachpb.Span, 0, len(targets)+1)
  addTablePrefix := func(id uint32) {
- tablePrefix := roachpb.Key(keys.MakeTablePrefix(id))
+ tablePrefix := keys.TODOSQLCodec.TablePrefix(id)
  spansToProtect = append(spansToProtect, roachpb.Span{
  Key: tablePrefix,
  EndKey: tablePrefix.PrefixEnd(),

pkg/ccl/changefeedccl/changefeed_test.go

@@ -790,7 +790,7 @@ func TestChangefeedSchemaChangeAllowBackfill(t *testing.T) {
 func fetchDescVersionModificationTime(
  t testing.TB, db *gosql.DB, f cdctest.TestFeedFactory, tableName string, version int,
 ) hlc.Timestamp {
- tblKey := roachpb.Key(keys.MakeTablePrefix(keys.DescriptorTableID))
+ tblKey := keys.SystemSQLCodec.TablePrefix(keys.DescriptorTableID)
  header := roachpb.RequestHeader{
  Key: tblKey,
  EndKey: tblKey.PrefixEnd(),
@@ -822,7 +822,7 @@ func fetchDescVersionModificationTime(
  continue
  }
  k := it.UnsafeKey()
- remaining, _, _, err := sqlbase.DecodeTableIDIndexID(k.Key)
+ remaining, _, _, err := keys.SystemSQLCodec.DecodeIndexPrefix(k.Key)
  if err != nil {
  t.Fatal(err)
  }
@@ -2204,8 +2204,8 @@ func TestChangefeedProtectedTimestamps(t *testing.T) {
  }
  mkCheckRecord = func(t *testing.T, tableID int) func(r *ptpb.Record) error {
  expectedKeys := map[string]struct{}{
- string(keys.MakeTablePrefix(uint32(tableID))): {},
- string(keys.MakeTablePrefix(keys.DescriptorTableID)): {},
+ string(keys.SystemSQLCodec.TablePrefix(uint32(tableID))): {},
+ string(keys.SystemSQLCodec.TablePrefix(keys.DescriptorTableID)): {},
  }
  return func(ptr *ptpb.Record) error {
  if ptr == nil {

pkg/ccl/changefeedccl/kvfeed/kv_feed_test.go

@@ -40,7 +40,7 @@ func TestKVFeed(t *testing.T) {
  }
  kv := func(tableID uint32, k, v string, ts hlc.Timestamp) roachpb.KeyValue {
  vDatum := tree.DString(k)
- key, err := sqlbase.EncodeTableKey(keys.MakeTablePrefix(tableID), &vDatum, encoding.Ascending)
+ key, err := sqlbase.EncodeTableKey(keys.SystemSQLCodec.TablePrefix(tableID), &vDatum, encoding.Ascending)
  if err != nil {
  panic(err)
  }
@@ -356,7 +356,7 @@ var _ schemaFeed = (*rawTableFeed)(nil)
 
 func tableSpan(tableID uint32) roachpb.Span {
  return roachpb.Span{
- Key: keys.MakeTablePrefix(tableID),
- EndKey: roachpb.Key(keys.MakeTablePrefix(tableID)).PrefixEnd(),
+ Key: keys.SystemSQLCodec.TablePrefix(tableID),
+ EndKey: keys.SystemSQLCodec.TablePrefix(tableID).PrefixEnd(),
  }
 }

pkg/ccl/changefeedccl/rowfetcher_cache.go

@@ -11,6 +11,7 @@ package changefeedccl
 import (
  "context"
 
+ "github.com/cockroachdb/cockroach/pkg/keys"
  "github.com/cockroachdb/cockroach/pkg/roachpb"
  "github.com/cockroachdb/cockroach/pkg/sql"
  "github.com/cockroachdb/cockroach/pkg/sql/row"
@@ -43,8 +44,12 @@ func (c *rowFetcherCache) TableDescForKey(
  ctx context.Context, key roachpb.Key, ts hlc.Timestamp,
 ) (*sqlbase.ImmutableTableDescriptor, error) {
  var tableDesc *sqlbase.ImmutableTableDescriptor
+ key, err := keys.TODOSQLCodec.StripTenantPrefix(key)
+ if err != nil {
+ return nil, err
+ }
  for skippedCols := 0; ; {
- remaining, tableID, _, err := sqlbase.DecodeTableIDIndexID(key)
+ remaining, tableID, _, err := sqlbase.DecodePartialTableIDIndexID(key)
  if err != nil {
  return nil, err
  }

pkg/ccl/changefeedccl/schemafeed/schema_feed.go

@@ -452,7 +452,7 @@ func fetchTableDescriptorVersions(
  log.Infof(ctx, `fetching table descs (%s,%s]`, startTS, endTS)
  }
  start := timeutil.Now()
- span := roachpb.Span{Key: keys.MakeTablePrefix(keys.DescriptorTableID)}
+ span := roachpb.Span{Key: keys.TODOSQLCodec.TablePrefix(keys.DescriptorTableID)}
  span.EndKey = span.Key.PrefixEnd()
  header := roachpb.Header{Timestamp: endTS}
  req := &roachpb.ExportRequest{
@@ -486,7 +486,7 @@ func fetchTableDescriptorVersions(
  return nil
  }
  k := it.UnsafeKey()
- remaining, _, _, err := sqlbase.DecodeTableIDIndexID(k.Key)
+ remaining, _, _, err := keys.TODOSQLCodec.DecodeIndexPrefix(k.Key)
  if err != nil {
  return err
  }

pkg/ccl/importccl/bench_test.go

@@ -61,7 +61,7 @@ func BenchmarkImportWorkload(b *testing.B) {
 
  t := tableSSTable{
  meta: table,
- span: roachpb.Span{Key: keys.MakeTablePrefix(uint32(tableID))},
+ span: roachpb.Span{Key: keys.SystemSQLCodec.TablePrefix(uint32(tableID))},
  sstData: sst,
  }
  t.span.EndKey = t.span.Key.PrefixEnd()

pkg/ccl/importccl/import_processor.go

@@ -15,6 +15,7 @@ import (
  "time"
 
  "github.com/cockroachdb/cockroach/pkg/ccl/storageccl"
+ "github.com/cockroachdb/cockroach/pkg/keys"
  "github.com/cockroachdb/cockroach/pkg/kv/kvserver/storagebase"
  "github.com/cockroachdb/cockroach/pkg/roachpb"
  "github.com/cockroachdb/cockroach/pkg/sql/execinfra"
@@ -319,7 +320,7 @@ func ingestKvs(
  // number of L0 (and total) files, but with a lower memory usage.
  for kvBatch := range kvCh {
  for _, kv := range kvBatch.KVs {
- _, _, indexID, indexErr := sqlbase.DecodeTableIDIndexID(kv.Key)
+ _, _, indexID, indexErr := keys.TODOSQLCodec.DecodeIndexPrefix(kv.Key)
  if indexErr != nil {
  return indexErr
  }

pkg/ccl/storageccl/bench_test.go

@@ -189,7 +189,7 @@ func BenchmarkImport(b *testing.B) {
  OldID: uint32(tableDesc.ID), NewDesc: newDescBytes,
  })
  }
- newStartKey := roachpb.Key(keys.MakeTablePrefix(uint32(id)))
+ newStartKey := keys.SystemSQLCodec.TablePrefix(uint32(id))
 
  b.StartTimer()
  var files []roachpb.ImportRequest_File

pkg/ccl/storageccl/key_rewriter.go

@@ -11,6 +11,7 @@ package storageccl
 import (
  "bytes"
 
+ "github.com/cockroachdb/cockroach/pkg/keys"
  "github.com/cockroachdb/cockroach/pkg/roachpb"
  "github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
  "github.com/cockroachdb/cockroach/pkg/util/encoding"
@@ -116,10 +117,7 @@ func MakeKeyRewriter(descs map[sqlbase.ID]*sqlbase.TableDescriptor) (*KeyRewrite
 // function, but it takes into account interleaved ancestors, which we don't
 // want here.
 func makeKeyRewriterPrefixIgnoringInterleaved(tableID sqlbase.ID, indexID sqlbase.IndexID) []byte {
- var key []byte
- key = encoding.EncodeUvarintAscending(key, uint64(tableID))
- key = encoding.EncodeUvarintAscending(key, uint64(indexID))
- return key
+ return keys.TODOSQLCodec.IndexPrefix(uint32(tableID), uint32(indexID))
 }
 
 // RewriteKey modifies key (possibly in place), changing all table IDs to their
@@ -142,7 +140,7 @@ func (kr *KeyRewriter) RewriteKey(key []byte, isFromSpan bool) ([]byte, bool, er
  // Fetch the original table ID for descriptor lookup. Ignore errors because
  // they will be caught later on if tableID isn't in descs or kr doesn't
  // perform a rewrite.
- _, tableID, _ := encoding.DecodeUvarintAscending(key)
+ _, tableID, _ := keys.TODOSQLCodec.DecodeTablePrefix(key)
  // Rewrite the first table ID.
  key, ok := kr.prefixes.rewriteKey(key)
  if !ok {
@@ -153,15 +151,15 @@ func (kr *KeyRewriter) RewriteKey(key []byte, isFromSpan bool) ([]byte, bool, er
  return nil, false, errors.Errorf("missing descriptor for table %d", tableID)
  }
  // Check if this key may have interleaved children.
- k, _, indexID, err := sqlbase.DecodeTableIDIndexID(key)
+ k, _, indexID, err := keys.TODOSQLCodec.DecodeIndexPrefix(key)
  if err != nil {
  return nil, false, err
  }
  if len(k) == 0 {
  // If there isn't any more data, we are at some split boundary.
  return key, true, nil
  }
- idx, err := desc.FindIndexByID(indexID)
+ idx, err := desc.FindIndexByID(sqlbase.IndexID(indexID))
  if err != nil {
  return nil, false, err
  }

pkg/ccl/utilccl/sampledataccl/bankdata_test.go

@@ -73,7 +73,7 @@ func TestToBackup(t *testing.T) {
  t.Run("NextKeyValues", func(t *testing.T) {
  for _, requestedKVs := range []int{2, 3} {
  newTableID := sqlbase.ID(keys.MaxReservedDescID + requestedKVs)
- newTablePrefix := roachpb.Key(keys.MakeTablePrefix(uint32(newTableID)))
+ newTablePrefix := keys.SystemSQLCodec.TablePrefix(uint32(newTableID))
 
  keys := make(map[string]struct{}, rows)
  for {

pkg/cmd/zerosum/main.go

@@ -216,7 +216,7 @@ func (z *zeroSum) monkey(tableID uint32, d time.Duration) {
  for {
  time.Sleep(time.Duration(rand.Float64() * float64(d)))
 
- key := keys.MakeTablePrefix(tableID)
+ key := keys.SystemSQLCodec.TablePrefix(tableID)
  key = encoding.EncodeVarintAscending(key, int64(zipf.Uint64()))
 
  switch r.Intn(2) {

pkg/config/keys.go

@@ -13,32 +13,23 @@ package config
 import (
  "github.com/cockroachdb/cockroach/pkg/keys"
  "github.com/cockroachdb/cockroach/pkg/roachpb"
- "github.com/cockroachdb/cockroach/pkg/util/encoding"
 )
 
 // MakeZoneKeyPrefix returns the key prefix for id's row in the system.zones
 // table.
 func MakeZoneKeyPrefix(id uint32) roachpb.Key {
- return keys.ZoneKeyPrefix(id)
+ return keys.TODOSQLCodec.ZoneKeyPrefix(id)
 }
 
 // MakeZoneKey returns the key for id's entry in the system.zones table.
 func MakeZoneKey(id uint32) roachpb.Key {
- return keys.ZoneKey(id)
+ return keys.TODOSQLCodec.ZoneKey(id)
 }
 
 // DecodeObjectID decodes the object ID from the front of key. It returns the
 // decoded object ID, the remainder of the key, and whether the result is valid
 // (i.e., whether the key was within the structured key space).
 func DecodeObjectID(key roachpb.RKey) (uint32, []byte, bool) {
- if key.Equal(roachpb.RKeyMax) {
- return 0, nil, false
- }
- if encoding.PeekType(key) != encoding.Int {
- // TODO(marc): this should eventually return SystemDatabaseID.
- return 0, nil, false
- }
- // Consume first encoded int.
- rem, id64, err := encoding.DecodeUvarintAscending(key)
- return uint32(id64), rem, err == nil
+ rem, id, err := keys.TODOSQLCodec.DecodeTablePrefix(key.AsRawKey())
+ return id, rem, err == nil
 }

pkg/config/keys_test.go

@@ -37,9 +37,9 @@ func TestDecodeObjectID(t *testing.T) {
  {roachpb.RKeyMax, nil, false, 0},
 
  // Valid, even if there are things after the ID.
- {testutils.MakeKey(keys.MakeTablePrefix(42), roachpb.RKey("\xff")), []byte{'\xff'}, true, 42},
- {keys.MakeTablePrefix(0), []byte{}, true, 0},
- {keys.MakeTablePrefix(999), []byte{}, true, 999},
+ {testutils.MakeKey(keys.SystemSQLCodec.TablePrefix(42), roachpb.RKey("\xff")), []byte{'\xff'}, true, 42},
+ {roachpb.RKey(keys.SystemSQLCodec.TablePrefix(0)), []byte{}, true, 0},
+ {roachpb.RKey(keys.SystemSQLCodec.TablePrefix(999)), []byte{}, true, 999},
  }
 
  for tcNum, tc := range testCases {