storage: keyspace management

client/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00
-	github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a
+	github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab
 	github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee
 	github.com/prometheus/client_golang v1.11.0
 	github.com/stretchr/testify v1.7.0

client/go.sum

@@ -104,8 +104,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm
 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
-github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E=
-github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab h1:rb720P/QawBTbC50fjwig1LIpu30ObIiThKokBt5mMk=
+github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee h1:VO2t6IBpfvW34TdtD/G10VvnGqjLic1jzOuHjUb5VqM=
 github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/pingcap/errcode v0.3.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a
+	github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab
 	github.com/pingcap/log v0.0.0-20210906054005-afc726e70354
 	github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d
 	github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594

go.sum

@@ -417,8 +417,8 @@ github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMt
 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk=
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a h1:TxdHGOFeNa1q1mVv6TgReayf26iI4F8PQUm6RnZ/V/E=
-github.com/pingcap/kvproto v0.0.0-20220510035547-0e2f26c0a46a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab h1:rb720P/QawBTbC50fjwig1LIpu30ObIiThKokBt5mMk=
+github.com/pingcap/kvproto v0.0.0-20220805093305-ab1ee4d521ab/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=

server/storage/endpoint/key_path.go

@@ -32,8 +32,11 @@ const (
 	customScheduleConfigPath   = "scheduler_config"
 	gcWorkerServiceSafePointID = "gc_worker"
 	minResolvedTS              = "min_resolved_ts"
-	keySpaceSafePointPrefix    = "key_space/gc_safepoint"
-	keySpaceGCSafePointSuffix  = "gc"
+	keyspaceSafePointPrefix    = "keyspaces/gc_safepoint"
+	keyspaceGCSafePointSuffix  = "gc"
+	keyspacePrefix             = "keyspaces"
+	keyspaceMetaInfix          = "meta"
+	keyspaceIDInfix            = "id"
 )
 
 // AppendToRootPath appends the given key to the rootPath.
@@ -108,31 +111,58 @@ func MinResolvedTSPath() string {
 }
 
 // KeySpaceServiceSafePointPrefix returns the prefix of given service's service safe point.
-// Prefix: /key_space/gc_safepoint/{space_id}/service/
+// Prefix: /keyspaces/gc_safepoint/{space_id}/service/
 func KeySpaceServiceSafePointPrefix(spaceID string) string {
-	return path.Join(keySpaceSafePointPrefix, spaceID, "service") + "/"
+	return path.Join(keyspaceSafePointPrefix, spaceID, "service") + "/"
 }
 
 // KeySpaceGCSafePointPath returns the gc safe point's path of the given key-space.
-// Path: /key_space/gc_safepoint/{space_id}/gc
+// Path: /keyspaces/gc_safepoint/{space_id}/gc
 func KeySpaceGCSafePointPath(spaceID string) string {
-	return path.Join(keySpaceSafePointPrefix, spaceID, keySpaceGCSafePointSuffix)
+	return path.Join(keyspaceSafePointPrefix, spaceID, keyspaceGCSafePointSuffix)
 }
 
 // KeySpaceServiceSafePointPath returns the path of given service's service safe point.
-// Path: /key_space/gc_safepoint/{space_id}/service/{service_id}
+// Path: /keyspaces/gc_safepoint/{space_id}/service/{service_id}
 func KeySpaceServiceSafePointPath(spaceID, serviceID string) string {
 	return path.Join(KeySpaceServiceSafePointPrefix(spaceID), serviceID)
 }
 
 // KeySpaceSafePointPrefix returns prefix for all key-spaces' safe points.
-// Path: /key_space/gc_safepoint/
+// Path: /keyspaces/gc_safepoint/
 func KeySpaceSafePointPrefix() string {
-	return keySpaceSafePointPrefix + "/"
+	return keyspaceSafePointPrefix + "/"
 }
 
 // KeySpaceGCSafePointSuffix returns the suffix for any gc safepoint.
 // Postfix: /gc
 func KeySpaceGCSafePointSuffix() string {
-	return "/" + keySpaceGCSafePointSuffix
+	return "/" + keyspaceGCSafePointSuffix
+}
+
+// KeyspaceMetaPrefix returns the prefix of keyspaces' metadata.
+// Prefix: keyspaces/meta/
+func KeyspaceMetaPrefix() string {
+	return path.Join(keyspacePrefix, keyspaceMetaInfix) + "/"
+}
+
+// KeyspaceMetaPath returns the path to the given keyspace's metadata.
+// Path: keyspaces/meta/{space_id}
+func KeyspaceMetaPath(spaceID uint32) string {
+	idStr := encodeKeyspaceID(spaceID)
+	return path.Join(KeyspaceMetaPrefix(), idStr)
+}
+
+// KeyspaceIDPath returns the path to keyspace id from the given name.
+// Path: keyspaces/id/{name}
+func KeyspaceIDPath(name string) string {
+	return path.Join(keyspacePrefix, keyspaceIDInfix, name)
+}
+
+// encodeKeyspaceID from uint32 to string.
+// It adds extra padding to make encoded ID ordered.
+// Encoded ID can be decoded directly with strconv.ParseUint.
+// Width of the padded keyspaceID is 8 (decimal representation of uint24max is 16777215).
+func encodeKeyspaceID(spaceID uint32) string {
+	return fmt.Sprintf("%08d", spaceID)
 }

server/storage/endpoint/keyspace.go

@@ -0,0 +1,116 @@
+// Copyright 2022 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package endpoint
+
+import (
+	"strconv"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
+	"go.etcd.io/etcd/clientv3"
+)
+
+const (
+	// spaceIDBase is base used to encode/decode spaceID.
+	// It's set to 10 for better readability.
+	spaceIDBase = 10
+	// spaceIDBitSizeMax is the max bitSize of spaceID.
+	// It's currently set to 24 (3bytes).
+	spaceIDBitSizeMax = 24
+)
+
+// KeyspaceStorage defines storage operations on keyspace related data.
+type KeyspaceStorage interface {
+	// SaveKeyspace saves the given keyspace to the storage.
+	SaveKeyspace(*keyspacepb.KeyspaceMeta) error
+	// LoadKeyspace loads keyspace specified by spaceID.
+	LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error)
+	// RemoveKeyspace removes target keyspace specified by spaceID.
+	RemoveKeyspace(spaceID uint32) error
+	// LoadRangeKeyspace loads no more than limit keyspaces starting at startID.
+	LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error)
+	// SaveKeyspaceIDByName saves keyspace name to ID lookup information.
+	// It saves the ID onto the path encoded with name.
+	SaveKeyspaceIDByName(spaceID uint32, name string) error
+	// LoadKeyspaceIDByName loads keyspace ID for the given keyspace specified by name.
+	// It first constructs path to spaceID with the given name, then attempt to retrieve
+	// target spaceID. If the target keyspace does not exist, result boolean is set to false.
+	LoadKeyspaceIDByName(name string) (bool, uint32, error)
+}
+
+var _ KeyspaceStorage = (*StorageEndpoint)(nil)
+
+// SaveKeyspace saves the given keyspace to the storage.
+func (se *StorageEndpoint) SaveKeyspace(keyspace *keyspacepb.KeyspaceMeta) error {
+	key := KeyspaceMetaPath(keyspace.GetId())
+	return se.saveProto(key, keyspace)
+}
+
+// LoadKeyspace loads keyspace specified by spaceID.
+func (se *StorageEndpoint) LoadKeyspace(spaceID uint32, keyspace *keyspacepb.KeyspaceMeta) (bool, error) {
+	key := KeyspaceMetaPath(spaceID)
+	return se.loadProto(key, keyspace)
+}
+
+// RemoveKeyspace removes target keyspace specified by spaceID.
+func (se *StorageEndpoint) RemoveKeyspace(spaceID uint32) error {
+	key := KeyspaceMetaPath(spaceID)
+	return se.Remove(key)
+}
+
+// LoadRangeKeyspace loads keyspaces starting at startID.
+// limit specifies the limit of loaded keyspaces.
+func (se *StorageEndpoint) LoadRangeKeyspace(startID uint32, limit int) ([]*keyspacepb.KeyspaceMeta, error) {
+	startKey := KeyspaceMetaPath(startID)
+	endKey := clientv3.GetPrefixRangeEnd(KeyspaceMetaPrefix())
+	keys, values, err := se.LoadRange(startKey, endKey, limit)
+	if err != nil {
+		return nil, err
+	}
+	if len(keys) == 0 {
+		return []*keyspacepb.KeyspaceMeta{}, nil
+	}
+	keyspaces := make([]*keyspacepb.KeyspaceMeta, 0, len(keys))
+	for _, value := range values {
+		keyspace := &keyspacepb.KeyspaceMeta{}
+		if err = proto.Unmarshal([]byte(value), keyspace); err != nil {
+			return nil, err
+		}
+		keyspaces = append(keyspaces, keyspace)
+	}
+	return keyspaces, nil
+}
+
+// SaveKeyspaceIDByName saves keyspace name to ID lookup information to storage.
+func (se *StorageEndpoint) SaveKeyspaceIDByName(spaceID uint32, name string) error {
+	key := KeyspaceIDPath(name)
+	idStr := strconv.FormatUint(uint64(spaceID), spaceIDBase)
+	return se.Save(key, idStr)
+}
+
+// LoadKeyspaceIDByName loads keyspace ID for the given keyspace name
+func (se *StorageEndpoint) LoadKeyspaceIDByName(name string) (bool, uint32, error) {
+	key := KeyspaceIDPath(name)
+	idStr, err := se.Load(key)
+	// Failed to load the keyspaceID if loading operation errored, or if keyspace does not exist.
+	if err != nil || idStr == "" {
+		return false, 0, err
+	}
+	id64, err := strconv.ParseUint(idStr, spaceIDBase, spaceIDBitSizeMax)
+	if err != nil {
+		return false, 0, err
+	}
+	return true, uint32(id64), nil
+}

server/storage/keyspace_test.go

@@ -0,0 +1,148 @@
+// Copyright 2022 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/pingcap/kvproto/pkg/keyspacepb"
+	"github.com/stretchr/testify/require"
+	"github.com/tikv/pd/server/storage/endpoint"
+)
+
+func TestSaveLoadKeyspace(t *testing.T) {
+	re := require.New(t)
+	storage := NewStorageWithMemoryBackend()
+
+	keyspaces := makeTestKeyspaces()
+	for _, keyspace := range keyspaces {
+		re.NoError(storage.SaveKeyspace(keyspace))
+	}
+
+	for _, keyspace := range keyspaces {
+		spaceID := keyspace.GetId()
+		loadedKeyspace := &keyspacepb.KeyspaceMeta{}
+		// Test load keyspace.
+		success, err := storage.LoadKeyspace(spaceID, loadedKeyspace)
+		re.True(success)
+		re.NoError(err)
+		re.Equal(keyspace, loadedKeyspace)
+		// Test remove keyspace.
+		re.NoError(storage.RemoveKeyspace(spaceID))
+		success, err = storage.LoadKeyspace(spaceID, loadedKeyspace)
+		// Loading a non-existing keyspace should be unsuccessful.
+		re.False(success)
+		// Loading a non-existing keyspace should not return error.
+		re.NoError(err)
+	}
+}
+
+func TestLoadRangeKeyspaces(t *testing.T) {
+	re := require.New(t)
+	storage := NewStorageWithMemoryBackend()
+
+	keyspaces := makeTestKeyspaces()
+	for _, keyspace := range keyspaces {
+		re.NoError(storage.SaveKeyspace(keyspace))
+	}
+
+	// Load all keyspaces.
+	loadedKeyspaces, err := storage.LoadRangeKeyspace(keyspaces[0].GetId(), 0)
+	re.NoError(err)
+	re.ElementsMatch(keyspaces, loadedKeyspaces)
+
+	// Load keyspaces with id >= second test keyspace's id.
+	loadedKeyspaces2, err := storage.LoadRangeKeyspace(keyspaces[1].GetId(), 0)
+	re.NoError(err)
+	re.ElementsMatch(keyspaces[1:], loadedKeyspaces2)
+
+	// Load keyspace with the smallest id.
+	loadedKeyspace3, err := storage.LoadRangeKeyspace(1, 1)
+	re.NoError(err)
+	re.ElementsMatch(keyspaces[:1], loadedKeyspace3)
+}
+
+func TestSaveLoadKeyspaceID(t *testing.T) {
+	re := require.New(t)
+	storage := NewStorageWithMemoryBackend()
+
+	ids := []uint32{100, 200, 300}
+	names := []string{"keyspace1", "keyspace2", "keyspace3"}
+	for i := range ids {
+		re.NoError(storage.SaveKeyspaceIDByName(ids[i], names[i]))
+	}
+
+	for i := range names {
+		success, id, err := storage.LoadKeyspaceIDByName(names[i])
+		re.NoError(err)
+		re.True(success)
+		re.Equal(ids[i], id)
+	}
+	// Loading non-existing id should return false, 0, nil.
+	success, id, err := storage.LoadKeyspaceIDByName("non-existing")
+	re.NoError(err)
+	re.False(success)
+	re.Equal(uint32(0), id)
+}
+
+func makeTestKeyspaces() []*keyspacepb.KeyspaceMeta {
+	now := time.Now().Unix()
+	return []*keyspacepb.KeyspaceMeta{
+		{
+			Id:             10,
+			Name:           "keyspace1",
+			State:          keyspacepb.KeyspaceState_ENABLED,
+			CreatedAt:      now,
+			StateChangedAt: now,
+			Config: map[string]string{
+				"gc_life_time": "6000",
+				"gc_interval":  "3000",
+			},
+		},
+		{
+			Id:             11,
+			Name:           "keyspace2",
+			State:          keyspacepb.KeyspaceState_ARCHIVED,
+			CreatedAt:      now + 300,
+			StateChangedAt: now + 300,
+			Config: map[string]string{
+				"gc_life_time": "1000",
+				"gc_interval":  "5000",
+			},
+		},
+		{
+			Id:             100,
+			Name:           "keyspace3",
+			State:          keyspacepb.KeyspaceState_DISABLED,
+			CreatedAt:      now + 500,
+			StateChangedAt: now + 500,
+			Config: map[string]string{
+				"gc_life_time": "4000",
+				"gc_interval":  "2000",
+			},
+		},
+	}
+}
+
+// TestEncodeSpaceID test spaceID encoding.
+func TestEncodeSpaceID(t *testing.T) {
+	re := require.New(t)
+	re.Equal("keyspaces/meta/00000000", endpoint.KeyspaceMetaPath(0))
+	re.Equal("keyspaces/meta/16777215", endpoint.KeyspaceMetaPath(1<<24-1))
+	re.Equal("keyspaces/meta/00000100", endpoint.KeyspaceMetaPath(100))
+	re.Equal("keyspaces/meta/00000011", endpoint.KeyspaceMetaPath(11))
+	re.Equal("keyspaces/meta/00000010", endpoint.KeyspaceMetaPath(10))
+}

server/storage/storage.go

@@ -40,6 +40,7 @@ type Storage interface {
 	endpoint.GCSafePointStorage
 	endpoint.MinResolvedTSStorage
 	endpoint.KeySpaceGCSafePointStorage
+	endpoint.KeyspaceStorage
 }
 
 // NewStorageWithMemoryBackend creates a new storage with memory backend.