feat: Add Table-Store (aka ORM) package - Table and Indexable

docs/core/store.md

@@ -222,6 +222,24 @@ When `Store.{Get, Set}()` is called, the store forwards the call to its parent,
 
 When `Store.Iterator()` is called, it does not simply prefix the `Store.prefix`, since it does not work as intended. In that case, some of the elements are traversed even they are not starting with the prefix.
 
+## Table Store
+
+The table-store package provides a framework for creating relational database tables with primary and secondary keys.
+
+```go
+type Table struct {
+	model       reflect.Type
+	prefix      byte
+	afterSave   []AfterSaveInterceptor
+	afterDelete []AfterDeleteInterceptor
+	cdc         codec.Codec
+}
+```
+
+Such table can be built given a `codec.ProtoMarshaler` model type, a prefix to access the underlying prefix store used to store table data as well as a `Codec` for marshalling/unmarshalling.
+In the prefix store, entities are stored by an unique identifier called `RowID` which can be based either on an `uint64` auto-increment counter or dynamic size bytes.
+Regular CRUD operations can be performed on a table, these methods take a `sdk.KVStore` as parameter to get the table prefix store.
+
 ## Next {hide}
 
 Learn about [encoding](./encoding.md) {hide}

store/README.md

@@ -127,3 +127,23 @@ type Store struct {
 ```
 
 `Store.Store` is a `dbadapter.Store` with a `dbm.NewMemDB()`. All `KVStore` methods are reused. When `Store.Commit()` is called, new `dbadapter.Store` is assigned, discarding previous reference and making it garbage collected.
+
+## Table
+
+The table-store package provides a framework for creating relational database tables with primary and secondary keys.
+
+```go
+type Table struct {
+	model       reflect.Type
+	prefix      byte
+	afterSave   []AfterSaveInterceptor
+	afterDelete []AfterDeleteInterceptor
+	cdc         codec.Codec
+}
+```
+
+Such table can be built given a `codec.ProtoMarshaler` model type, a prefix to access the underlying prefix store used to store table data as well as a `Codec` for marshalling/unmarshalling.
+In the prefix store, entities are stored by an unique identifier called `RowID` which can be based either on an `uint64` auto-increment counter or dynamic size bytes.
+Regular CRUD operations can be performed on a table, these methods take a `sdk.KVStore` as parameter to get the table prefix store.
+
+

store/table/index.go

@@ -0,0 +1,37 @@
+package table
+
+// PrefixRange turns a prefix into a (start, end) range. The start is the given prefix value and
+// the end is calculated by adding 1 bit to the start value. Nil is not allowed as prefix.
+// 		Example: []byte{1, 3, 4} becomes []byte{1, 3, 5}
+// 				 []byte{15, 42, 255, 255} becomes []byte{15, 43, 0, 0}
+//
+// In case of an overflow the end is set to nil.
+//		Example: []byte{255, 255, 255, 255} becomes nil
+//
+func PrefixRange(prefix []byte) ([]byte, []byte) {
+	if prefix == nil {
+		panic("nil key not allowed")
+	}
+	// special case: no prefix is whole range
+	if len(prefix) == 0 {
+		return nil, nil
+	}
+
+	// copy the prefix and update last byte
+	end := make([]byte, len(prefix))
+	copy(end, prefix)
+	l := len(end) - 1
+	end[l]++
+
+	// wait, what if that overflowed?....
+	for end[l] == 0 && l > 0 {
+		l--
+		end[l]++
+	}
+
+	// okay, funny guy, you gave us FFF, no end to this range...
+	if l == 0 && end[0] == 0 {
+		end = nil
+	}
+	return prefix, end
+}

store/table/index_key_codec.go

@@ -0,0 +1,66 @@
+package table
+
+// Max255DynamicLengthIndexKeyCodec works with up to 255 byte dynamic size RowIDs.
+// They are encoded as `concat(searchableKey, rowID, len(rowID)[0])`.
+type Max255DynamicLengthIndexKeyCodec struct{}
+
+// BuildIndexKey builds the index key by appending searchableKey with rowID and length int.
+// The RowID length must not be greater than 255.
+func (Max255DynamicLengthIndexKeyCodec) BuildIndexKey(searchableKey []byte, rowID RowID) []byte {
+	rowIDLen := len(rowID)
+	switch {
+	case rowIDLen == 0:
+		panic("Empty RowID")
+	case rowIDLen > 255:
+		panic("RowID exceeds max size")
+	}
+
+	searchableKeyLen := len(searchableKey)
+	res := make([]byte, searchableKeyLen+rowIDLen+1)
+	copy(res, searchableKey)
+	copy(res[searchableKeyLen:], rowID)
+	res[searchableKeyLen+rowIDLen] = byte(rowIDLen)
+	return res
+}
+
+// StripRowID returns the RowID from the combined persistentIndexKey. It is the reverse operation to BuildIndexKey
+// but with the searchableKey and length int dropped.
+func (Max255DynamicLengthIndexKeyCodec) StripRowID(persistentIndexKey []byte) RowID {
+	n := len(persistentIndexKey)
+	searchableKeyLen := persistentIndexKey[n-1]
+	return persistentIndexKey[n-int(searchableKeyLen)-1 : n-1]
+}
+
+// FixLengthIndexKeyCodec expects the RowID to always have the same length with all entries.
+// They are encoded as `concat(searchableKey, rowID)`.
+type FixLengthIndexKeyCodec struct {
+	rowIDLength int
+}
+
+// FixLengthIndexKeys is a constructor for FixLengthIndexKeyCodec.
+func FixLengthIndexKeys(rowIDLength int) *FixLengthIndexKeyCodec {
+	return &FixLengthIndexKeyCodec{rowIDLength: rowIDLength}
+}
+
+// BuildIndexKey builds the index key by appending searchableKey with rowID.
+// The RowID length must not be greater than what is defined by rowIDLength in construction.
+func (c FixLengthIndexKeyCodec) BuildIndexKey(searchableKey []byte, rowID RowID) []byte {
+	switch n := len(rowID); {
+	case n == 0:
+		panic("Empty RowID")
+	case n > c.rowIDLength:
+		panic("RowID exceeds max size")
+	}
+	n := len(searchableKey)
+	res := make([]byte, n+c.rowIDLength)
+	copy(res, searchableKey)
+	copy(res[n:], rowID)
+	return res
+}
+
+// StripRowID returns the RowID from the combined persistentIndexKey. It is the reverse operation to BuildIndexKey
+// but with the searchableKey dropped.
+func (c FixLengthIndexKeyCodec) StripRowID(persistentIndexKey []byte) RowID {
+	n := len(persistentIndexKey)
+	return persistentIndexKey[n-c.rowIDLength:]
+}

store/table/index_key_codec_test.go

@@ -0,0 +1,115 @@
+package table
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEncodeIndexKey(t *testing.T) {
+	specs := map[string]struct {
+		srcKey   []byte
+		srcRowID RowID
+		enc      IndexKeyCodec
+		expKey   []byte
+		expPanic bool
+	}{
+		"dynamic length example 1": {
+			srcKey:   []byte{0x0, 0x1, 0x2},
+			srcRowID: []byte{0x3, 0x4},
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expKey:   []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x2},
+		},
+		"dynamic length example 2": {
+			srcKey:   []byte{0x0, 0x1},
+			srcRowID: []byte{0x2, 0x3, 0x4},
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expKey:   []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x3},
+		},
+		"dynamic length max row ID": {
+			srcKey:   []byte{0x0, 0x1},
+			srcRowID: []byte(strings.Repeat("a", 255)),
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expKey:   append(append([]byte{0x0, 0x1}, []byte(strings.Repeat("a", 255))...), 0xff),
+		},
+		"dynamic length panics with empty rowID": {
+			srcKey:   []byte{0x0, 0x1},
+			srcRowID: []byte{},
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expPanic: true,
+		},
+		"dynamic length exceeds max row ID": {
+			srcKey:   []byte{0x0, 0x1},
+			srcRowID: []byte(strings.Repeat("a", 256)),
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expPanic: true,
+		},
+		"uint64 example": {
+			srcKey:   []byte{0x0, 0x1, 0x2},
+			srcRowID: []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8},
+			enc:      FixLengthIndexKeys(8),
+			expKey:   []byte{0x0, 0x1, 0x2, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8},
+		},
+		"uint64 panics with empty rowID": {
+			srcKey:   []byte{0x0, 0x1},
+			srcRowID: []byte{},
+			enc:      FixLengthIndexKeys(8),
+			expPanic: true,
+		},
+		"uint64 exceeds max bytes in rowID": {
+			srcKey:   []byte{0x0, 0x1},
+			srcRowID: []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9},
+			enc:      FixLengthIndexKeys(8),
+			expPanic: true,
+		},
+	}
+	for msg, spec := range specs {
+		t.Run(msg, func(t *testing.T) {
+			if spec.expPanic {
+				require.Panics(t,
+					func() {
+						_ = spec.enc.BuildIndexKey(spec.srcKey, spec.srcRowID)
+					})
+				return
+			}
+			got := spec.enc.BuildIndexKey(spec.srcKey, spec.srcRowID)
+			assert.Equal(t, spec.expKey, got)
+		})
+	}
+}
+func TestDecodeIndexKey(t *testing.T) {
+	specs := map[string]struct {
+		srcKey   []byte
+		enc      IndexKeyCodec
+		expRowID RowID
+	}{
+		"dynamic length example 1": {
+			srcKey:   []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x2},
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expRowID: []byte{0x3, 0x4},
+		},
+		"dynamic length example 2": {
+			srcKey:   []byte{0x0, 0x1, 0x2, 0x3, 0x4, 0x3},
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expRowID: []byte{0x2, 0x3, 0x4},
+		},
+		"dynamic length max row ID": {
+			srcKey:   append(append([]byte{0x0, 0x1}, []byte(strings.Repeat("a", 255))...), 0xff),
+			enc:      Max255DynamicLengthIndexKeyCodec{},
+			expRowID: []byte(strings.Repeat("a", 255)),
+		},
+		"uint64 example": {
+			srcKey:   []byte{0x0, 0x1, 0x2, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8},
+			expRowID: []byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8},
+			enc:      FixLengthIndexKeys(8),
+		},
+	}
+	for msg, spec := range specs {
+		t.Run(msg, func(t *testing.T) {
+			gotRow := spec.enc.StripRowID(spec.srcKey)
+			assert.Equal(t, spec.expRowID, gotRow)
+		})
+	}
+}

store/table/index_test.go

@@ -0,0 +1,39 @@
+package table
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrefixRange(t *testing.T) {
+	cases := map[string]struct {
+		src      []byte
+		expStart []byte
+		expEnd   []byte
+		expPanic bool
+	}{
+		"normal":                 {src: []byte{1, 3, 4}, expStart: []byte{1, 3, 4}, expEnd: []byte{1, 3, 5}},
+		"normal short":           {src: []byte{79}, expStart: []byte{79}, expEnd: []byte{80}},
+		"empty case":             {src: []byte{}},
+		"roll-over example 1":    {src: []byte{17, 28, 255}, expStart: []byte{17, 28, 255}, expEnd: []byte{17, 29, 0}},
+		"roll-over example 2":    {src: []byte{15, 42, 255, 255}, expStart: []byte{15, 42, 255, 255}, expEnd: []byte{15, 43, 0, 0}},
+		"pathological roll-over": {src: []byte{255, 255, 255, 255}, expStart: []byte{255, 255, 255, 255}},
+		"nil prohibited":         {expPanic: true},
+	}
+
+	for testName, tc := range cases {
+		t.Run(testName, func(t *testing.T) {
+			if tc.expPanic {
+				require.Panics(t, func() {
+					PrefixRange(tc.src)
+				})
+				return
+			}
+			start, end := PrefixRange(tc.src)
+			assert.Equal(t, tc.expStart, start)
+			assert.Equal(t, tc.expEnd, end)
+		})
+	}
+}

store/table/sequence.go

@@ -0,0 +1,81 @@
+package table
+
+import (
+	"encoding/binary"
+
+	"github.com/cosmos/cosmos-sdk/store/prefix"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/errors"
+)
+
+// sequenceStorageKey is a fix key to read/ write data on the storage layer
+var sequenceStorageKey = []byte{0x1}
+
+// sequence is a persistent unique key generator based on a counter.
+type Sequence struct {
+	prefix byte
+}
+
+func NewSequence(prefix byte) Sequence {
+	return Sequence{
+		prefix: prefix,
+	}
+}
+
+// NextVal increments and persists the counter by one and returns the value.
+func (s Sequence) NextVal(store sdk.KVStore) uint64 {
+	pStore := prefix.NewStore(store, []byte{s.prefix})
+	v := pStore.Get(sequenceStorageKey)
+	seq := DecodeSequence(v)
+	seq++
+	pStore.Set(sequenceStorageKey, EncodeSequence(seq))
+	return seq
+}
+
+// CurVal returns the last value used. 0 if none.
+func (s Sequence) CurVal(store sdk.KVStore) uint64 {
+	pStore := prefix.NewStore(store, []byte{s.prefix})
+	v := pStore.Get(sequenceStorageKey)
+	return DecodeSequence(v)
+}
+
+// PeekNextVal returns the CurVal + increment step. Not persistent.
+func (s Sequence) PeekNextVal(store sdk.KVStore) uint64 {
+	pStore := prefix.NewStore(store, []byte{s.prefix})
+	v := pStore.Get(sequenceStorageKey)
+	return DecodeSequence(v) + 1
+}
+
+// InitVal sets the start value for the sequence. It must be called only once on an empty DB.
+// Otherwise an error is returned when the key exists. The given start value is stored as current
+// value.
+//
+// It is recommended to call this method only for a sequence start value other than `1` as the
+// method consumes unnecessary gas otherwise. A scenario would be an import from genesis.
+func (s Sequence) InitVal(store sdk.KVStore, seq uint64) error {
+	pStore := prefix.NewStore(store, []byte{s.prefix})
+	if pStore.Has(sequenceStorageKey) {
+		return errors.Wrap(ErrUniqueConstraint, "already initialized")
+	}
+	pStore.Set(sequenceStorageKey, EncodeSequence(seq))
+	return nil
+}
+
+// DecodeSequence converts the binary representation into an Uint64 value.
+func DecodeSequence(bz []byte) uint64 {
+	if bz == nil {
+		return 0
+	}
+	val := binary.BigEndian.Uint64(bz)
+	return val
+}
+
+// EncodedSeqLength number of bytes used for the binary representation of a sequence value.
+const EncodedSeqLength = 8
+
+// EncodeSequence converts the sequence value into the binary representation.
+func EncodeSequence(val uint64) []byte {
+	bz := make([]byte, EncodedSeqLength)
+	binary.BigEndian.PutUint64(bz, val)
+	return bz
+}