Atree Register Inlining and Data Deduplication

basicarray.go

@@ -76,7 +76,7 @@ func newBasicArrayDataSlabFromData(
 		)
 	}
 
-	cborDec := decMode.NewByteStreamDecoder(data[2:])
+	cborDec := decMode.NewByteStreamDecoder(data[versionAndFlagSize:])
 
 	elemCount, err := cborDec.DecodeArrayHead()
 	if err != nil {
@@ -85,7 +85,7 @@ func newBasicArrayDataSlabFromData(
 
 	elements := make([]Storable, elemCount)
 	for i := 0; i < int(elemCount); i++ {
-		storable, err := decodeStorable(cborDec, SlabIDUndefined)
+		storable, err := decodeStorable(cborDec, id, nil)
 		if err != nil {
 			// Wrap err as external error (if needed) because err is returned by StorableDecoder callback.
 			return nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to decode array element")
@@ -101,10 +101,17 @@ func newBasicArrayDataSlabFromData(
 
 func (a *BasicArrayDataSlab) Encode(enc *Encoder) error {
 
-	flag := maskBasicArray | maskSlabRoot
+	const version = 1
+
+	h, err := newArraySlabHead(version, slabBasicArray)
+	if err != nil {
+		return NewEncodingError(err)
+	}
+
+	h.setRoot()
 
 	// Encode flag
-	_, err := enc.Write([]byte{0x0, flag})
+	_, err = enc.Write(h[:])
 	if err != nil {
 		return NewEncodingError(err)
 	}

cmd/main/main.go

@@ -77,6 +77,14 @@ type testTypeInfo struct{}
 
 var _ atree.TypeInfo = testTypeInfo{}
 
+func (testTypeInfo) IsComposite() bool {
+	return false
+}
+
+func (i testTypeInfo) ID() string {
+	return fmt.Sprintf("uint64(%d)", i)
+}
+
 func (testTypeInfo) Encode(e *cbor.StreamEncoder) error {
 	return e.EncodeUint8(42)
 }
@@ -86,7 +94,7 @@ func (i testTypeInfo) Equal(other atree.TypeInfo) bool {
 	return ok
 }
 
-func decodeStorable(dec *cbor.StreamDecoder, _ atree.SlabID) (atree.Storable, error) {
+func decodeStorable(dec *cbor.StreamDecoder, _ atree.SlabID, _ []atree.ExtraData) (atree.Storable, error) {
 	tagNumber, err := dec.DecodeTagNumber()
 	if err != nil {
 		return nil, err

cmd/stress/storable.go

@@ -413,7 +413,7 @@ func (v StringValue) String() string {
 	return v.str
 }
 
-func decodeStorable(dec *cbor.StreamDecoder, _ atree.SlabID) (atree.Storable, error) {
+func decodeStorable(dec *cbor.StreamDecoder, _ atree.SlabID, _ []atree.ExtraData) (atree.Storable, error) {
 	t, err := dec.NextType()
 	if err != nil {
 		return nil, err

cmd/stress/typeinfo.go

@@ -19,6 +19,8 @@
 package main
 
 import (
+	"fmt"
+
 	"github.com/onflow/atree"
 
 	"github.com/fxamacker/cbor/v2"
@@ -30,6 +32,14 @@ type testTypeInfo struct {
 
 var _ atree.TypeInfo = testTypeInfo{}
 
+func (i testTypeInfo) IsComposite() bool {
+	return false
+}
+
+func (i testTypeInfo) ID() string {
+	return fmt.Sprintf("uint64(%d)", i)
+}
+
 func (i testTypeInfo) Encode(e *cbor.StreamEncoder) error {
 	return e.EncodeUint64(i.value)
 }

encode.go

@@ -30,19 +30,45 @@ type Encoder struct {
 	io.Writer
 	CBOR    *cbor.StreamEncoder
 	Scratch [64]byte
+	encMode cbor.EncMode
 }
 
 func NewEncoder(w io.Writer, encMode cbor.EncMode) *Encoder {
 	streamEncoder := encMode.NewStreamEncoder(w)
 	return &Encoder{
-		Writer: w,
-		CBOR:   streamEncoder,
+		Writer:  w,
+		CBOR:    streamEncoder,
+		encMode: encMode,
 	}
 }
 
+// encodeStorableAsElement encodes storable as Array or OrderedMap element.
+// Storable is encode as an inlined ArrayDataSlab or MapDataSlab if it is ArrayDataSlab or MapDataSlab.
+func encodeStorableAsElement(enc *Encoder, storable Storable, inlinedTypeInfo *inlinedExtraData) error {
+
+	switch storable := storable.(type) {
+
+	case *ArrayDataSlab:
+		return storable.encodeAsInlined(enc, inlinedTypeInfo)
+
+	case *MapDataSlab:
+		return storable.encodeAsInlined(enc, inlinedTypeInfo)
+
+	default:
+		err := storable.Encode(enc)
+		if err != nil {
+			// Wrap err as external error (if needed) because err is returned by Storable interface.
+			return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode map value")
+		}
+	}
+
+	return nil
+}
+
 type StorableDecoder func(
 	decoder *cbor.StreamDecoder,
 	storableSlabID SlabID,
+	inlinedExtraData []ExtraData,
 ) (
 	Storable,
 	error,
@@ -101,7 +127,7 @@ func DecodeSlab(
 
 	case slabStorable:
 		cborDec := decMode.NewByteStreamDecoder(data[versionAndFlagSize:])
-		storable, err := decodeStorable(cborDec, id)
+		storable, err := decodeStorable(cborDec, id, nil)
 		if err != nil {
 			// Wrap err as external error (if needed) because err is returned by StorableDecoder callback.
 			return nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to decode slab storable")
@@ -116,7 +142,6 @@ func DecodeSlab(
 	}
 }
 
-// TODO: make it inline
 func GetUintCBORSize(n uint64) uint32 {
 	if n <= 23 {
 		return 1