Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce RAM and persistent storage by deduplicating inlined dict type info #369

Merged
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Deduplicate inlined type info if repeated
fxamacker committed Mar 8, 2024
commit 35fdb7e4e109b999d627cafe9ff89df86e21dd24
4 changes: 4 additions & 0 deletions array.go
Original file line number Diff line number Diff line change
@@ -298,6 +298,10 @@ func (a *ArrayExtraData) isExtraData() bool {
return true
}

func (a *ArrayExtraData) Type() TypeInfo {
return a.TypeInfo
}

// Encode encodes extra data as CBOR array:
//
// [type info]
67 changes: 20 additions & 47 deletions array_test.go
Original file line number Diff line number Diff line change
@@ -3186,17 +3186,14 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of type info
0x81,
// type info
0x18, 0x2b,
0x80,
// element 1: array of extra data
0x81,
// array extra data
0xd8, 0xf7,
0x81,
// array type info ref
0xd8, 0xf6,
0x00,
0x18, 0x2b,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
@@ -3275,21 +3272,17 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x82,
0x18, 0x2c,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2c,
// inlined array extra data
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x01,
0x18, 0x2b,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
@@ -3372,20 +3365,16 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x82,
0x18, 0x2c,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2c,
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x01,
0x18, 0x2b,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
@@ -3480,33 +3469,25 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x84,
0x18, 0x2c,
0x18, 0x2b,
0x18, 0x2e,
0x18, 0x2d,
0x80,
// element 1: array of inlined extra data
0x84,
// typeInfo3
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2c,
// typeInfo2
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x01,
0x18, 0x2b,
// typeInfo5
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x02,
0x18, 0x2e,
// typeInfo4
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x03,
0x18, 0x2d,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x02,
@@ -3633,15 +3614,13 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// inlined array extra data
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2b,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x0b,
// CBOR encoded array elements
@@ -3787,20 +3766,16 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined extra data
0x82,
0x18, 0x2c,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2c,
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x01,
0x18, 0x2b,
// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x0b,
// CBOR encoded array elements
@@ -4117,15 +4092,13 @@ func TestArrayEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of type info
0x81,
0x18, 0x2c,
0x80,
// element 1: array of extra data
0x81,
// type info
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2c,

// CBOR encoded array head (fixed size 3 byte)
0x99, 0x00, 0x0b,
4 changes: 4 additions & 0 deletions map.go
Original file line number Diff line number Diff line change
@@ -494,6 +494,10 @@ func (m *MapExtraData) isExtraData() bool {
return true
}

func (m *MapExtraData) Type() TypeInfo {
return m.TypeInfo
}

// Encode encodes extra data as CBOR array:
//
// [type info, count, seed]
129 changes: 34 additions & 95 deletions map_test.go
Original file line number Diff line number Diff line change
@@ -7663,15 +7663,13 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// inlined array extra data
0xd8, 0xf7,
0x81,
0xd8, 0xf6,
0x00,
0x18, 0x2b,

// the following encoded data is valid CBOR

@@ -7729,7 +7727,7 @@ func TestMapEncodeDecode(t *testing.T) {
require.Equal(t, 2, len(meta.childrenHeaders))
require.Equal(t, uint32(len(stored[id2])), meta.childrenHeaders[0].size)

const inlinedExtraDataSize = 11
const inlinedExtraDataSize = 8
require.Equal(t, uint32(len(stored[id3])-inlinedExtraDataSize+slabIDSize), meta.childrenHeaders[1].size)

// Decode data to new storage
@@ -7827,7 +7825,6 @@ func TestMapEncodeDecode(t *testing.T) {
0x01,
// seed
0x1b, 0xa9, 0x3a, 0x2d, 0x6f, 0x53, 0x49, 0xaa, 0xdd,
// element 1
// inlined map extra data
0xd8, 0xf8,
0x83,
@@ -8008,18 +8005,14 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x82,
0x18, 0x2c,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x82,
// element 0
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x00,
0x18, 0x2c,
// count: 1
0x01,
// seed
@@ -8028,9 +8021,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x01,
0x18, 0x2b,
// count: 1
0x01,
// seed
@@ -8486,20 +8477,14 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of type info
0x84,
0x18, 0x2c,
0x18, 0x2e,
0x18, 0x2b,
0x18, 0x2d,
0x80,
// element 1: array of extra data
0x84,
// element 0
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info: 44
0xd8, 0xf6,
0x00,
0x18, 0x2c,
// count: 1
0x01,
// seed
@@ -8509,9 +8494,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info: 46
0xd8, 0xf6,
0x01,
0x18, 0x2e,
// count: 1
0x01,
// seed
@@ -8521,9 +8504,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info: 43
0xd8, 0xf6,
0x02,
0x18, 0x2b,
// count: 1
0x01,
// seed
@@ -8533,9 +8514,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info: 45
0xd8, 0xf6,
0x03,
0x18, 0x2d,
// count: 1
0x01,
// seed
@@ -9264,20 +9243,14 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x84,
0x18, 0x2b,
0x18, 0x2c,
0x18, 0x2d,
0x18, 0x2e,
0x80,
// element 1: array of inlined extra data
0x84,
// element 0
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x00,
0x18, 0x2b,
// count: 1
0x01,
// seed
@@ -9286,9 +9259,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x01,
0x18, 0x2c,
// count: 1
0x01,
// seed
@@ -9297,19 +9268,15 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x02,
0x18, 0x2d,
// count: 1
0x01,
// seed
0x1b, 0x8d, 0x99, 0xcc, 0x54, 0xc8, 0x6b, 0xab, 0x50,
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x03,
0x18, 0x2e,
// count: 1
0x01,
// seed
@@ -9459,20 +9426,14 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x84,
0x18, 0x2b,
0x18, 0x2c,
0x18, 0x2d,
0x18, 0x2e,
0x80,
// element 1: array of inlined extra data
0x84,
// element 0
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x00,
0x18, 0x2b,
// count: 1
0x01,
// seed
@@ -9481,9 +9442,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x01,
0x18, 0x2c,
// count: 1
0x01,
// seed
@@ -9492,19 +9451,15 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x02,
0x18, 0x2d,
// count: 1
0x01,
// seed
0x1b, 0x79, 0xb3, 0x45, 0x84, 0x9e, 0x66, 0xa5, 0xa4,
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x03,
0x18, 0x2e,
// count: 1
0x01,
// seed
@@ -10597,17 +10552,14 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// element 0
// inlined map extra data
0xd8, 0xf8,
0x83,
// type info
0xd8, 0xf6,
0x00,
0x18, 0x2b,
// count: 1
0x01,
// seed
@@ -11078,17 +11030,14 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// element 0
// inlined array extra data
0xd8, 0xf7,
0x81,
// type info
0xd8, 0xf6,
0x00,
0x18, 0x2b,

// the following encoded data is valid CBOR

@@ -11379,8 +11328,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0xd8, 0xf6, 0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// element 0
@@ -11390,8 +11338,7 @@ func TestMapEncodeDecode(t *testing.T) {
// map extra data
0x83,
// type info
0xd8, 0xf6,
0x00,
0xd8, 0xf6, 0x18, 0x2b,
// count
0x01,
// seed
@@ -11553,8 +11500,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0xd8, 0xf6, 0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// element 0
@@ -11564,8 +11510,7 @@ func TestMapEncodeDecode(t *testing.T) {
// map extra data
0x83,
// type info
0xd8, 0xf6,
0x00,
0xd8, 0xf6, 0x18, 0x2b,
// count: 2
0x02,
// seed
@@ -11734,8 +11679,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x81,
0xd8, 0xf6, 0x18, 0x2b,
0x80,
// element 1: array of inlined extra data
0x81,
// element 0
@@ -11745,8 +11689,7 @@ func TestMapEncodeDecode(t *testing.T) {
// map extra data
0x83,
// type info
0xd8, 0xf6,
0x00,
0xd8, 0xf6, 0x18, 0x2b,
// count: 2
0x02,
// seed
@@ -12379,9 +12322,7 @@ func TestMapEncodeDecode(t *testing.T) {
// inlined extra data
0x82,
// element 0: array of inlined type info
0x82,
0xd8, 0xf6, 0x18, 0x2b,
0xd8, 0xf6, 0x18, 0x2c,
0x80,
// element 1: array of inlined extra data
0x82,
// element 0
@@ -12391,8 +12332,7 @@ func TestMapEncodeDecode(t *testing.T) {
// map extra data
0x83,
// type info
0xd8, 0xf6,
0x00,
0xd8, 0xf6, 0x18, 0x2b,
// count: 2
0x02,
// seed
@@ -12410,8 +12350,7 @@ func TestMapEncodeDecode(t *testing.T) {
// map extra data
0x83,
// type info
0xd8, 0xf6,
0x01,
0xd8, 0xf6, 0x18, 0x2c,
// count: 2
0x02,
// seed
209 changes: 132 additions & 77 deletions typeinfo.go
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@
package atree

import (
"bytes"
"encoding/binary"
"fmt"
"sort"
@@ -51,8 +52,49 @@ func defaultEncodeTypeInfo(enc *Encoder, typeInfo TypeInfo) error {
return typeInfo.Encode(enc.CBOR)
}

func decodeTypeInfoRefIfNeeded(inlinedTypeInfo []TypeInfo, defaultTypeInfoDecoder TypeInfoDecoder) TypeInfoDecoder {
if len(inlinedTypeInfo) == 0 {
return defaultTypeInfoDecoder
}

return func(decoder *cbor.StreamDecoder) (TypeInfo, error) {
rawTypeInfo, err := decoder.DecodeRawBytes()
if err != nil {
return nil, NewDecodingError(fmt.Errorf("failed to decode raw type info: %w", err))
}

if len(rawTypeInfo) > len(typeInfoRefTagHeadAndTagNumber) &&
bytes.Equal(
rawTypeInfo[:len(typeInfoRefTagHeadAndTagNumber)],
typeInfoRefTagHeadAndTagNumber) {

// Type info is encoded as type info ref.

var index uint64

err = cbor.Unmarshal(rawTypeInfo[len(typeInfoRefTagHeadAndTagNumber):], &index)
if err != nil {
return nil, NewDecodingError(err)
}

if index >= uint64(len(inlinedTypeInfo)) {
return nil, NewDecodingError(fmt.Errorf("failed to decode type info ref: expect index < %d, got %d", len(inlinedTypeInfo), index))
}

return inlinedTypeInfo[int(index)], nil
}

// Decode type info as is.

dec := cbor.NewByteStreamDecoder(rawTypeInfo)

return defaultTypeInfoDecoder(dec)
}
}

type ExtraData interface {
isExtraData() bool
Type() TypeInfo
Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error
}

@@ -74,6 +116,10 @@ func (c *compactMapExtraData) isExtraData() bool {
return true
}

func (c *compactMapExtraData) Type() TypeInfo {
return c.mapExtraData.TypeInfo
}

func (c *compactMapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {
err := enc.CBOR.EncodeArrayHead(compactMapExtraDataLength)
if err != nil {
@@ -210,17 +256,10 @@ type compactMapTypeInfo struct {
keys []ComparableStorable
}

type extraDataInfo struct {
data ExtraData
typeInfoIndex int
}

type InlinedExtraData struct {
extraData []extraDataInfo // Used to encode deduplicated ExtraData in order
typeInfo []TypeInfo // Used to encode deduplicated TypeInfo in order
extraData []ExtraData // Used to encode deduplicated ExtraData in order
compactMapTypeSet map[string]compactMapTypeInfo // Used to deduplicate compactMapExtraData by TypeInfo.Identifier() + sorted field names
arrayExtraDataSet map[string]int // Used to deduplicate arrayExtraData by TypeInfo.Identifier()
typeInfoSet map[string]int // Used to deduplicate TypeInfo by TypeInfo.Identifier()
}

func newInlinedExtraData() *InlinedExtraData {
@@ -230,27 +269,32 @@ func newInlinedExtraData() *InlinedExtraData {

const inlinedExtraDataArrayCount = 2

var typeInfoRefTagHeadAndTagNumber = []byte{0xd8, CBORTagTypeInfoRef}

// Encode encodes inlined extra data as 2-element array:
//
// +-----------------------+------------------------+
// | [+ inlined type info] | [+ inlined extra data] |
// +-----------------------+------------------------+
func (ied *InlinedExtraData) Encode(enc *Encoder) error {

typeInfos, typeInfoIndexes := findDuplicateTypeInfo(ied.extraData)

var err error

err = enc.CBOR.EncodeArrayHead(inlinedExtraDataArrayCount)
if err != nil {
return NewEncodingError(err)
}

// element 0: deduplicated array of type info
err = enc.CBOR.EncodeArrayHead(uint64(len(ied.typeInfo)))
// element 0: array of duplicate type info
err = enc.CBOR.EncodeArrayHead(uint64(len(typeInfos)))
if err != nil {
return NewEncodingError(err)
}

// Encode inlined type info
for _, typeInfo := range ied.typeInfo {
// Encode type info
for _, typeInfo := range typeInfos {
err = typeInfo.Encode(enc.CBOR)
if err != nil {
return NewEncodingError(err)
@@ -267,7 +311,7 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error {
for _, extraData := range ied.extraData {
var tagNum uint64

switch extraData.data.(type) {
switch extraData.(type) {
case *ArrayExtraData:
tagNum = CBORTagInlinedArrayExtraData

@@ -286,14 +330,18 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error {
return NewEncodingError(err)
}

err = extraData.data.Encode(enc, func(enc *Encoder, typeInfo TypeInfo) error {
id := typeInfo.Identifier()
index, exist := ied.typeInfoSet[id]
err = extraData.Encode(enc, func(enc *Encoder, typeInfo TypeInfo) error {
index, exist := typeInfoIndexes[typeInfo.Identifier()]
if !exist {
return NewEncodingError(fmt.Errorf("failed to encode type info ref %s (%T)", id, typeInfo))
// typeInfo is not encoded separately, so encode typeInfo as is here.
err = typeInfo.Encode(enc.CBOR)
if err != nil {
return NewEncodingError(err)
}
return nil
}

err := enc.CBOR.EncodeTagHead(CBORTagTypeInfoRef)
err := enc.CBOR.EncodeRawBytes(typeInfoRefTagHeadAndTagNumber)
if err != nil {
return NewEncodingError(err)
}
@@ -318,11 +366,65 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error {
return nil
}

func findDuplicateTypeInfo(extraData []ExtraData) ([]TypeInfo, map[string]int) {
if len(extraData) < 2 {
// No duplicate type info
return nil, nil
}

// typeInfoSet is used to deduplicate TypeInfo.
// typeInfoSet key: TypeInfo.Identifier()
// typeInfoSet value: indexes of extra data containing this type info
typeInfoSet := make(map[string][]int, len(extraData))

for i, data := range extraData {
typeID := data.Type().Identifier()

indexes := typeInfoSet[typeID]
typeInfoSet[typeID] = append(indexes, i)
}

if len(extraData) == len(typeInfoSet) {
// No duplicate type info
return nil, nil
}

firstExtraDataIndexContainingDuplicateTypeInfo := make([]int, 0, len(typeInfoSet))
for _, v := range typeInfoSet {
if len(v) > 1 {
firstExtraDataIndexContainingDuplicateTypeInfo = append(firstExtraDataIndexContainingDuplicateTypeInfo, v[0])
}
}

switch len(firstExtraDataIndexContainingDuplicateTypeInfo) {
case 1:
extraDataIndex := firstExtraDataIndexContainingDuplicateTypeInfo[0]
typeInfo := extraData[extraDataIndex].Type()
return []TypeInfo{typeInfo}, map[string]int{typeInfo.Identifier(): 0}

default:
sort.Ints(firstExtraDataIndexContainingDuplicateTypeInfo)

typeInfos := make([]TypeInfo, 0, len(firstExtraDataIndexContainingDuplicateTypeInfo))
typeInfoIndexes := make(map[string]int)

for _, extraDataIndex := range firstExtraDataIndexContainingDuplicateTypeInfo {
index := len(typeInfos)

typeInfo := extraData[extraDataIndex].Type()
typeInfos = append(typeInfos, typeInfo)
typeInfoIndexes[typeInfo.Identifier()] = index
}

return typeInfos, typeInfoIndexes
}
}

func newInlinedExtraDataFromData(
data []byte,
decMode cbor.DecMode,
decodeStorable StorableDecoder,
decodeTypeInfo TypeInfoDecoder,
defaultDecodeTypeInfo TypeInfoDecoder,
) ([]ExtraData, []byte, error) {

dec := decMode.NewByteStreamDecoder(data)
@@ -336,43 +438,21 @@ func newInlinedExtraDataFromData(
return nil, nil, NewDecodingError(fmt.Errorf("failed to decode inlined extra data: expect %d elements, got %d elements", inlinedExtraDataArrayCount, count))
}

// element 0: array of deduplicated type info
// element 0: array of duplicate type info
typeInfoCount, err := dec.DecodeArrayHead()
if err != nil {
return nil, nil, NewDecodingError(err)
}

if typeInfoCount == 0 {
return nil, nil, NewDecodingError(fmt.Errorf("failed to decode inlined extra data: expect at least one inlined type info"))
}

inlinedTypeInfo := make([]TypeInfo, typeInfoCount)
inlinedTypeInfo := make([]TypeInfo, int(typeInfoCount))
for i := uint64(0); i < typeInfoCount; i++ {
inlinedTypeInfo[i], err = decodeTypeInfo(dec)
inlinedTypeInfo[i], err = defaultDecodeTypeInfo(dec)
if err != nil {
return nil, nil, err
return nil, nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to decode typeInfo")
}
}

typeInfoRefDecoder := func(decoder *cbor.StreamDecoder) (TypeInfo, error) {
tagNum, err := decoder.DecodeTagNumber()
if err != nil {
return nil, err
}
if tagNum != CBORTagTypeInfoRef {
return nil, NewDecodingError(fmt.Errorf("failed to decode type info ref: expect tag number %d, got %d", CBORTagTypeInfoRef, tagNum))
}

index, err := decoder.DecodeUint64()
if err != nil {
return nil, NewDecodingError(err)
}
if index >= uint64(len(inlinedTypeInfo)) {
return nil, NewDecodingError(fmt.Errorf("failed to decode type info ref: expect index < %d, got %d", len(inlinedTypeInfo), index))
}

return inlinedTypeInfo[int(index)], nil
}
decodeTypeInfo := decodeTypeInfoRefIfNeeded(inlinedTypeInfo, defaultDecodeTypeInfo)

// element 1: array of deduplicated extra data info
extraDataCount, err := dec.DecodeArrayHead()
@@ -393,19 +473,19 @@ func newInlinedExtraDataFromData(

switch tagNum {
case CBORTagInlinedArrayExtraData:
inlinedExtraData[i], err = newArrayExtraData(dec, typeInfoRefDecoder)
inlinedExtraData[i], err = newArrayExtraData(dec, decodeTypeInfo)
if err != nil {
return nil, nil, err
}

case CBORTagInlinedMapExtraData:
inlinedExtraData[i], err = newMapExtraData(dec, typeInfoRefDecoder)
inlinedExtraData[i], err = newMapExtraData(dec, decodeTypeInfo)
if err != nil {
return nil, nil, err
}

case CBORTagInlinedCompactMapExtraData:
inlinedExtraData[i], err = newCompactMapExtraData(dec, typeInfoRefDecoder, decodeStorable)
inlinedExtraData[i], err = newCompactMapExtraData(dec, decodeTypeInfo, decodeStorable)
if err != nil {
return nil, nil, err
}
@@ -418,25 +498,6 @@ func newInlinedExtraDataFromData(
return inlinedExtraData, data[dec.NumBytesDecoded():], nil
}

// addTypeInfo returns index of deduplicated type info.
func (ied *InlinedExtraData) addTypeInfo(typeInfo TypeInfo) int {
if ied.typeInfoSet == nil {
ied.typeInfoSet = make(map[string]int)
}

id := typeInfo.Identifier()
index, exist := ied.typeInfoSet[id]
if exist {
return index
}

index = len(ied.typeInfo)
ied.typeInfo = append(ied.typeInfo, typeInfo)
ied.typeInfoSet[id] = index

return index
}

// addArrayExtraData returns index of deduplicated array extra data.
// Array extra data is deduplicated by array type info ID because array
// extra data only contains type info.
@@ -451,10 +512,8 @@ func (ied *InlinedExtraData) addArrayExtraData(data *ArrayExtraData) int {
return index
}

typeInfoIndex := ied.addTypeInfo(data.TypeInfo)

index = len(ied.extraData)
ied.extraData = append(ied.extraData, extraDataInfo{data, typeInfoIndex})
ied.extraData = append(ied.extraData, data)
ied.arrayExtraDataSet[id] = index

return index
@@ -463,10 +522,8 @@ func (ied *InlinedExtraData) addArrayExtraData(data *ArrayExtraData) int {
// addMapExtraData returns index of map extra data.
// Map extra data is not deduplicated because it also contains count and seed.
func (ied *InlinedExtraData) addMapExtraData(data *MapExtraData) int {
typeInfoIndex := ied.addTypeInfo(data.TypeInfo)

index := len(ied.extraData)
ied.extraData = append(ied.extraData, extraDataInfo{data, typeInfoIndex})
ied.extraData = append(ied.extraData, data)
return index
}

@@ -494,10 +551,8 @@ func (ied *InlinedExtraData) addCompactMapExtraData(
keys: keys,
}

typeInfoIndex := ied.addTypeInfo(data.TypeInfo)

index := len(ied.extraData)
ied.extraData = append(ied.extraData, extraDataInfo{compactMapData, typeInfoIndex})
ied.extraData = append(ied.extraData, compactMapData)

ied.compactMapTypeSet[id] = compactMapTypeInfo{
keys: keys,