diff --git a/array.go b/array.go index d8f39487..69affd85 100644 --- a/array.go +++ b/array.go @@ -298,16 +298,20 @@ func (a *ArrayExtraData) isExtraData() bool { return true } +func (a *ArrayExtraData) Type() TypeInfo { + return a.TypeInfo +} + // Encode encodes extra data as CBOR array: // // [type info] -func (a *ArrayExtraData) Encode(enc *Encoder) error { +func (a *ArrayExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error { err := enc.CBOR.EncodeArrayHead(arrayExtraDataLength) if err != nil { return NewEncodingError(err) } - err = a.TypeInfo.Encode(enc.CBOR) + err = encodeTypeInfo(enc, a.TypeInfo) if err != nil { // Wrap err as external error (if needed) because err is returned by TypeInfo interface. return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info") @@ -840,7 +844,8 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error { // Encode extra data if a.extraData != nil { - err = a.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = a.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // err is already categorized by ArrayExtraData.Encode(). return err @@ -1738,7 +1743,8 @@ func (a *ArrayMetaDataSlab) Encode(enc *Encoder) error { // Encode extra data if present if a.extraData != nil { - err = a.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = a.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // Don't need to wrap because err is already categorized by ArrayExtraData.Encode(). return err diff --git a/array_debug.go b/array_debug.go index eb0d2fe7..7ffa335d 100644 --- a/array_debug.go +++ b/array_debug.go @@ -861,12 +861,27 @@ func hasInlinedComposite(data []byte) (bool, error) { // Parse inlined extra data to find compact map extra data. dec := cbor.NewStreamDecoder(bytes.NewBuffer(data)) + count, err := dec.DecodeArrayHead() if err != nil { return false, NewDecodingError(err) } + if count != inlinedExtraDataArrayCount { + return false, NewDecodingError(fmt.Errorf("failed to decode inlined extra data, expect %d elements, got %d elements", inlinedExtraDataArrayCount, count)) + } - for i := uint64(0); i < count; i++ { + // Skip element 0 (inlined type info) + err = dec.Skip() + if err != nil { + return false, NewDecodingError(err) + } + + // Decoding element 1 (inlined extra data) + extraDataCount, err := dec.DecodeArrayHead() + if err != nil { + return false, NewDecodingError(err) + } + for i := uint64(0); i < extraDataCount; i++ { tagNum, err := dec.DecodeTagNumber() if err != nil { return false, NewDecodingError(err) diff --git a/array_test.go b/array_test.go index 7c64d61c..8f42abac 100644 --- a/array_test.go +++ b/array_test.go @@ -3184,10 +3184,15 @@ func TestArrayEncodeDecode(t *testing.T) { 0x18, 0x2a, // inlined extra data + 0x82, + // element 0: array of type info + 0x80, + // element 1: array of extra data 0x81, - // inlined array extra data + // array extra data 0xd8, 0xf7, 0x81, + // array type info ref 0x18, 0x2b, // CBOR encoded array head (fixed size 3 byte) @@ -3266,6 +3271,10 @@ func TestArrayEncodeDecode(t *testing.T) { // inlined extra data 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data + 0x82, // inlined array extra data 0xd8, 0xf7, 0x81, @@ -3355,6 +3364,10 @@ func TestArrayEncodeDecode(t *testing.T) { // inlined extra data 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data + 0x82, // inlined array extra data 0xd8, 0xf7, 0x81, @@ -3454,6 +3467,10 @@ func TestArrayEncodeDecode(t *testing.T) { 0x18, 0x2a, // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x84, // typeInfo3 0xd8, 0xf7, @@ -3595,6 +3612,10 @@ func TestArrayEncodeDecode(t *testing.T) { // array data slab flag 0x00, // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // inlined array extra data 0xd8, 0xf7, @@ -3744,6 +3765,10 @@ func TestArrayEncodeDecode(t *testing.T) { 0x00, // inlined extra data 0x82, + // element 0: array of inlined extra data + 0x80, + // element 1: array of inlined extra data + 0x82, // inlined array extra data 0xd8, 0xf7, 0x81, @@ -4064,7 +4089,11 @@ func TestArrayEncodeDecode(t *testing.T) { // array data slab flag (has pointer) 0x40, - // inlined array of extra data + // inlined extra data + 0x82, + // element 0: array of type info + 0x80, + // element 1: array of extra data 0x81, // type info 0xd8, 0xf7, diff --git a/map.go b/map.go index 57891d7a..d34af66c 100644 --- a/map.go +++ b/map.go @@ -494,17 +494,21 @@ func (m *MapExtraData) isExtraData() bool { return true } +func (m *MapExtraData) Type() TypeInfo { + return m.TypeInfo +} + // Encode encodes extra data as CBOR array: // // [type info, count, seed] -func (m *MapExtraData) Encode(enc *Encoder) error { +func (m *MapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error { err := enc.CBOR.EncodeArrayHead(mapExtraDataLength) if err != nil { return NewEncodingError(err) } - err = m.TypeInfo.Encode(enc.CBOR) + err = encodeTypeInfo(enc, m.TypeInfo) if err != nil { // Wrap err as external error (if needed) because err is returned by TypeInfo interface. return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info") @@ -2917,7 +2921,8 @@ func (m *MapDataSlab) Encode(enc *Encoder) error { // Encode extra data if m.extraData != nil { - err = m.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = m.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode(). return err @@ -3918,7 +3923,8 @@ func (m *MapMetaDataSlab) Encode(enc *Encoder) error { // Encode extra data if present if m.extraData != nil { - err = m.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = m.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode(). return err diff --git a/map_test.go b/map_test.go index ba8ca3a0..397bdce1 100644 --- a/map_test.go +++ b/map_test.go @@ -7660,7 +7660,11 @@ func TestMapEncodeDecode(t *testing.T) { // flag: has inlined slab + map data 0x08, - // inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // inlined array extra data 0xd8, 0xf7, @@ -7723,7 +7727,7 @@ func TestMapEncodeDecode(t *testing.T) { require.Equal(t, 2, len(meta.childrenHeaders)) require.Equal(t, uint32(len(stored[id2])), meta.childrenHeaders[0].size) - const inlinedExtraDataSize = 6 + const inlinedExtraDataSize = 8 require.Equal(t, uint32(len(stored[id3])-inlinedExtraDataSize+slabIDSize), meta.childrenHeaders[1].size) // Decode data to new storage @@ -7803,24 +7807,30 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x82, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed 0x1b, 0xa9, 0x3a, 0x2d, 0x6f, 0x53, 0x49, 0xaa, 0xdd, - // element 1 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -7992,13 +8002,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x82, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2c, // count: 1 0x01, @@ -8008,7 +8021,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -8183,14 +8195,20 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8200,7 +8218,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8210,7 +8229,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8220,7 +8240,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8453,13 +8474,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of type info + 0x80, + // element 1: array of extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 44 0x18, 0x2c, // count: 1 0x01, @@ -8470,7 +8494,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 46 0x18, 0x2e, // count: 1 0x01, @@ -8481,7 +8504,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 43 0x18, 0x2b, // count: 1 0x01, @@ -8492,7 +8514,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 45 0x18, 0x2d, // count: 1 0x01, @@ -8720,14 +8741,20 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8737,7 +8764,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8747,7 +8775,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8756,7 +8785,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8903,14 +8933,20 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8920,7 +8956,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8930,7 +8967,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8939,7 +8977,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -9201,13 +9240,16 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -9217,7 +9259,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2c, // count: 1 0x01, @@ -9227,7 +9268,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2d, // count: 1 0x01, @@ -9236,7 +9276,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2e, // count: 1 0x01, @@ -9384,13 +9423,16 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -9400,7 +9442,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2c, // count: 1 0x01, @@ -9410,7 +9451,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2d, // count: 1 0x01, @@ -9419,7 +9459,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2e, // count: 1 0x01, @@ -10510,13 +10549,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // array of inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -10985,13 +11027,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // array of inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined array extra data 0xd8, 0xf7, 0x81, - // type info 0x18, 0x2b, // the following encoded data is valid CBOR @@ -11280,7 +11325,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 1 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined composite extra data @@ -11448,7 +11497,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 1 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined composite extra data @@ -11623,7 +11676,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 1 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined composite extra data @@ -11813,7 +11870,12 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 3 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0xd8, 0xf6, 0x18, 0x2b, + // element 1: array of inlined extra data 0x83, // element 0 // inlined composite extra data @@ -11822,7 +11884,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -11841,7 +11904,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -11860,7 +11924,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -12049,7 +12114,12 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0xd8, 0xf6, 0x18, 0x2b, + // element 1: array of inlined extra data 0x82, // element 0 // inlined map extra data @@ -12058,7 +12128,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -12076,7 +12147,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -12247,7 +12319,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x82, // element 0 // inlined composite extra data diff --git a/storable.go b/storable.go index a59529c8..634c4572 100644 --- a/storable.go +++ b/storable.go @@ -76,6 +76,8 @@ const ( // As of Oct. 2, 2023, Cadence uses tag numbers from 128 to 224. // See runtime/interpreter/encode.go at github.com/onflow/cadence. + CBORTagTypeInfoRef = 246 + CBORTagInlinedArrayExtraData = 247 CBORTagInlinedMapExtraData = 248 CBORTagInlinedCompactMapExtraData = 249 diff --git a/storage.go b/storage.go index 0b324381..934678ec 100644 --- a/storage.go +++ b/storage.go @@ -910,7 +910,7 @@ func (s *PersistentSlabStorage) FastCommit(numWorkers int) error { // process the results while encoders are working // we need to capture them inside a map // again so we can apply them in order of keys - encSlabByID := make(map[SlabID][]byte) + encSlabByID := make(map[SlabID][]byte, len(keysWithOwners)) for i := 0; i < len(keysWithOwners); i++ { result := <-results // if any error return diff --git a/typeinfo.go b/typeinfo.go index 86c9fe67..61cdcc49 100644 --- a/typeinfo.go +++ b/typeinfo.go @@ -19,6 +19,7 @@ package atree import ( + "bytes" "encoding/binary" "fmt" "sort" @@ -41,9 +42,60 @@ type TypeInfoDecoder func( error, ) +// encodeTypeInfo encodes TypeInfo either: +// - as is (for TypeInfo in root slab extra data section), or +// - as index of inlined TypeInfos (for TypeInfo in inlined slab extra data section) +type encodeTypeInfo func(*Encoder, TypeInfo) error + +// defaultEncodeTypeInfo encodes TypeInfo as is. +func defaultEncodeTypeInfo(enc *Encoder, typeInfo TypeInfo) error { + return typeInfo.Encode(enc.CBOR) +} + +func decodeTypeInfoRefIfNeeded(inlinedTypeInfo []TypeInfo, defaultTypeInfoDecoder TypeInfoDecoder) TypeInfoDecoder { + if len(inlinedTypeInfo) == 0 { + return defaultTypeInfoDecoder + } + + return func(decoder *cbor.StreamDecoder) (TypeInfo, error) { + rawTypeInfo, err := decoder.DecodeRawBytes() + if err != nil { + return nil, NewDecodingError(fmt.Errorf("failed to decode raw type info: %w", err)) + } + + if len(rawTypeInfo) > len(typeInfoRefTagHeadAndTagNumber) && + bytes.Equal( + rawTypeInfo[:len(typeInfoRefTagHeadAndTagNumber)], + typeInfoRefTagHeadAndTagNumber) { + + // Type info is encoded as type info ref. + + var index uint64 + + err = cbor.Unmarshal(rawTypeInfo[len(typeInfoRefTagHeadAndTagNumber):], &index) + if err != nil { + return nil, NewDecodingError(err) + } + + if index >= uint64(len(inlinedTypeInfo)) { + return nil, NewDecodingError(fmt.Errorf("failed to decode type info ref: expect index < %d, got %d", len(inlinedTypeInfo), index)) + } + + return inlinedTypeInfo[int(index)], nil + } + + // Decode type info as is. + + dec := cbor.NewByteStreamDecoder(rawTypeInfo) + + return defaultTypeInfoDecoder(dec) + } +} + type ExtraData interface { isExtraData() bool - Encode(enc *Encoder) error + Type() TypeInfo + Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error } // compactMapExtraData is used for inlining compact values. @@ -64,14 +116,18 @@ func (c *compactMapExtraData) isExtraData() bool { return true } -func (c *compactMapExtraData) Encode(enc *Encoder) error { +func (c *compactMapExtraData) Type() TypeInfo { + return c.mapExtraData.TypeInfo +} + +func (c *compactMapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error { err := enc.CBOR.EncodeArrayHead(compactMapExtraDataLength) if err != nil { return NewEncodingError(err) } // element 0: map extra data - err = c.mapExtraData.Encode(enc) + err = c.mapExtraData.Encode(enc, encodeTypeInfo) if err != nil { return err } @@ -201,25 +257,60 @@ type compactMapTypeInfo struct { } type InlinedExtraData struct { - extraData []ExtraData - compactMapTypes map[string]compactMapTypeInfo - arrayTypes map[string]int + extraData []ExtraData // Used to encode deduplicated ExtraData in order + compactMapTypeSet map[string]compactMapTypeInfo // Used to deduplicate compactMapExtraData by TypeInfo.Identifier() + sorted field names + arrayExtraDataSet map[string]int // Used to deduplicate arrayExtraData by TypeInfo.Identifier() } func newInlinedExtraData() *InlinedExtraData { + // Maps used for deduplication are initialized lazily. return &InlinedExtraData{} } -// Encode encodes inlined extra data as CBOR array. +const inlinedExtraDataArrayCount = 2 + +var typeInfoRefTagHeadAndTagNumber = []byte{0xd8, CBORTagTypeInfoRef} + +// Encode encodes inlined extra data as 2-element array: +// +// +-----------------------+------------------------+ +// | [+ inlined type info] | [+ inlined extra data] | +// +-----------------------+------------------------+ func (ied *InlinedExtraData) Encode(enc *Encoder) error { - err := enc.CBOR.EncodeArrayHead(uint64(len(ied.extraData))) + + typeInfos, typeInfoIndexes := findDuplicateTypeInfo(ied.extraData) + + var err error + + err = enc.CBOR.EncodeArrayHead(inlinedExtraDataArrayCount) + if err != nil { + return NewEncodingError(err) + } + + // element 0: array of duplicate type info + err = enc.CBOR.EncodeArrayHead(uint64(len(typeInfos))) if err != nil { return NewEncodingError(err) } - var tagNum uint64 + // Encode type info + for _, typeInfo := range typeInfos { + err = typeInfo.Encode(enc.CBOR) + if err != nil { + return NewEncodingError(err) + } + } + // element 1: deduplicated array of extra data + err = enc.CBOR.EncodeArrayHead(uint64(len(ied.extraData))) + if err != nil { + return NewEncodingError(err) + } + + // Encode inlined extra data for _, extraData := range ied.extraData { + var tagNum uint64 + switch extraData.(type) { case *ArrayExtraData: tagNum = CBORTagInlinedArrayExtraData @@ -239,7 +330,29 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error { return NewEncodingError(err) } - err = extraData.Encode(enc) + err = extraData.Encode(enc, func(enc *Encoder, typeInfo TypeInfo) error { + index, exist := typeInfoIndexes[typeInfo.Identifier()] + if !exist { + // typeInfo is not encoded separately, so encode typeInfo as is here. + err = typeInfo.Encode(enc.CBOR) + if err != nil { + return NewEncodingError(err) + } + return nil + } + + err := enc.CBOR.EncodeRawBytes(typeInfoRefTagHeadAndTagNumber) + if err != nil { + return NewEncodingError(err) + } + + err = enc.CBOR.EncodeUint64(uint64(index)) + if err != nil { + return NewEncodingError(err) + } + + return nil + }) if err != nil { return err } @@ -253,11 +366,65 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error { return nil } +func findDuplicateTypeInfo(extraData []ExtraData) ([]TypeInfo, map[string]int) { + if len(extraData) < 2 { + // No duplicate type info + return nil, nil + } + + // typeInfoSet is used to deduplicate TypeInfo. + // typeInfoSet key: TypeInfo.Identifier() + // typeInfoSet value: indexes of extra data containing this type info + typeInfoSet := make(map[string][]int, len(extraData)) + + for i, data := range extraData { + typeID := data.Type().Identifier() + + indexes := typeInfoSet[typeID] + typeInfoSet[typeID] = append(indexes, i) + } + + if len(extraData) == len(typeInfoSet) { + // No duplicate type info + return nil, nil + } + + firstExtraDataIndexContainingDuplicateTypeInfo := make([]int, 0, len(typeInfoSet)) + for _, v := range typeInfoSet { + if len(v) > 1 { + firstExtraDataIndexContainingDuplicateTypeInfo = append(firstExtraDataIndexContainingDuplicateTypeInfo, v[0]) + } + } + + switch len(firstExtraDataIndexContainingDuplicateTypeInfo) { + case 1: + extraDataIndex := firstExtraDataIndexContainingDuplicateTypeInfo[0] + typeInfo := extraData[extraDataIndex].Type() + return []TypeInfo{typeInfo}, map[string]int{typeInfo.Identifier(): 0} + + default: + sort.Ints(firstExtraDataIndexContainingDuplicateTypeInfo) + + typeInfos := make([]TypeInfo, 0, len(firstExtraDataIndexContainingDuplicateTypeInfo)) + typeInfoIndexes := make(map[string]int) + + for _, extraDataIndex := range firstExtraDataIndexContainingDuplicateTypeInfo { + index := len(typeInfos) + + typeInfo := extraData[extraDataIndex].Type() + typeInfos = append(typeInfos, typeInfo) + typeInfoIndexes[typeInfo.Identifier()] = index + } + + return typeInfos, typeInfoIndexes + } +} + func newInlinedExtraDataFromData( data []byte, decMode cbor.DecMode, decodeStorable StorableDecoder, - decodeTypeInfo TypeInfoDecoder, + defaultDecodeTypeInfo TypeInfoDecoder, ) ([]ExtraData, []byte, error) { dec := decMode.NewByteStreamDecoder(data) @@ -267,12 +434,38 @@ func newInlinedExtraDataFromData( return nil, nil, NewDecodingError(err) } - if count == 0 { + if count != inlinedExtraDataArrayCount { + return nil, nil, NewDecodingError(fmt.Errorf("failed to decode inlined extra data: expect %d elements, got %d elements", inlinedExtraDataArrayCount, count)) + } + + // element 0: array of duplicate type info + typeInfoCount, err := dec.DecodeArrayHead() + if err != nil { + return nil, nil, NewDecodingError(err) + } + + inlinedTypeInfo := make([]TypeInfo, int(typeInfoCount)) + for i := uint64(0); i < typeInfoCount; i++ { + inlinedTypeInfo[i], err = defaultDecodeTypeInfo(dec) + if err != nil { + return nil, nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to decode typeInfo") + } + } + + decodeTypeInfo := decodeTypeInfoRefIfNeeded(inlinedTypeInfo, defaultDecodeTypeInfo) + + // element 1: array of deduplicated extra data info + extraDataCount, err := dec.DecodeArrayHead() + if err != nil { + return nil, nil, NewDecodingError(err) + } + + if extraDataCount == 0 { return nil, nil, NewDecodingError(fmt.Errorf("failed to decode inlined extra data: expect at least one inlined extra data")) } - inlinedExtraData := make([]ExtraData, count) - for i := uint64(0); i < count; i++ { + inlinedExtraData := make([]ExtraData, extraDataCount) + for i := uint64(0); i < extraDataCount; i++ { tagNum, err := dec.DecodeTagNumber() if err != nil { return nil, nil, NewDecodingError(err) @@ -309,19 +502,20 @@ func newInlinedExtraDataFromData( // Array extra data is deduplicated by array type info ID because array // extra data only contains type info. func (ied *InlinedExtraData) addArrayExtraData(data *ArrayExtraData) int { - if ied.arrayTypes == nil { - ied.arrayTypes = make(map[string]int) + if ied.arrayExtraDataSet == nil { + ied.arrayExtraDataSet = make(map[string]int) } id := data.TypeInfo.Identifier() - index, exist := ied.arrayTypes[id] + index, exist := ied.arrayExtraDataSet[id] if exist { return index } index = len(ied.extraData) ied.extraData = append(ied.extraData, data) - ied.arrayTypes[id] = index + ied.arrayExtraDataSet[id] = index + return index } @@ -341,12 +535,12 @@ func (ied *InlinedExtraData) addCompactMapExtraData( keys []ComparableStorable, ) (int, []ComparableStorable) { - if ied.compactMapTypes == nil { - ied.compactMapTypes = make(map[string]compactMapTypeInfo) + if ied.compactMapTypeSet == nil { + ied.compactMapTypeSet = make(map[string]compactMapTypeInfo) } id := makeCompactMapTypeID(data.TypeInfo, keys) - info, exist := ied.compactMapTypes[id] + info, exist := ied.compactMapTypeSet[id] if exist { return info.index, info.keys } @@ -360,7 +554,7 @@ func (ied *InlinedExtraData) addCompactMapExtraData( index := len(ied.extraData) ied.extraData = append(ied.extraData, compactMapData) - ied.compactMapTypes[id] = compactMapTypeInfo{ + ied.compactMapTypeSet[id] = compactMapTypeInfo{ keys: keys, index: index, }