diff --git a/array.go b/array.go index c0a2be0..fc385f4 100644 --- a/array.go +++ b/array.go @@ -298,16 +298,20 @@ func (a *ArrayExtraData) isExtraData() bool { return true } +func (a *ArrayExtraData) Type() TypeInfo { + return a.TypeInfo +} + // Encode encodes extra data as CBOR array: // // [type info] -func (a *ArrayExtraData) Encode(enc *Encoder) error { +func (a *ArrayExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error { err := enc.CBOR.EncodeArrayHead(arrayExtraDataLength) if err != nil { return NewEncodingError(err) } - err = a.TypeInfo.Encode(enc.CBOR) + err = encodeTypeInfo(enc, a.TypeInfo) if err != nil { // Wrap err as external error (if needed) because err is returned by TypeInfo interface. return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info") @@ -840,7 +844,8 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error { // Encode extra data if a.extraData != nil { - err = a.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = a.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // err is already categorized by ArrayExtraData.Encode(). return err @@ -1729,7 +1734,8 @@ func (a *ArrayMetaDataSlab) Encode(enc *Encoder) error { // Encode extra data if present if a.extraData != nil { - err = a.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = a.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // Don't need to wrap because err is already categorized by ArrayExtraData.Encode(). return err diff --git a/array_debug.go b/array_debug.go index eb0d2fe..7ffa335 100644 --- a/array_debug.go +++ b/array_debug.go @@ -861,12 +861,27 @@ func hasInlinedComposite(data []byte) (bool, error) { // Parse inlined extra data to find compact map extra data. dec := cbor.NewStreamDecoder(bytes.NewBuffer(data)) + count, err := dec.DecodeArrayHead() if err != nil { return false, NewDecodingError(err) } + if count != inlinedExtraDataArrayCount { + return false, NewDecodingError(fmt.Errorf("failed to decode inlined extra data, expect %d elements, got %d elements", inlinedExtraDataArrayCount, count)) + } - for i := uint64(0); i < count; i++ { + // Skip element 0 (inlined type info) + err = dec.Skip() + if err != nil { + return false, NewDecodingError(err) + } + + // Decoding element 1 (inlined extra data) + extraDataCount, err := dec.DecodeArrayHead() + if err != nil { + return false, NewDecodingError(err) + } + for i := uint64(0); i < extraDataCount; i++ { tagNum, err := dec.DecodeTagNumber() if err != nil { return false, NewDecodingError(err) diff --git a/array_test.go b/array_test.go index 09304fb..41830fd 100644 --- a/array_test.go +++ b/array_test.go @@ -3185,10 +3185,15 @@ func TestArrayEncodeDecode(t *testing.T) { 0x18, 0x2a, // inlined extra data + 0x82, + // element 0: array of type info + 0x80, + // element 1: array of extra data 0x81, - // inlined array extra data + // array extra data 0xd8, 0xf7, 0x81, + // array type info ref 0x18, 0x2b, // CBOR encoded array head (fixed size 3 byte) @@ -3267,6 +3272,10 @@ func TestArrayEncodeDecode(t *testing.T) { // inlined extra data 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data + 0x82, // inlined array extra data 0xd8, 0xf7, 0x81, @@ -3356,6 +3365,10 @@ func TestArrayEncodeDecode(t *testing.T) { // inlined extra data 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data + 0x82, // inlined array extra data 0xd8, 0xf7, 0x81, @@ -3455,6 +3468,10 @@ func TestArrayEncodeDecode(t *testing.T) { 0x18, 0x2a, // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x84, // typeInfo3 0xd8, 0xf7, @@ -3596,6 +3613,10 @@ func TestArrayEncodeDecode(t *testing.T) { // array data slab flag 0x00, // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // inlined array extra data 0xd8, 0xf7, @@ -3745,6 +3766,10 @@ func TestArrayEncodeDecode(t *testing.T) { 0x00, // inlined extra data 0x82, + // element 0: array of inlined extra data + 0x80, + // element 1: array of inlined extra data + 0x82, // inlined array extra data 0xd8, 0xf7, 0x81, @@ -4065,7 +4090,11 @@ func TestArrayEncodeDecode(t *testing.T) { // array data slab flag (has pointer) 0x40, - // inlined array of extra data + // inlined extra data + 0x82, + // element 0: array of type info + 0x80, + // element 1: array of extra data 0x81, // type info 0xd8, 0xf7, diff --git a/map.go b/map.go index d74e957..9b086fa 100644 --- a/map.go +++ b/map.go @@ -494,17 +494,21 @@ func (m *MapExtraData) isExtraData() bool { return true } +func (m *MapExtraData) Type() TypeInfo { + return m.TypeInfo +} + // Encode encodes extra data as CBOR array: // // [type info, count, seed] -func (m *MapExtraData) Encode(enc *Encoder) error { +func (m *MapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error { err := enc.CBOR.EncodeArrayHead(mapExtraDataLength) if err != nil { return NewEncodingError(err) } - err = m.TypeInfo.Encode(enc.CBOR) + err = encodeTypeInfo(enc, m.TypeInfo) if err != nil { // Wrap err as external error (if needed) because err is returned by TypeInfo interface. return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info") @@ -2916,7 +2920,8 @@ func (m *MapDataSlab) Encode(enc *Encoder) error { // Encode extra data if m.extraData != nil { - err = m.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = m.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode(). return err @@ -3909,7 +3914,8 @@ func (m *MapMetaDataSlab) Encode(enc *Encoder) error { // Encode extra data if present if m.extraData != nil { - err = m.extraData.Encode(enc) + // Use defaultEncodeTypeInfo to encode root level TypeInfo as is. + err = m.extraData.Encode(enc, defaultEncodeTypeInfo) if err != nil { // Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode(). return err diff --git a/map_test.go b/map_test.go index d951aa9..344ccdd 100644 --- a/map_test.go +++ b/map_test.go @@ -7661,7 +7661,11 @@ func TestMapEncodeDecode(t *testing.T) { // flag: has inlined slab + map data 0x08, - // inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // inlined array extra data 0xd8, 0xf7, @@ -7724,7 +7728,7 @@ func TestMapEncodeDecode(t *testing.T) { require.Equal(t, 2, len(meta.childrenHeaders)) require.Equal(t, uint32(len(stored[id2])), meta.childrenHeaders[0].size) - const inlinedExtraDataSize = 6 + const inlinedExtraDataSize = 8 require.Equal(t, uint32(len(stored[id3])-inlinedExtraDataSize+slabIDSize), meta.childrenHeaders[1].size) // Decode data to new storage @@ -7804,24 +7808,30 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x82, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed 0x1b, 0xa9, 0x3a, 0x2d, 0x6f, 0x53, 0x49, 0xaa, 0xdd, - // element 1 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -7993,13 +8003,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x82, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2c, // count: 1 0x01, @@ -8009,7 +8022,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -8184,14 +8196,20 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8201,7 +8219,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8211,7 +8230,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8221,7 +8241,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8454,13 +8475,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of type info + 0x80, + // element 1: array of extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 44 0x18, 0x2c, // count: 1 0x01, @@ -8471,7 +8495,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 46 0x18, 0x2e, // count: 1 0x01, @@ -8482,7 +8505,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 43 0x18, 0x2b, // count: 1 0x01, @@ -8493,7 +8515,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info: 45 0x18, 0x2d, // count: 1 0x01, @@ -8721,14 +8742,20 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8738,7 +8765,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8748,7 +8776,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8757,7 +8786,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8904,14 +8934,20 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0x18, 0x2b, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8921,7 +8957,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8931,7 +8968,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -8940,7 +8978,8 @@ func TestMapEncodeDecode(t *testing.T) { 0xd8, 0xf8, 0x83, // type info - 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -9202,13 +9241,16 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -9218,7 +9260,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2c, // count: 1 0x01, @@ -9228,7 +9269,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2d, // count: 1 0x01, @@ -9237,7 +9277,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2e, // count: 1 0x01, @@ -9385,13 +9424,16 @@ func TestMapEncodeDecode(t *testing.T) { // flag: map data 0x08, - // 4 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x84, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -9401,7 +9443,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2c, // count: 1 0x01, @@ -9411,7 +9452,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2d, // count: 1 0x01, @@ -9420,7 +9460,6 @@ func TestMapEncodeDecode(t *testing.T) { // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2e, // count: 1 0x01, @@ -10511,13 +10550,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // array of inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined map extra data 0xd8, 0xf8, 0x83, - // type info 0x18, 0x2b, // count: 1 0x01, @@ -10986,13 +11028,16 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // array of inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined array extra data 0xd8, 0xf7, 0x81, - // type info 0x18, 0x2b, // the following encoded data is valid CBOR @@ -11281,7 +11326,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 1 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined composite extra data @@ -11449,7 +11498,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 1 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined composite extra data @@ -11624,7 +11677,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 1 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x81, // element 0 // inlined composite extra data @@ -11814,7 +11871,12 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 3 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0xd8, 0xf6, 0x18, 0x2b, + // element 1: array of inlined extra data 0x83, // element 0 // inlined composite extra data @@ -11823,7 +11885,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -11842,7 +11905,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -11861,7 +11925,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -12050,7 +12115,12 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x81, + 0xd8, 0xf6, 0x18, 0x2b, + // element 1: array of inlined extra data 0x82, // element 0 // inlined map extra data @@ -12059,7 +12129,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 2 0x02, // seed @@ -12077,7 +12148,8 @@ func TestMapEncodeDecode(t *testing.T) { // map extra data 0x83, // type info - 0xd8, 0xf6, 0x18, 0x2b, + 0xd8, 0xf6, + 0x00, // count: 1 0x01, // seed @@ -12248,7 +12320,11 @@ func TestMapEncodeDecode(t *testing.T) { // seed 0x1b, 0x52, 0xa8, 0x78, 0x3, 0x85, 0x2c, 0xaa, 0x49, - // 2 inlined slab extra data + // inlined extra data + 0x82, + // element 0: array of inlined type info + 0x80, + // element 1: array of inlined extra data 0x82, // element 0 // inlined composite extra data diff --git a/storable.go b/storable.go index a59529c..634c457 100644 --- a/storable.go +++ b/storable.go @@ -76,6 +76,8 @@ const ( // As of Oct. 2, 2023, Cadence uses tag numbers from 128 to 224. // See runtime/interpreter/encode.go at github.com/onflow/cadence. + CBORTagTypeInfoRef = 246 + CBORTagInlinedArrayExtraData = 247 CBORTagInlinedMapExtraData = 248 CBORTagInlinedCompactMapExtraData = 249 diff --git a/storage.go b/storage.go index 9e4eb7d..3f79d40 100644 --- a/storage.go +++ b/storage.go @@ -910,7 +910,7 @@ func (s *PersistentSlabStorage) FastCommit(numWorkers int) error { // process the results while encoders are working // we need to capture them inside a map // again so we can apply them in order of keys - encSlabByID := make(map[SlabID][]byte) + encSlabByID := make(map[SlabID][]byte, len(keysWithOwners)) for i := 0; i < len(keysWithOwners); i++ { result := <-results // if any error return diff --git a/typeinfo.go b/typeinfo.go index 86c9fe6..61cdcc4 100644 --- a/typeinfo.go +++ b/typeinfo.go @@ -19,6 +19,7 @@ package atree import ( + "bytes" "encoding/binary" "fmt" "sort" @@ -41,9 +42,60 @@ type TypeInfoDecoder func( error, ) +// encodeTypeInfo encodes TypeInfo either: +// - as is (for TypeInfo in root slab extra data section), or +// - as index of inlined TypeInfos (for TypeInfo in inlined slab extra data section) +type encodeTypeInfo func(*Encoder, TypeInfo) error + +// defaultEncodeTypeInfo encodes TypeInfo as is. +func defaultEncodeTypeInfo(enc *Encoder, typeInfo TypeInfo) error { + return typeInfo.Encode(enc.CBOR) +} + +func decodeTypeInfoRefIfNeeded(inlinedTypeInfo []TypeInfo, defaultTypeInfoDecoder TypeInfoDecoder) TypeInfoDecoder { + if len(inlinedTypeInfo) == 0 { + return defaultTypeInfoDecoder + } + + return func(decoder *cbor.StreamDecoder) (TypeInfo, error) { + rawTypeInfo, err := decoder.DecodeRawBytes() + if err != nil { + return nil, NewDecodingError(fmt.Errorf("failed to decode raw type info: %w", err)) + } + + if len(rawTypeInfo) > len(typeInfoRefTagHeadAndTagNumber) && + bytes.Equal( + rawTypeInfo[:len(typeInfoRefTagHeadAndTagNumber)], + typeInfoRefTagHeadAndTagNumber) { + + // Type info is encoded as type info ref. + + var index uint64 + + err = cbor.Unmarshal(rawTypeInfo[len(typeInfoRefTagHeadAndTagNumber):], &index) + if err != nil { + return nil, NewDecodingError(err) + } + + if index >= uint64(len(inlinedTypeInfo)) { + return nil, NewDecodingError(fmt.Errorf("failed to decode type info ref: expect index < %d, got %d", len(inlinedTypeInfo), index)) + } + + return inlinedTypeInfo[int(index)], nil + } + + // Decode type info as is. + + dec := cbor.NewByteStreamDecoder(rawTypeInfo) + + return defaultTypeInfoDecoder(dec) + } +} + type ExtraData interface { isExtraData() bool - Encode(enc *Encoder) error + Type() TypeInfo + Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error } // compactMapExtraData is used for inlining compact values. @@ -64,14 +116,18 @@ func (c *compactMapExtraData) isExtraData() bool { return true } -func (c *compactMapExtraData) Encode(enc *Encoder) error { +func (c *compactMapExtraData) Type() TypeInfo { + return c.mapExtraData.TypeInfo +} + +func (c *compactMapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error { err := enc.CBOR.EncodeArrayHead(compactMapExtraDataLength) if err != nil { return NewEncodingError(err) } // element 0: map extra data - err = c.mapExtraData.Encode(enc) + err = c.mapExtraData.Encode(enc, encodeTypeInfo) if err != nil { return err } @@ -201,25 +257,60 @@ type compactMapTypeInfo struct { } type InlinedExtraData struct { - extraData []ExtraData - compactMapTypes map[string]compactMapTypeInfo - arrayTypes map[string]int + extraData []ExtraData // Used to encode deduplicated ExtraData in order + compactMapTypeSet map[string]compactMapTypeInfo // Used to deduplicate compactMapExtraData by TypeInfo.Identifier() + sorted field names + arrayExtraDataSet map[string]int // Used to deduplicate arrayExtraData by TypeInfo.Identifier() } func newInlinedExtraData() *InlinedExtraData { + // Maps used for deduplication are initialized lazily. return &InlinedExtraData{} } -// Encode encodes inlined extra data as CBOR array. +const inlinedExtraDataArrayCount = 2 + +var typeInfoRefTagHeadAndTagNumber = []byte{0xd8, CBORTagTypeInfoRef} + +// Encode encodes inlined extra data as 2-element array: +// +// +-----------------------+------------------------+ +// | [+ inlined type info] | [+ inlined extra data] | +// +-----------------------+------------------------+ func (ied *InlinedExtraData) Encode(enc *Encoder) error { - err := enc.CBOR.EncodeArrayHead(uint64(len(ied.extraData))) + + typeInfos, typeInfoIndexes := findDuplicateTypeInfo(ied.extraData) + + var err error + + err = enc.CBOR.EncodeArrayHead(inlinedExtraDataArrayCount) + if err != nil { + return NewEncodingError(err) + } + + // element 0: array of duplicate type info + err = enc.CBOR.EncodeArrayHead(uint64(len(typeInfos))) if err != nil { return NewEncodingError(err) } - var tagNum uint64 + // Encode type info + for _, typeInfo := range typeInfos { + err = typeInfo.Encode(enc.CBOR) + if err != nil { + return NewEncodingError(err) + } + } + // element 1: deduplicated array of extra data + err = enc.CBOR.EncodeArrayHead(uint64(len(ied.extraData))) + if err != nil { + return NewEncodingError(err) + } + + // Encode inlined extra data for _, extraData := range ied.extraData { + var tagNum uint64 + switch extraData.(type) { case *ArrayExtraData: tagNum = CBORTagInlinedArrayExtraData @@ -239,7 +330,29 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error { return NewEncodingError(err) } - err = extraData.Encode(enc) + err = extraData.Encode(enc, func(enc *Encoder, typeInfo TypeInfo) error { + index, exist := typeInfoIndexes[typeInfo.Identifier()] + if !exist { + // typeInfo is not encoded separately, so encode typeInfo as is here. + err = typeInfo.Encode(enc.CBOR) + if err != nil { + return NewEncodingError(err) + } + return nil + } + + err := enc.CBOR.EncodeRawBytes(typeInfoRefTagHeadAndTagNumber) + if err != nil { + return NewEncodingError(err) + } + + err = enc.CBOR.EncodeUint64(uint64(index)) + if err != nil { + return NewEncodingError(err) + } + + return nil + }) if err != nil { return err } @@ -253,11 +366,65 @@ func (ied *InlinedExtraData) Encode(enc *Encoder) error { return nil } +func findDuplicateTypeInfo(extraData []ExtraData) ([]TypeInfo, map[string]int) { + if len(extraData) < 2 { + // No duplicate type info + return nil, nil + } + + // typeInfoSet is used to deduplicate TypeInfo. + // typeInfoSet key: TypeInfo.Identifier() + // typeInfoSet value: indexes of extra data containing this type info + typeInfoSet := make(map[string][]int, len(extraData)) + + for i, data := range extraData { + typeID := data.Type().Identifier() + + indexes := typeInfoSet[typeID] + typeInfoSet[typeID] = append(indexes, i) + } + + if len(extraData) == len(typeInfoSet) { + // No duplicate type info + return nil, nil + } + + firstExtraDataIndexContainingDuplicateTypeInfo := make([]int, 0, len(typeInfoSet)) + for _, v := range typeInfoSet { + if len(v) > 1 { + firstExtraDataIndexContainingDuplicateTypeInfo = append(firstExtraDataIndexContainingDuplicateTypeInfo, v[0]) + } + } + + switch len(firstExtraDataIndexContainingDuplicateTypeInfo) { + case 1: + extraDataIndex := firstExtraDataIndexContainingDuplicateTypeInfo[0] + typeInfo := extraData[extraDataIndex].Type() + return []TypeInfo{typeInfo}, map[string]int{typeInfo.Identifier(): 0} + + default: + sort.Ints(firstExtraDataIndexContainingDuplicateTypeInfo) + + typeInfos := make([]TypeInfo, 0, len(firstExtraDataIndexContainingDuplicateTypeInfo)) + typeInfoIndexes := make(map[string]int) + + for _, extraDataIndex := range firstExtraDataIndexContainingDuplicateTypeInfo { + index := len(typeInfos) + + typeInfo := extraData[extraDataIndex].Type() + typeInfos = append(typeInfos, typeInfo) + typeInfoIndexes[typeInfo.Identifier()] = index + } + + return typeInfos, typeInfoIndexes + } +} + func newInlinedExtraDataFromData( data []byte, decMode cbor.DecMode, decodeStorable StorableDecoder, - decodeTypeInfo TypeInfoDecoder, + defaultDecodeTypeInfo TypeInfoDecoder, ) ([]ExtraData, []byte, error) { dec := decMode.NewByteStreamDecoder(data) @@ -267,12 +434,38 @@ func newInlinedExtraDataFromData( return nil, nil, NewDecodingError(err) } - if count == 0 { + if count != inlinedExtraDataArrayCount { + return nil, nil, NewDecodingError(fmt.Errorf("failed to decode inlined extra data: expect %d elements, got %d elements", inlinedExtraDataArrayCount, count)) + } + + // element 0: array of duplicate type info + typeInfoCount, err := dec.DecodeArrayHead() + if err != nil { + return nil, nil, NewDecodingError(err) + } + + inlinedTypeInfo := make([]TypeInfo, int(typeInfoCount)) + for i := uint64(0); i < typeInfoCount; i++ { + inlinedTypeInfo[i], err = defaultDecodeTypeInfo(dec) + if err != nil { + return nil, nil, wrapErrorfAsExternalErrorIfNeeded(err, "failed to decode typeInfo") + } + } + + decodeTypeInfo := decodeTypeInfoRefIfNeeded(inlinedTypeInfo, defaultDecodeTypeInfo) + + // element 1: array of deduplicated extra data info + extraDataCount, err := dec.DecodeArrayHead() + if err != nil { + return nil, nil, NewDecodingError(err) + } + + if extraDataCount == 0 { return nil, nil, NewDecodingError(fmt.Errorf("failed to decode inlined extra data: expect at least one inlined extra data")) } - inlinedExtraData := make([]ExtraData, count) - for i := uint64(0); i < count; i++ { + inlinedExtraData := make([]ExtraData, extraDataCount) + for i := uint64(0); i < extraDataCount; i++ { tagNum, err := dec.DecodeTagNumber() if err != nil { return nil, nil, NewDecodingError(err) @@ -309,19 +502,20 @@ func newInlinedExtraDataFromData( // Array extra data is deduplicated by array type info ID because array // extra data only contains type info. func (ied *InlinedExtraData) addArrayExtraData(data *ArrayExtraData) int { - if ied.arrayTypes == nil { - ied.arrayTypes = make(map[string]int) + if ied.arrayExtraDataSet == nil { + ied.arrayExtraDataSet = make(map[string]int) } id := data.TypeInfo.Identifier() - index, exist := ied.arrayTypes[id] + index, exist := ied.arrayExtraDataSet[id] if exist { return index } index = len(ied.extraData) ied.extraData = append(ied.extraData, data) - ied.arrayTypes[id] = index + ied.arrayExtraDataSet[id] = index + return index } @@ -341,12 +535,12 @@ func (ied *InlinedExtraData) addCompactMapExtraData( keys []ComparableStorable, ) (int, []ComparableStorable) { - if ied.compactMapTypes == nil { - ied.compactMapTypes = make(map[string]compactMapTypeInfo) + if ied.compactMapTypeSet == nil { + ied.compactMapTypeSet = make(map[string]compactMapTypeInfo) } id := makeCompactMapTypeID(data.TypeInfo, keys) - info, exist := ied.compactMapTypes[id] + info, exist := ied.compactMapTypeSet[id] if exist { return info.index, info.keys } @@ -360,7 +554,7 @@ func (ied *InlinedExtraData) addCompactMapExtraData( index := len(ied.extraData) ied.extraData = append(ied.extraData, compactMapData) - ied.compactMapTypes[id] = compactMapTypeInfo{ + ied.compactMapTypeSet[id] = compactMapTypeInfo{ keys: keys, index: index, }