Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce RAM and persistent storage by deduplicating inlined dict type info #369

Merged
merged 3 commits into from
Mar 14, 2024
Merged

Reduce RAM and persistent storage by deduplicating inlined dict type info #369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
83d8870
Deduplicate inlined dict type info to reduce RAM
fxamacker Feb 28, 2024
6825950
Preallocate map in FastCommit
fxamacker Feb 28, 2024
35fdb7e
Deduplicate inlined type info if repeated
fxamacker Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions array.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,16 +298,20 @@ func (a *ArrayExtraData) isExtraData() bool {
return true
}

func (a *ArrayExtraData) Type() TypeInfo {
return a.TypeInfo
}

// Encode encodes extra data as CBOR array:
//
// [type info]
func (a *ArrayExtraData) Encode(enc *Encoder) error {
func (a *ArrayExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {
err := enc.CBOR.EncodeArrayHead(arrayExtraDataLength)
if err != nil {
return NewEncodingError(err)
}

err = a.TypeInfo.Encode(enc.CBOR)
err = encodeTypeInfo(enc, a.TypeInfo)
if err != nil {
// Wrap err as external error (if needed) because err is returned by TypeInfo interface.
return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info")
Expand Down Expand Up @@ -840,7 +844,8 @@ func (a *ArrayDataSlab) Encode(enc *Encoder) error {

// Encode extra data
if a.extraData != nil {
err = a.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = a.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// err is already categorized by ArrayExtraData.Encode().
return err
Expand Down Expand Up @@ -1738,7 +1743,8 @@ func (a *ArrayMetaDataSlab) Encode(enc *Encoder) error {

// Encode extra data if present
if a.extraData != nil {
err = a.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = a.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// Don't need to wrap because err is already categorized by ArrayExtraData.Encode().
return err
Expand Down
17 changes: 16 additions & 1 deletion array_debug.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,12 +861,27 @@ func hasInlinedComposite(data []byte) (bool, error) {

// Parse inlined extra data to find compact map extra data.
dec := cbor.NewStreamDecoder(bytes.NewBuffer(data))

count, err := dec.DecodeArrayHead()
if err != nil {
return false, NewDecodingError(err)
}
if count != inlinedExtraDataArrayCount {
return false, NewDecodingError(fmt.Errorf("failed to decode inlined extra data, expect %d elements, got %d elements", inlinedExtraDataArrayCount, count))
}

for i := uint64(0); i < count; i++ {
// Skip element 0 (inlined type info)
err = dec.Skip()
if err != nil {
return false, NewDecodingError(err)
}

// Decoding element 1 (inlined extra data)
extraDataCount, err := dec.DecodeArrayHead()
if err != nil {
return false, NewDecodingError(err)
}
for i := uint64(0); i < extraDataCount; i++ {
tagNum, err := dec.DecodeTagNumber()
if err != nil {
return false, NewDecodingError(err)
Expand Down
33 changes: 31 additions & 2 deletions array_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3184,10 +3184,15 @@ func TestArrayEncodeDecode(t *testing.T) {
0x18, 0x2a,

// inlined extra data
0x82,
// element 0: array of type info
0x80,
// element 1: array of extra data
0x81,
// inlined array extra data
// array extra data
0xd8, 0xf7,
0x81,
// array type info ref
0x18, 0x2b,

// CBOR encoded array head (fixed size 3 byte)
Expand Down Expand Up @@ -3266,6 +3271,10 @@ func TestArrayEncodeDecode(t *testing.T) {

// inlined extra data
0x82,
// element 0: array of inlined type info
0x80,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
Expand Down Expand Up @@ -3355,6 +3364,10 @@ func TestArrayEncodeDecode(t *testing.T) {

// inlined extra data
0x82,
// element 0: array of inlined type info
0x80,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
Expand Down Expand Up @@ -3454,6 +3467,10 @@ func TestArrayEncodeDecode(t *testing.T) {
0x18, 0x2a,

// inlined extra data
0x82,
// element 0: array of inlined type info
0x80,
// element 1: array of inlined extra data
0x84,
// typeInfo3
0xd8, 0xf7,
Expand Down Expand Up @@ -3595,6 +3612,10 @@ func TestArrayEncodeDecode(t *testing.T) {
// array data slab flag
0x00,
// inlined extra data
0x82,
// element 0: array of inlined type info
0x80,
// element 1: array of inlined extra data
0x81,
// inlined array extra data
0xd8, 0xf7,
Expand Down Expand Up @@ -3744,6 +3765,10 @@ func TestArrayEncodeDecode(t *testing.T) {
0x00,
// inlined extra data
0x82,
// element 0: array of inlined extra data
0x80,
// element 1: array of inlined extra data
0x82,
// inlined array extra data
0xd8, 0xf7,
0x81,
Expand Down Expand Up @@ -4064,7 +4089,11 @@ func TestArrayEncodeDecode(t *testing.T) {
// array data slab flag (has pointer)
0x40,

// inlined array of extra data
// inlined extra data
0x82,
// element 0: array of type info
0x80,
// element 1: array of extra data
0x81,
// type info
0xd8, 0xf7,
Expand Down
14 changes: 10 additions & 4 deletions map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,17 +494,21 @@ func (m *MapExtraData) isExtraData() bool {
return true
}

func (m *MapExtraData) Type() TypeInfo {
return m.TypeInfo
}

// Encode encodes extra data as CBOR array:
//
// [type info, count, seed]
func (m *MapExtraData) Encode(enc *Encoder) error {
func (m *MapExtraData) Encode(enc *Encoder, encodeTypeInfo encodeTypeInfo) error {

err := enc.CBOR.EncodeArrayHead(mapExtraDataLength)
if err != nil {
return NewEncodingError(err)
}

err = m.TypeInfo.Encode(enc.CBOR)
err = encodeTypeInfo(enc, m.TypeInfo)
if err != nil {
// Wrap err as external error (if needed) because err is returned by TypeInfo interface.
return wrapErrorfAsExternalErrorIfNeeded(err, "failed to encode type info")
Expand Down Expand Up @@ -2917,7 +2921,8 @@ func (m *MapDataSlab) Encode(enc *Encoder) error {

// Encode extra data
if m.extraData != nil {
err = m.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = m.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode().
return err
Expand Down Expand Up @@ -3918,7 +3923,8 @@ func (m *MapMetaDataSlab) Encode(enc *Encoder) error {

// Encode extra data if present
if m.extraData != nil {
err = m.extraData.Encode(enc)
// Use defaultEncodeTypeInfo to encode root level TypeInfo as is.
err = m.extraData.Encode(enc, defaultEncodeTypeInfo)
if err != nil {
// Don't need to wrap error as external error because err is already categorized by MapExtraData.Encode().
return err
Expand Down
Loading
Loading