Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use encoded type info to deduplicate extra data #381

Merged
merged 1 commit into from
Apr 4, 2024
Merged

Use encoded type info to deduplicate extra data #381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions array.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -720,15 +720,16 @@ func (a *ArrayDataSlab) encodeAsInlined(enc *Encoder) error {
fmt.Errorf("failed to encode standalone array data slab as inlined"))
}

extraDataIndex := enc.inlinedExtraData().addArrayExtraData(a.extraData)
extraDataIndex, err := enc.inlinedExtraData().addArrayExtraData(a.extraData)
if err != nil {
return NewEncodingError(err)
}

if extraDataIndex > maxInlinedExtraDataIndex {
return NewEncodingError(
fmt.Errorf("failed to encode inlined array data slab: extra data index %d exceeds limit %d", extraDataIndex, maxInlinedExtraDataIndex))
}

var err error

// Encode tag number and array head of 3 elements
err = enc.CBOR.EncodeRawBytes([]byte{
// tag number
Expand Down
41 changes: 40 additions & 1 deletion cmd/stress/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@
package main

import (
"bytes"
"fmt"
"math"
"math/rand"
"reflect"
"sync"
"time"

"github.com/fxamacker/cbor/v2"

"github.com/onflow/atree"
)

Expand Down Expand Up @@ -540,5 +544,40 @@ func (v mapValue) Storable(atree.SlabStorage, atree.Address, uint64) (atree.Stor
}

var typeInfoComparator = func(a atree.TypeInfo, b atree.TypeInfo) bool {
return a.Identifier() == b.Identifier()
aID, _ := getEncodedTypeInfo(a)
bID, _ := getEncodedTypeInfo(b)
return aID == bID
}

func getEncodedTypeInfo(ti atree.TypeInfo) (string, error) {
b := getTypeIDBuffer()
defer putTypeIDBuffer(b)

enc := cbor.NewStreamEncoder(b)
err := ti.Encode(enc)
if err != nil {
return "", err
}
enc.Flush()

return b.String(), nil
}

const defaultTypeIDBufferSize = 256

var typeIDBufferPool = sync.Pool{
New: func() interface{} {
e := new(bytes.Buffer)
e.Grow(defaultTypeIDBufferSize)
return e
},
}

func getTypeIDBuffer() *bytes.Buffer {
return typeIDBufferPool.Get().(*bytes.Buffer)
}

func putTypeIDBuffer(e *bytes.Buffer) {
e.Reset()
typeIDBufferPool.Put(e)
}
Comment on lines +547 to 583
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this code is a duplicate of the code in typeinfo.go. Maybe the same code can be reused here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch! I don't like duplicate code too, but in this PR, I intentionally wrote same code in 2 packages within atree (atree and stress test main) in order to avoid exporting them and having to support their use by 3rd parties.

14 changes: 8 additions & 6 deletions map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3007,14 +3007,15 @@ func (m *MapDataSlab) encodeAsInlined(enc *Encoder) error {

func (m *MapDataSlab) encodeAsInlinedMap(enc *Encoder) error {

extraDataIndex := enc.inlinedExtraData().addMapExtraData(m.extraData)
extraDataIndex, err := enc.inlinedExtraData().addMapExtraData(m.extraData)
if err != nil {
return NewEncodingError(err)
}

if extraDataIndex > maxInlinedExtraDataIndex {
return NewEncodingError(fmt.Errorf("extra data index %d exceeds limit %d", extraDataIndex, maxInlinedExtraDataIndex))
}

var err error

// Encode tag number and array head of 3 elements
err = enc.CBOR.EncodeRawBytes([]byte{
// tag number
Expand Down Expand Up @@ -3067,7 +3068,10 @@ func encodeAsInlinedCompactMap(
values []Storable,
) error {

extraDataIndex, cachedKeys := enc.inlinedExtraData().addCompactMapExtraData(extraData, hkeys, keys)
extraDataIndex, cachedKeys, err := enc.inlinedExtraData().addCompactMapExtraData(extraData, hkeys, keys)
if err != nil {
return NewEncodingError(err)
}

if len(keys) != len(cachedKeys) {
return NewEncodingError(fmt.Errorf("number of elements %d is different from number of elements in cached compact map type %d", len(keys), len(cachedKeys)))
Expand All @@ -3078,8 +3082,6 @@ func encodeAsInlinedCompactMap(
return NewEncodingError(fmt.Errorf("extra data index %d exceeds limit %d", extraDataIndex, maxInlinedExtraDataIndex))
}

var err error

// Encode tag number and array head of 3 elements
err = enc.CBOR.EncodeRawBytes([]byte{
// tag number
Expand Down
Loading
Loading