Skip to content

Commit

Permalink
Merge pull request #377 from onflow/fxamacker/add-inlining-map-settype
Browse files Browse the repository at this point in the history 
Add support for changing type info of atree maps (atree inlining branch)
  • Loading branch information
@fxamacker
fxamacker authored Mar 8, 2024
2 parents 36e70a5 + 95dad1c commit 43b2a0d
Show file tree
Hide file tree
Showing 2 changed files with 315 additions and 0 deletions.
19 changes: 19 additions & 0 deletions map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5493,6 +5493,25 @@ func (m *OrderedMap) Type() TypeInfo {
return nil
}

func (m *OrderedMap) SetType(typeInfo TypeInfo) error {
extraData := m.root.ExtraData()
extraData.TypeInfo = typeInfo

m.root.SetExtraData(extraData)

if m.Inlined() {
// Map is inlined.

// Notify parent container so parent slab is saved in storage with updated TypeInfo of inlined array.
return m.notifyParentIfNeeded()
}

// Map is standalone.

// Store modified root slab in storage since typeInfo is part of extraData stored in root slab.
return storeSlab(m.Storage, m.root)
}

func (m *OrderedMap) String() string {
iterator, err := m.ReadOnlyIterator()
if err != nil {
Expand Down
296 changes: 296 additions & 0 deletions map_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"math"
"math/rand"
"reflect"
"runtime"
"sort"
"strings"
"testing"
Expand Down Expand Up @@ -18337,3 +18338,298 @@ func TestMapWithOutdatedCallback(t *testing.T) {
valueEqual(t, expectedKeyValues, parentMap)
})
}

func TestMapSetType(t *testing.T) {
typeInfo := testTypeInfo{42}
newTypeInfo := testTypeInfo{43}
address := Address{1, 2, 3, 4, 5, 6, 7, 8}

t.Run("empty", func(t *testing.T) {
storage := newTestPersistentStorage(t)

m, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)
require.Equal(t, uint64(0), m.Count())
require.Equal(t, typeInfo, m.Type())
require.True(t, m.root.IsData())

seed := m.root.ExtraData().Seed

err = m.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, uint64(0), m.Count())
require.Equal(t, newTypeInfo, m.Type())
require.Equal(t, seed, m.root.ExtraData().Seed)

// Commit modified slabs in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

testExistingMapSetType(t, m.SlabID(), storage.baseStorage, newTypeInfo, m.Count(), seed)
})

t.Run("data slab root", func(t *testing.T) {
storage := newTestPersistentStorage(t)

m, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)

mapSize := 10
for i := 0; i < mapSize; i++ {
v := Uint64Value(i)
existingStorable, err := m.Set(compare, hashInputProvider, v, v)
require.NoError(t, err)
require.Nil(t, existingStorable)
}

require.Equal(t, uint64(mapSize), m.Count())
require.Equal(t, typeInfo, m.Type())
require.True(t, m.root.IsData())

seed := m.root.ExtraData().Seed

err = m.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, newTypeInfo, m.Type())
require.Equal(t, uint64(mapSize), m.Count())
require.Equal(t, seed, m.root.ExtraData().Seed)

// Commit modified slabs in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

testExistingMapSetType(t, m.SlabID(), storage.baseStorage, newTypeInfo, m.Count(), seed)
})

t.Run("metadata slab root", func(t *testing.T) {
storage := newTestPersistentStorage(t)

m, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)

mapSize := 10_000
for i := 0; i < mapSize; i++ {
v := Uint64Value(i)
existingStorable, err := m.Set(compare, hashInputProvider, v, v)
require.NoError(t, err)
require.Nil(t, existingStorable)
}

require.Equal(t, uint64(mapSize), m.Count())
require.Equal(t, typeInfo, m.Type())
require.False(t, m.root.IsData())

seed := m.root.ExtraData().Seed

err = m.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, newTypeInfo, m.Type())
require.Equal(t, uint64(mapSize), m.Count())
require.Equal(t, seed, m.root.ExtraData().Seed)

// Commit modified slabs in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

testExistingMapSetType(t, m.SlabID(), storage.baseStorage, newTypeInfo, m.Count(), seed)
})

t.Run("inlined in parent container root data slab", func(t *testing.T) {
storage := newTestPersistentStorage(t)

parentMap, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)

childMap, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)

childMapSeed := childMap.root.ExtraData().Seed

existingStorable, err := parentMap.Set(compare, hashInputProvider, Uint64Value(0), childMap)
require.NoError(t, err)
require.Nil(t, existingStorable)

require.Equal(t, uint64(1), parentMap.Count())
require.Equal(t, typeInfo, parentMap.Type())
require.True(t, parentMap.root.IsData())
require.False(t, parentMap.Inlined())

require.Equal(t, uint64(0), childMap.Count())
require.Equal(t, typeInfo, childMap.Type())
require.True(t, childMap.root.IsData())
require.True(t, childMap.Inlined())

err = childMap.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, newTypeInfo, childMap.Type())
require.Equal(t, uint64(0), childMap.Count())
require.Equal(t, childMapSeed, childMap.root.ExtraData().Seed)

// Commit modified slabs in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

testExistingInlinedMapSetType(
t,
parentMap.SlabID(),
Uint64Value(0),
storage.baseStorage,
newTypeInfo,
childMap.Count(),
childMapSeed,
)
})

t.Run("inlined in parent container non-root data slab", func(t *testing.T) {
storage := newTestPersistentStorage(t)

parentMap, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)

childMap, err := NewMap(storage, address, newBasicDigesterBuilder(), typeInfo)
require.NoError(t, err)

childMapSeed := childMap.root.ExtraData().Seed

mapSize := 10_000
for i := 0; i < mapSize-1; i++ {
v := Uint64Value(i)
existingStorable, err := parentMap.Set(compare, hashInputProvider, v, v)
require.NoError(t, err)
require.Nil(t, existingStorable)
}

existingStorable, err := parentMap.Set(compare, hashInputProvider, Uint64Value(mapSize-1), childMap)
require.NoError(t, err)
require.Nil(t, existingStorable)

require.Equal(t, uint64(mapSize), parentMap.Count())
require.Equal(t, typeInfo, parentMap.Type())
require.False(t, parentMap.root.IsData())
require.False(t, parentMap.Inlined())

require.Equal(t, uint64(0), childMap.Count())
require.Equal(t, typeInfo, childMap.Type())
require.True(t, childMap.root.IsData())
require.True(t, childMap.Inlined())

err = childMap.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, newTypeInfo, childMap.Type())
require.Equal(t, uint64(0), childMap.Count())
require.Equal(t, childMapSeed, childMap.root.ExtraData().Seed)

// Commit modified slabs in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

testExistingInlinedMapSetType(
t,
parentMap.SlabID(),
Uint64Value(mapSize-1),
storage.baseStorage,
newTypeInfo,
childMap.Count(),
childMapSeed,
)
})
}

func testExistingMapSetType(
t *testing.T,
id SlabID,
baseStorage BaseStorage,
expectedTypeInfo testTypeInfo,
expectedCount uint64,
expectedSeed uint64,
) {
newTypeInfo := testTypeInfo{value: expectedTypeInfo.value + 1}

// Create storage from existing data
storage := newTestPersistentStorageWithBaseStorage(t, baseStorage)

// Load existing map by ID
m, err := NewMapWithRootID(storage, id, newBasicDigesterBuilder())
require.NoError(t, err)
require.Equal(t, expectedCount, m.Count())
require.Equal(t, expectedTypeInfo, m.Type())
require.Equal(t, expectedSeed, m.root.ExtraData().Seed)

// Modify type info of existing map
err = m.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, expectedCount, m.Count())
require.Equal(t, newTypeInfo, m.Type())
require.Equal(t, expectedSeed, m.root.ExtraData().Seed)

// Commit data in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

// Create storage from existing data
storage2 := newTestPersistentStorageWithBaseStorage(t, storage.baseStorage)

// Load existing map again from storage
m2, err := NewMapWithRootID(storage2, id, newBasicDigesterBuilder())
require.NoError(t, err)
require.Equal(t, expectedCount, m2.Count())
require.Equal(t, newTypeInfo, m2.Type())
require.Equal(t, expectedSeed, m2.root.ExtraData().Seed)
}

func testExistingInlinedMapSetType(
t *testing.T,
parentID SlabID,
inlinedChildKey Value,
baseStorage BaseStorage,
expectedTypeInfo testTypeInfo,
expectedCount uint64,
expectedSeed uint64,
) {
newTypeInfo := testTypeInfo{value: expectedTypeInfo.value + 1}

// Create storage from existing data
storage := newTestPersistentStorageWithBaseStorage(t, baseStorage)

// Load existing map by ID
parentMap, err := NewMapWithRootID(storage, parentID, newBasicDigesterBuilder())
require.NoError(t, err)

element, err := parentMap.Get(compare, hashInputProvider, inlinedChildKey)
require.NoError(t, err)

childMap, ok := element.(*OrderedMap)
require.True(t, ok)

require.Equal(t, expectedCount, childMap.Count())
require.Equal(t, expectedTypeInfo, childMap.Type())
require.Equal(t, expectedSeed, childMap.root.ExtraData().Seed)

// Modify type info of existing map
err = childMap.SetType(newTypeInfo)
require.NoError(t, err)
require.Equal(t, expectedCount, childMap.Count())
require.Equal(t, newTypeInfo, childMap.Type())
require.Equal(t, expectedSeed, childMap.root.ExtraData().Seed)

// Commit data in storage
err = storage.FastCommit(runtime.NumCPU())
require.NoError(t, err)

// Create storage from existing data
storage2 := newTestPersistentStorageWithBaseStorage(t, storage.baseStorage)

// Load existing map again from storage
parentMap2, err := NewMapWithRootID(storage2, parentID, newBasicDigesterBuilder())
require.NoError(t, err)

element2, err := parentMap2.Get(compare, hashInputProvider, inlinedChildKey)
require.NoError(t, err)

childMap2, ok := element2.(*OrderedMap)
require.True(t, ok)

require.Equal(t, expectedCount, childMap2.Count())
require.Equal(t, newTypeInfo, childMap2.Type())
require.Equal(t, expectedSeed, childMap.root.ExtraData().Seed)
}

0 comments on commit 43b2a0d

Please sign in to comment.