Skip to content

Commit

Permalink
List of changes for converting a sepolia database (ethereum#182)
Browse files Browse the repository at this point in the history 
* naive conversion rebased on top of beverly hills

* changes for the sepolia shadow fork conversion

* fixes to please the linter

* fixes to please the linter
  • Loading branch information
@gballet
gballet authored Mar 17, 2023
1 parent 2d5ce8a commit 92ab166
Show file tree
Hide file tree
Showing 2 changed files with 250 additions and 0 deletions.
246 changes: 246 additions & 0 deletions cmd/geth/verkle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,25 @@ package main

import (
"bytes"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"os"
"runtime"
"time"

"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
tutils "github.com/ethereum/go-ethereum/trie/utils"
"github.com/gballet/go-verkle"
"github.com/holiman/uint256"
cli "github.com/urfave/cli/v2"
)

Expand All @@ -41,6 +49,20 @@ var (
Category: "MISCELLANEOUS COMMANDS",
Description: "",
Subcommands: []*cli.Command{
{
Name: "to-verkle",
Usage: "use the snapshot to compute a translation of a MPT into a verkle tree",
ArgsUsage: "<root>",
Action: convertToVerkle,
Flags: flags.Merge([]cli.Flag{}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: `
geth verkle to-verkle <state-root>
This command takes a snapshot and inserts its values in a fresh verkle tree.
The argument is interpreted as the root hash. If none is provided, the latest
block is used.
`,
},
{
Name: "verify",
Usage: "verify the conversion of a MPT into a verkle tree",
Expand Down Expand Up @@ -68,6 +90,230 @@ in which key1, key2, ... are expanded.
}
)

func convertToVerkle(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()

chaindb := utils.MakeChainDatabase(ctx, stack, false)
if chaindb == nil {
return errors.New("nil chaindb")
}
headBlock := rawdb.ReadHeadBlock(chaindb)
if headBlock == nil {
log.Error("Failed to load head block")
return errors.New("no head block")
}
if ctx.NArg() > 1 {
log.Error("Too many arguments given")
return errors.New("too many arguments")
}
var (
root common.Hash
err error
)
if ctx.NArg() == 1 {
root, err = parseRoot(ctx.Args().First())
if err != nil {
log.Error("Failed to resolve state root", "error", err)
return err
}
log.Info("Start traversing the state", "root", root)
} else {
root = headBlock.Root()
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}

var (
accounts int
lastReport time.Time
start = time.Now()
vRoot = verkle.New().(*verkle.InternalNode)
)

saveverkle := func(node verkle.VerkleNode) {
comm := node.Commit()
s, err := node.Serialize()
if err != nil {
panic(err)
}
commB := comm.Bytes()
if err := chaindb.Put(commB[:], s); err != nil {
panic(err)
}
}

snaptree, err := snapshot.New(snapshot.Config{CacheSize: 256}, chaindb, trie.NewDatabase(chaindb), root)
if err != nil {
return err
}
accIt, err := snaptree.AccountIterator(root, common.Hash{})
if err != nil {
return err
}
defer accIt.Release()

// root.FlushAtDepth(depth, saveverkle)

// Process all accounts sequentially
for accIt.Next() {
accounts += 1
acc, err := snapshot.FullAccount(accIt.Account())
if err != nil {
log.Error("Invalid account encountered during traversal", "error", err)
return err
}

// Store the basic account data
var (
nonce, balance, version [32]byte
newValues = make([][]byte, 256)
)
newValues[0] = version[:]
newValues[1] = balance[:]
newValues[2] = nonce[:]
newValues[4] = version[:] // memory-saving trick: by default, an account has 0 size
binary.LittleEndian.PutUint64(nonce[:8], acc.Nonce)
for i, b := range acc.Balance.Bytes() {
balance[len(acc.Balance.Bytes())-1-i] = b
}
addr := rawdb.ReadPreimage(chaindb, accIt.Hash())
if addr == nil {
return fmt.Errorf("could not find preimage for address %x %v %v", accIt.Hash(), acc, accIt.Error())
}
addrPoint := tutils.EvaluateAddressPoint(addr)
stem := tutils.GetTreeKeyVersionWithEvaluatedAddress(addrPoint)

// Store the account code if present
if !bytes.Equal(acc.CodeHash, emptyCode) {
code := rawdb.ReadCode(chaindb, common.BytesToHash(acc.CodeHash))
chunks := trie.ChunkifyCode(code)

for i := 0; i < 128 && i < len(chunks)/32; i++ {
newValues[128+i] = chunks[32*i : 32*(i+1)]
}

for i := 128; i < len(chunks)/32; {
values := make([][]byte, 256)
chunkkey := tutils.GetTreeKeyCodeChunkWithEvaluatedAddress(addrPoint, uint256.NewInt(uint64(i)))
j := i
for ; (j-i) < 256 && j < len(chunks)/32; j++ {
values[(j-128)%256] = chunks[32*j : 32*(j+1)]
}
i = j

// Otherwise, store the previous group in the tree with a
// stem insertion.
vRoot.InsertStem(chunkkey[:31], values, chaindb.Get)
}

// Write the code size in the account header group
var size [32]byte
newValues[3] = acc.CodeHash[:]
newValues[4] = size[:]
binary.LittleEndian.PutUint64(size[:8], uint64(len(code)))
}

// Save every slot into the tree
if !bytes.Equal(acc.Root, emptyRoot[:]) {
var translatedStorage = map[string][][]byte{}

storageIt, err := snaptree.StorageIterator(root, accIt.Hash(), common.Hash{})
if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "error", err)
return err
}
for storageIt.Next() {
// The value is RLP-encoded, decode it
var (
value []byte // slot value after RLP decoding
safeValue [32]byte // 32-byte aligned value
)
if err := rlp.DecodeBytes(storageIt.Slot(), &value); err != nil {
return fmt.Errorf("error decoding bytes %x: %w", storageIt.Slot(), err)
}
copy(safeValue[32-len(value):], value)

slotnr := rawdb.ReadPreimage(chaindb, storageIt.Hash())
if slotnr == nil {
return fmt.Errorf("could not find preimage for slot %x", storageIt.Hash())
}

// if the slot belongs to the header group, store it there - and skip
// calculating the slot key.
slotnrbig := uint256.NewInt(0).SetBytes(slotnr)
if slotnrbig.Cmp(uint256.NewInt(64)) < 0 {
newValues[64+slotnr[31]] = safeValue[:]
continue
}

// Slot not in the header group, get its tree key
slotkey := tutils.GetTreeKeyStorageSlotWithEvaluatedAddress(addrPoint, slotnrbig)

// Create the group if need be
values := translatedStorage[string(slotkey[:31])]
if values == nil {
values = make([][]byte, 256)
}

// Store value in group
values[slotkey[31]] = safeValue[:]
translatedStorage[string(slotkey[:31])] = values

// Dump the stuff to disk if we ran out of space
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
if mem.Alloc > 25*1024*1024*1024 {
fmt.Println("Memory usage exceeded threshold, calling mitigation function")
for s, vs := range translatedStorage {
var k [31]byte
copy(k[:], []byte(s))
// reminder that InsertStem will merge leaves
// if they exist.
vRoot.InsertStem(k[:31], vs, chaindb.Get)
}
translatedStorage = make(map[string][][]byte)
vRoot.FlushAtDepth(2, saveverkle)
}
}
for s, vs := range translatedStorage {
var k [31]byte
copy(k[:], []byte(s))
vRoot.InsertStem(k[:31], vs, chaindb.Get)
}
storageIt.Release()
if storageIt.Error() != nil {
log.Error("Failed to traverse storage trie", "root", acc.Root, "error", storageIt.Error())
return storageIt.Error()
}
}
// Finish with storing the complete account header group inside the tree.
vRoot.InsertStem(stem[:31], newValues, chaindb.Get)

if time.Since(lastReport) > time.Second*8 {
log.Info("Traversing state", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start)))
lastReport = time.Now()
}

var mem runtime.MemStats
runtime.ReadMemStats(&mem)
if mem.Alloc > 25*1024*1024*1024 {
fmt.Println("Memory usage exceeded threshold, calling mitigation function")
vRoot.FlushAtDepth(2, saveverkle)
}
}
if accIt.Error() != nil {
log.Error("Failed to compute commitment", "root", root, "error", accIt.Error())
return accIt.Error()
}
log.Info("Wrote all leaves", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start)))

vRoot.Commit()
vRoot.Flush(saveverkle)

log.Info("Conversion complete", "root commitment", fmt.Sprintf("%x", vRoot.Commit().Bytes()), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}

// recurse into each child to ensure they can be loaded from the db. The tree isn't rebuilt
// (only its nodes are loaded) so there is no need to flush them, the garbage collector should
// take care of that for us.
Expand Down
4 changes: 4 additions & 0 deletions trie/utils/verkle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ func GetTreeKeyVersion(address []byte) []byte {
return GetTreeKey(address, zero, VersionLeafKey)
}

func GetTreeKeyVersionWithEvaluatedAddress(addrp *verkle.Point) []byte {
return getTreeKeyWithEvaluatedAddess(addrp, zero, VersionLeafKey)
}

func GetTreeKeyBalance(address []byte) []byte {
return GetTreeKey(address, zero, BalanceLeafKey)
}
Expand Down

0 comments on commit 92ab166

Please sign in to comment.