Skip to content

Commit

Permalink
cmd, core, eth, trie: track deleted nodes (#576)
Browse files Browse the repository at this point in the history 
* trie: track deleted nodes

* core: track deleted nodes
  • Loading branch information
@Francesco4203 @huyngopt1994
Francesco4203 authored and huyngopt1994 committed Nov 21, 2024
1 parent 44d3f93 commit d3f93f9
Show file tree
Hide file tree
Showing 41 changed files with 805 additions and 229 deletions.
38 changes: 24 additions & 14 deletions cmd/ronin/dbcmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
Action: dbDumpTrie,
Name: "dumptrie",
Usage: "Show the storage key/values of a given storage trie",
ArgsUsage: "<hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
Flags: []cli.Flag{
utils.DataDirFlag,
utils.DBEngineFlag,
Expand Down Expand Up @@ -468,7 +468,7 @@ func dbPut(ctx *cli.Context) error {

// dbDumpTrie shows the key-value slots of a given storage trie
func dbDumpTrie(ctx *cli.Context) error {
if ctx.NArg() < 1 {
if ctx.NArg() < 3 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
Expand All @@ -477,29 +477,39 @@ func dbDumpTrie(ctx *cli.Context) error {
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
var (
root []byte
start []byte
max = int64(-1)
err error
state []byte
storage []byte
account []byte
start []byte
max = int64(-1)
err error
)
if root, err = hexutil.Decode(ctx.Args().Get(0)); err != nil {
log.Info("Could not decode the root", "error", err)
if state, err = hexutil.Decode(ctx.Args().Get(0)); err != nil {
log.Info("Could not decode the state", "error", err)
return err
}
stRoot := common.BytesToHash(root)
if ctx.NArg() >= 2 {
if start, err = hexutil.Decode(ctx.Args().Get(1)); err != nil {
if account, err = hexutil.Decode(ctx.Args().Get(1)); err != nil {
log.Info("Could not decode the account hash", "error", err)
return err
}
if storage, err = hexutil.Decode(ctx.Args().Get(2)); err != nil {
log.Info("Could not decode the storage trie root", "error", err)
return err
}
if ctx.NArg() > 3 {
if start, err = hexutil.Decode(ctx.Args().Get(3)); err != nil {
log.Info("Could not decode the seek position", "error", err)
return err
}
}
if ctx.NArg() >= 3 {
if max, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil {
if ctx.NArg() > 4 {
if max, err = strconv.ParseInt(ctx.Args().Get(4), 10, 64); err != nil {
log.Info("Could not decode the max count", "error", err)
return err
}
}
theTrie, err := trie.New(common.Hash{}, stRoot, trie.NewDatabase(db))
id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage))
theTrie, err := trie.New(id, trie.NewDatabase(db))
if err != nil {
return err
}
Expand Down
8 changes: 4 additions & 4 deletions cmd/ronin/snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ func traverseState(ctx *cli.Context) error {
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(common.Hash{}, root, triedb)
t, err := trie.NewSecure(trie.StateTrieID(root), triedb)
if err != nil {
log.Error("Failed to open trie", "root", root, "err", err)
return err
Expand All @@ -304,7 +304,7 @@ func traverseState(ctx *cli.Context) error {
return err
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb)
storageTrie, err := trie.NewSecure(trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root), triedb)
if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
return err
Expand Down Expand Up @@ -373,7 +373,7 @@ func traverseRawState(ctx *cli.Context) error {
log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
}
triedb := trie.NewDatabase(chaindb)
t, err := trie.NewSecure(common.Hash{}, root, triedb)
t, err := trie.NewSecure(trie.StateTrieID(root), triedb)
if err != nil {
log.Error("Failed to open trie", "root", root, "err", err)
return err
Expand Down Expand Up @@ -410,7 +410,7 @@ func traverseRawState(ctx *cli.Context) error {
return errors.New("invalid account")
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb)
storageTrie, err := trie.NewSecure(trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root), triedb)
if err != nil {
log.Error("Failed to open storage trie", "root", acc.Root, "err", err)
return errors.New("missing storage trie")
Expand Down
13 changes: 8 additions & 5 deletions core/blockchain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ var (
snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil)
snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil)

triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil)

blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil)
blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil)
Expand Down Expand Up @@ -823,10 +825,10 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
if block == nil {
return fmt.Errorf("non existent block [%x..]", hash[:4])
}
if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.triedb); err != nil {
return err
root := block.Root()
if !bc.HasState(root) {
return fmt.Errorf("non existent state [%x..]", root[:4])
}

// If all checks out, manually set the head block.
if !bc.chainmu.TryLock() {
return errChainStopped
Expand All @@ -838,7 +840,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error {
// Destroy any existing state snapshot and regenerate it in the background,
// also resuming the normal maintenance of any previously paused snapshot.
if bc.snaps != nil {
bc.snaps.Rebuild(block.Root())
bc.snaps.Rebuild(root)
}
log.Info("Committed new head block", "number", block.Number(), "hash", hash)
return nil
Expand Down Expand Up @@ -2034,8 +2036,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool, sidecars
accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them
storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them
snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them
triedbCommitTimer.Update(statedb.TrieDBCommits) // Triedb commits are complete, we can mark them

blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits)
blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits - statedb.TrieDBCommits)
blockInsertTimer.UpdateSince(start)
blockTxsGauge.Update(int64(len(block.Transactions())))
blockGasUsedGauge.Update(int64(block.GasUsed()))
Expand Down
8 changes: 4 additions & 4 deletions core/state/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ type Database interface {
OpenTrie(root common.Hash) (Trie, error)

// OpenStorageTrie opens the storage trie of an account.
OpenStorageTrie(addrHash, root common.Hash) (Trie, error)
OpenStorageTrie(stateRoot, addrHash, root common.Hash) (Trie, error)

// CopyTrie returns an independent copy of the given trie.
CopyTrie(Trie) Trie
Expand Down Expand Up @@ -145,16 +145,16 @@ type cachingDB struct {

// OpenTrie opens the main account trie at a specific root hash.
func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(common.Hash{}, root, db.triedb)
tr, err := trie.NewSecure(trie.StateTrieID(root), db.triedb)
if err != nil {
return nil, err
}
return tr, nil
}

// OpenStorageTrie opens the storage trie of an account.
func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(addrHash, root, db.triedb)
func (db *cachingDB) OpenStorageTrie(stateRoot, addrHash, root common.Hash) (Trie, error) {
tr, err := trie.NewSecure(trie.StorageTrieID(stateRoot, addrHash, root), db.triedb)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion core/state/iterator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (it *NodeIterator) step() error {
if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil {
return err
}
dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root)
dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root)
if err != nil {
return err
}
Expand Down
14 changes: 8 additions & 6 deletions core/state/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ package state
import "github.com/ethereum/go-ethereum/metrics"

var (
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
accountTrieUpdatedMeter = metrics.NewRegisteredMeter("state/update/accountnodes", nil)
storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil)
accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil)
storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil)
)
4 changes: 2 additions & 2 deletions core/state/pruner/pruner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
if genesis == nil {
return errors.New("missing genesis block")
}
t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db))
t, err := trie.NewSecure(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
if err != nil {
return err
}
Expand All @@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
return err
}
if acc.Root != emptyRoot {
storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db))
storageTrie, err := trie.NewSecure(trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root), trie.NewDatabase(db))
if err != nil {
return err
}
Expand Down
21 changes: 11 additions & 10 deletions core/state/snapshot/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error)
//
// The proof result will be returned if the range proving is finished, otherwise
// the error will be returned to abort the entire procedure.
func (dl *diskLayer) proveRange(stats *generatorStats, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
func (dl *diskLayer) proveRange(stats *generatorStats, trieID *trie.ID, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) {
var (
keys [][]byte
vals [][]byte
Expand Down Expand Up @@ -304,8 +304,9 @@ func (dl *diskLayer) proveRange(stats *generatorStats, owner common.Hash, root c
}(time.Now())

// The snap state is exhausted, pass the entire key/val set for verification
root := trieID.Root
if origin == nil && !diskMore {
stackTr := trie.NewStackTrieWithOwner(nil, owner)
stackTr := trie.NewStackTrie(nil)
for i, key := range keys {
stackTr.TryUpdate(key, vals[i])
}
Expand All @@ -319,7 +320,7 @@ func (dl *diskLayer) proveRange(stats *generatorStats, owner common.Hash, root c
return &proofResult{keys: keys, vals: vals}, nil
}
// Snap state is chunked, generate edge proofs for verification.
tr, err := trie.New(owner, root, dl.triedb)
tr, err := trie.New(trieID, dl.triedb)
if err != nil {
stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return nil, errMissingTrie
Expand Down Expand Up @@ -380,9 +381,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error
// generateRange generates the state segment with particular prefix. Generation can
// either verify the correctness of existing state through rangeproof and skip
// generation, or iterate trie to regenerate state on demand.
func (dl *diskLayer) generateRange(owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, stats *generatorStats, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
func (dl *diskLayer) generateRange(trieID *trie.ID, prefix []byte, kind string, origin []byte, max int, stats *generatorStats, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) {
// Use range prover to check the validity of the flat state in the range
result, err := dl.proveRange(stats, owner, root, prefix, kind, origin, max, valueConvertFn)
result, err := dl.proveRange(stats, trieID, prefix, kind, origin, max, valueConvertFn)
if err != nil {
return false, nil, err
}
Expand Down Expand Up @@ -431,7 +432,7 @@ func (dl *diskLayer) generateRange(owner common.Hash, root common.Hash, prefix [
if len(result.keys) > 0 {
snapNodeCache = rawdb.NewMemoryDatabase()
snapTrieDb := trie.NewDatabase(snapNodeCache)
snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb)
snapTrie := trie.NewEmpty(snapTrieDb)
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])
}
Expand All @@ -443,7 +444,7 @@ func (dl *diskLayer) generateRange(owner common.Hash, root common.Hash, prefix [
}
tr := result.tr
if tr == nil {
tr, err = trie.New(owner, root, dl.triedb)
tr, err = trie.New(trieID, dl.triedb)
if err != nil {
stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
return false, nil, errMissingTrie
Expand Down Expand Up @@ -526,7 +527,7 @@ func (dl *diskLayer) generateRange(owner common.Hash, root common.Hash, prefix [
} else {
snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds())
}
logger.Debug("Regenerated state range", "root", root, "last", hexutil.Encode(last),
logger.Debug("Regenerated state range", "root", trieID.Root, "last", hexutil.Encode(last),
"count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted)

// If there are either more trie items, or there are more snap items
Expand Down Expand Up @@ -700,7 +701,7 @@ func (dl *diskLayer) generate(stats *generatorStats) {
}
var storeOrigin = common.CopyBytes(storeMarker)
for {
exhausted, last, err := dl.generateRange(common.Hash{}, acc.Root, append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...), "storage", storeOrigin, storageCheckRange, stats, onStorage, nil)
exhausted, last, err := dl.generateRange(trie.StorageTrieID(dl.Root(), accountHash, acc.Root), append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...), "storage", storeOrigin, storageCheckRange, stats, onStorage, nil)
if err != nil {
return err
}
Expand All @@ -719,7 +720,7 @@ func (dl *diskLayer) generate(stats *generatorStats) {

// Global loop for regerating the entire state trie + all layered storage tries.
for {
exhausted, last, err := dl.generateRange(common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, "account", accOrigin, accountRange, stats, onAccount, FullAccountRLP)
exhausted, last, err := dl.generateRange(trie.StateTrieID(dl.root), rawdb.SnapshotAccountPrefix, "account", accOrigin, accountRange, stats, onAccount, FullAccountRLP)
// The procedure it aborted, either by external signal or internal error
if err != nil {
if abort == nil { // aborted by internal error, wait the signal
Expand Down
4 changes: 2 additions & 2 deletions core/state/snapshot/generate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ type testHelper struct {
func newHelper() *testHelper {
diskdb := rawdb.NewMemoryDatabase()
triedb := trie.NewDatabase(diskdb)
accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
accTrie, _ := trie.NewSecure(trie.StateTrieID(common.Hash{}), triedb)
return &testHelper{
diskdb: diskdb,
triedb: triedb,
Expand Down Expand Up @@ -177,7 +177,7 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string)
}

func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte {
stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb)
stTrie, _ := trie.NewSecure(trie.StorageTrieID(stateRoot, owner, common.Hash{}), t.triedb)
for i, k := range keys {
stTrie.Update([]byte(k), []byte(vals[i]))
}
Expand Down
4 changes: 2 additions & 2 deletions core/state/state_object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,9 @@ func (s *stateObject) getTrie(db Database) Trie {
}
if s.trie == nil {
var err error
s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root)
s.trie, err = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, s.data.Root)
if err != nil {
s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{})
s.trie, _ = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, common.Hash{})
s.setError(fmt.Errorf("can't create storage trie: %v", err))
}
}
Expand Down
Loading

0 comments on commit d3f93f9

Please sign in to comment.