From 76030b9f4c13b6357bff795765b52810693599ef Mon Sep 17 00:00:00 2001 From: Francesco4203 Date: Wed, 25 Sep 2024 14:42:29 +0700 Subject: [PATCH] all: prep for path-based trie storage --- cmd/ronin/snapshot.go | 4 +- core/rawdb/accessors_state.go | 26 --- core/rawdb/accessors_trie.go | 263 +++++++++++++++++++++++++ core/rawdb/schema.go | 14 ++ core/state/pruner/pruner.go | 4 +- core/state/snapshot/conversion.go | 12 +- core/state/snapshot/generate_test.go | 4 +- core/state/snapshot/snapshot.go | 4 +- core/state/sync.go | 2 +- core/state/sync_test.go | 6 +- eth/protocols/eth/handler_test.go | 2 +- eth/protocols/snap/sync.go | 18 +- eth/protocols/snap/sync_test.go | 10 +- les/downloader/downloader.go | 2 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- trie/database.go | 10 +- trie/schema.go | 96 --------- trie/sync.go | 11 +- trie/trie_test.go | 4 +- 19 files changed, 325 insertions(+), 169 deletions(-) create mode 100644 core/rawdb/accessors_trie.go delete mode 100644 trie/schema.go diff --git a/cmd/ronin/snapshot.go b/cmd/ronin/snapshot.go index 78b398a212..2a6a97f368 100644 --- a/cmd/ronin/snapshot.go +++ b/cmd/ronin/snapshot.go @@ -394,7 +394,7 @@ func traverseRawState(ctx *cli.Context) error { if node != (common.Hash{}) { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). - blob := rawdb.ReadTrieNode(chaindb, node) + blob := rawdb.ReadLegacyTrieNode(chaindb, node) if len(blob) == 0 { log.Error("Missing trie node(account)", "hash", node) return errors.New("missing account") @@ -423,7 +423,7 @@ func traverseRawState(ctx *cli.Context) error { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - blob := rawdb.ReadTrieNode(chaindb, node) + blob := rawdb.ReadLegacyTrieNode(chaindb, node) if len(blob) == 0 { log.Error("Missing trie node(storage)", "hash", node) return errors.New("missing storage") diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index d6b1053b60..1438aad0ff 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -61,35 +61,9 @@ func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) { } } -// HasTrieNode checks if the trie node with the provided hash is present in db. -func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { - ok, _ := db.Has(hash.Bytes()) - return ok -} - // DeleteCode deletes the specified contract code from the database. func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Delete(codeKey(hash)); err != nil { log.Crit("Failed to delete contract code", "err", err) } } - -// ReadTrieNode retrieves the trie node of the provided hash. -func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(hash.Bytes()) - return data -} - -// WriteTrieNode writes the provided trie node database. -func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { - if err := db.Put(hash.Bytes(), node); err != nil { - log.Crit("Failed to store trie node", "err", err) - } -} - -// DeleteTrieNode deletes the specified trie node from the database. -func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { - if err := db.Delete(hash.Bytes()); err != nil { - log.Crit("Failed to delete trie node", "err", err) - } -} diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go new file mode 100644 index 0000000000..e240213025 --- /dev/null +++ b/core/rawdb/accessors_trie.go @@ -0,0 +1,263 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package rawdb + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "golang.org/x/crypto/sha3" +) + +// HashScheme is the legacy hash-based state scheme with which trie nodes are +// stored in the disk with node hash as the database key. The advantage of this +// scheme is that different versions of trie nodes can be stored in disk, which +// is very beneficial for constructing archive nodes. The drawback is it will +// store different trie nodes on the same path to different locations on the disk +// with no data locality, and it's unfriendly for designing state pruning. +// +// Now this scheme is still kept for backward compatibility, and it will be used +// for archive node and some other tries(e.g. light trie). +const HashScheme = "hashScheme" + +// PathScheme is the new path-based state scheme with which trie nodes are stored +// in the disk with node path as the database key. This scheme will only store one +// version of state data in the disk, which means that the state pruning operation +// is native. At the same time, this scheme will put adjacent trie nodes in the same +// area of the disk with good data locality property. But this scheme needs to rely +// on extra state diffs to survive deep reorg. +const PathScheme = "pathScheme" + +// nodeHasher used to derive the hash of trie node. +type nodeHasher struct{ sha crypto.KeccakState } + +var hasherPool = sync.Pool{ + New: func() interface{} { return &nodeHasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} }, +} + +func newNodeHasher() *nodeHasher { return hasherPool.Get().(*nodeHasher) } +func returnHasherToPool(h *nodeHasher) { hasherPool.Put(h) } + +func (h *nodeHasher) hashData(data []byte) (n common.Hash) { + h.sha.Reset() + h.sha.Write(data) + h.sha.Read(n[:]) + return n +} + +// ReadAccountTrieNode retrieves the account trie node and the associated node +// hash with the specified node path. +func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) { + data, err := db.Get(accountTrieNodeKey(path)) + if err != nil { + return nil, common.Hash{} + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return data, hasher.hashData(data) +} + +// HasAccountTrieNode checks the account trie node presence with the specified +// node path and the associated node hash. +func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) bool { + data, err := db.Get(accountTrieNodeKey(path)) + if err != nil { + return false + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return hasher.hashData(data) == hash +} + +// WriteAccountTrieNode writes the provided account trie node into database. +func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) { + if err := db.Put(accountTrieNodeKey(path), node); err != nil { + log.Crit("Failed to store account trie node", "err", err) + } +} + +// DeleteAccountTrieNode deletes the specified account trie node from the database. +func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) { + if err := db.Delete(accountTrieNodeKey(path)); err != nil { + log.Crit("Failed to delete account trie node", "err", err) + } +} + +// ReadStorageTrieNode retrieves the storage trie node and the associated node +// hash with the specified node path. +func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) { + data, err := db.Get(storageTrieNodeKey(accountHash, path)) + if err != nil { + return nil, common.Hash{} + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return data, hasher.hashData(data) +} + +// HasStorageTrieNode checks the storage trie node presence with the provided +// node path and the associated node hash. +func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte, hash common.Hash) bool { + data, err := db.Get(storageTrieNodeKey(accountHash, path)) + if err != nil { + return false + } + hasher := newNodeHasher() + defer returnHasherToPool(hasher) + return hasher.hashData(data) == hash +} + +// WriteStorageTrieNode writes the provided storage trie node into database. +func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) { + if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil { + log.Crit("Failed to store storage trie node", "err", err) + } +} + +// DeleteStorageTrieNode deletes the specified storage trie node from the database. +func DeleteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte) { + if err := db.Delete(storageTrieNodeKey(accountHash, path)); err != nil { + log.Crit("Failed to delete storage trie node", "err", err) + } +} + +// ReadLegacyTrieNode retrieves the legacy trie node with the given +// associated node hash. +func ReadLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, err := db.Get(hash.Bytes()) + if err != nil { + return nil + } + return data +} + +// HasLegacyTrieNode checks if the trie node with the provided hash is present in db. +func HasLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { + ok, _ := db.Has(hash.Bytes()) + return ok +} + +// WriteLegacyTrieNode writes the provided legacy trie node to database. +func WriteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { + if err := db.Put(hash.Bytes(), node); err != nil { + log.Crit("Failed to store legacy trie node", "err", err) + } +} + +// DeleteLegacyTrieNode deletes the specified legacy trie node from database. +func DeleteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { + if err := db.Delete(hash.Bytes()); err != nil { + log.Crit("Failed to delete legacy trie node", "err", err) + } +} + +// HasTrieNode checks the trie node presence with the provided node info and +// the associated node hash. +func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) bool { + switch scheme { + case HashScheme: + return HasLegacyTrieNode(db, hash) + case PathScheme: + if owner == (common.Hash{}) { + return HasAccountTrieNode(db, path, hash) + } + return HasStorageTrieNode(db, owner, path, hash) + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} + +// ReadTrieNode retrieves the trie node from database with the provided node info +// and associated node hash. +// hashScheme-based lookup requires the following: +// - hash +// +// pathScheme-based lookup requires the following: +// - owner +// - path +func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte { + switch scheme { + case HashScheme: + return ReadLegacyTrieNode(db, hash) + case PathScheme: + var ( + blob []byte + nHash common.Hash + ) + if owner == (common.Hash{}) { + blob, nHash = ReadAccountTrieNode(db, path) + } else { + blob, nHash = ReadStorageTrieNode(db, owner, path) + } + if nHash != hash { + return nil + } + return blob + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} + +// WriteTrieNode writes the trie node into database with the provided node info +// and associated node hash. +// hashScheme-based lookup requires the following: +// - hash +// +// pathScheme-based lookup requires the following: +// - owner +// - path +func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte, scheme string) { + switch scheme { + case HashScheme: + WriteLegacyTrieNode(db, hash, node) + case PathScheme: + if owner == (common.Hash{}) { + WriteAccountTrieNode(db, path, node) + } else { + WriteStorageTrieNode(db, owner, path, node) + } + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} + +// DeleteTrieNode deletes the trie node from database with the provided node info +// and associated node hash. +// hashScheme-based lookup requires the following: +// - hash +// +// pathScheme-based lookup requires the following: +// - owner +// - path +func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, scheme string) { + switch scheme { + case HashScheme: + DeleteLegacyTrieNode(db, hash) + case PathScheme: + if owner == (common.Hash{}) { + DeleteAccountTrieNode(db, path) + } else { + DeleteStorageTrieNode(db, owner, path) + } + default: + panic(fmt.Sprintf("Unknown scheme %v", scheme)) + } +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index d92a0de0f9..ae9cbcff96 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -103,6 +103,10 @@ var ( internalTxsPrefix = []byte("itxs") // internalTxsPrefix + block hash -> internal transactions dirtyAccountsKey = []byte("dacc") // dirtyAccountsPrefix + block hash -> dirty accounts + // Path-based trie node scheme. + trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node + trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage configPrefix = []byte("ethereum-config-") // config prefix for the db genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db @@ -233,6 +237,16 @@ func genesisStateSpecKey(hash common.Hash) []byte { return append(genesisPrefix, hash.Bytes()...) } +// accountTrieNodeKey = trieNodeAccountPrefix + nodePath. +func accountTrieNodeKey(path []byte) []byte { + return append(trieNodeAccountPrefix, path...) +} + +// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { + return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...) +} + func snapshotConsortiumKey(hash common.Hash) []byte { return append(snapshotConsortiumPrefix, hash.Bytes()...) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 559507d7ff..34497569da 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -265,7 +265,7 @@ func (p *Pruner) Prune(root common.Hash) error { // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the // entire trie. - if blob := rawdb.ReadTrieNode(p.db, root); len(blob) == 0 { + if blob := rawdb.ReadLegacyTrieNode(p.db, root); len(blob) == 0 { // The special case is for clique based networks(rinkeby, goerli // and some other private networks), it's possible that two // consecutive blocks will have same root. In this case snapshot @@ -279,7 +279,7 @@ func (p *Pruner) Prune(root common.Hash) error { // as the pruning target. var found bool for i := len(layers) - 2; i >= 2; i-- { - if blob := rawdb.ReadTrieNode(p.db, layers[i].Root()); len(blob) != 0 { + if blob := rawdb.ReadLegacyTrieNode(p.db, layers[i].Root()); len(blob) != 0 { root = layers[i].Root() found = true log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 15bb43b842..abf541aef6 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -43,7 +43,7 @@ type trieKV struct { type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(db ethdb.KeyValueWriter, scheme trie.NodeScheme, owner common.Hash, in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan (trieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -52,12 +52,12 @@ type ( // GenerateAccountTrieRoot takes an account iterator and reproduces the root hash. func GenerateAccountTrieRoot(it AccountIterator) (common.Hash, error) { - return generateTrieRoot(nil, nil, it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) + return generateTrieRoot(nil, "", it, common.Hash{}, stackTrieGenerate, nil, newGenerateStats(), true) } // GenerateStorageTrieRoot takes a storage iterator and reproduces the root hash. func GenerateStorageTrieRoot(account common.Hash, it StorageIterator) (common.Hash, error) { - return generateTrieRoot(nil, nil, it, account, stackTrieGenerate, nil, newGenerateStats(), true) + return generateTrieRoot(nil, "", it, account, stackTrieGenerate, nil, newGenerateStats(), true) } // GenerateTrie takes the whole snapshot tree as the input, traverses all the @@ -243,7 +243,7 @@ func runReport(stats *generateStats, stop chan bool) { // generateTrieRoot generates the trie hash based on the snapshot iterator. // It can be used for generating account trie, storage trie or even the // whole state which connects the accounts and the corresponding storages. -func generateTrieRoot(db ethdb.KeyValueWriter, scheme trie.NodeScheme, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { +func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, account common.Hash, generatorFn trieGeneratorFn, leafCallback leafCallbackFn, stats *generateStats, report bool) (common.Hash, error) { var ( in = make(chan trieKV) // chan to pass leaves out = make(chan common.Hash, 1) // chan to collect result @@ -361,13 +361,13 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme trie.NodeScheme, it Iterat return stop(nil) } -func stackTrieGenerate(db ethdb.KeyValueWriter, scheme trie.NodeScheme, owner common.Hash, in chan trieKV, out chan common.Hash) { +func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan trieKV, out chan common.Hash) { var nodeWriter trie.NodeWriteFunc // Implement nodeWriter in case db is existed otherwise let it be nil. if db != nil { nodeWriter = func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - scheme.WriteTrieNode(db, owner, path, hash, blob) + rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme) } } t := trie.NewStackTrieWithOwner(nodeWriter, owner) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 3d59590c89..f5967b6674 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -115,12 +115,12 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { t.Helper() accIt := snap.AccountIterator(common.Hash{}) defer accIt.Release() - snapRoot, err := generateTrieRoot(nil, nil, accIt, common.Hash{}, stackTrieGenerate, + snapRoot, err := generateTrieRoot(nil, "", accIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) defer storageIt.Release() - hash, err := generateTrieRoot(nil, nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) if err != nil { return common.Hash{}, err } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index f111c96313..1fcb40a354 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -767,14 +767,14 @@ func (t *Tree) Verify(root common.Hash) error { } defer acctIt.Release() - got, err := generateTrieRoot(nil, nil, acctIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { + got, err := generateTrieRoot(nil, "", acctIt, common.Hash{}, stackTrieGenerate, func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { storageIt, err := t.StorageIterator(root, accountHash, common.Hash{}) if err != nil { return common.Hash{}, err } defer storageIt.Release() - hash, err := generateTrieRoot(nil, nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) + hash, err := generateTrieRoot(nil, "", storageIt, accountHash, stackTrieGenerate, nil, stat, false) if err != nil { return common.Hash{}, err } diff --git a/core/state/sync.go b/core/state/sync.go index e2b414259a..104f499fb0 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -27,7 +27,7 @@ import ( ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.SyncBloom, onLeaf func(keys [][]byte, leaf []byte) error, scheme trie.NodeScheme) *trie.Sync { +func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.SyncBloom, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync { // Register the storage slot callback if the external callback is specified. var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error if onLeaf != nil { diff --git a/core/state/sync_test.go b/core/state/sync_test.go index b35830d1a9..71ce0d235d 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -663,14 +663,14 @@ func TestIncompleteStateSync(t *testing.T) { for i, path := range addedPaths { owner, inner := trie.ResolvePath([]byte(path)) hash := addedHashes[i] - val := scheme.ReadTrieNode(dstDb, owner, inner, hash) + val := rawdb.ReadTrieNode(dstDb, owner, inner, hash, scheme) if val == nil { t.Error("missing trie node") } - scheme.DeleteTrieNode(dstDb, owner, inner, hash) + rawdb.DeleteTrieNode(dstDb, owner, inner, hash, scheme) if err := checkStateConsistency(dstDb, srcRoot); err == nil { t.Errorf("trie inconsistency not caught, missing: %v", path) } - scheme.WriteTrieNode(dstDb, owner, inner, hash, val) + rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme) } } diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index c805f76c11..a038316347 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -502,7 +502,7 @@ func testGetNodeData(t *testing.T, protocol uint, drop bool) { // Reconstruct state tree from the received data. reconstructDB := rawdb.NewMemoryDatabase() for i := 0; i < len(data); i++ { - rawdb.WriteTrieNode(reconstructDB, hashes[i], data[i]) + rawdb.WriteLegacyTrieNode(reconstructDB, hashes[i], data[i]) } // Sanity check whether all state matches. diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index b798f9afa2..d04358ebf3 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -415,7 +415,7 @@ type SyncPeer interface { // - The peer delivers a refusal to serve the requested state type Syncer struct { db ethdb.KeyValueStore // Database to store the trie nodes into (and dedup) - scheme trie.NodeScheme // Node scheme used in node database + scheme string // Node scheme used in node database root common.Hash // Current state trie root being synced tasks []*accountTask // Current account task set being synced @@ -481,7 +481,7 @@ type Syncer struct { // NewSyncer creates a new snapshot syncer to download the Ethereum state over the // snap protocol. -func NewSyncer(db ethdb.KeyValueStore, scheme trie.NodeScheme) *Syncer { +func NewSyncer(db ethdb.KeyValueStore, scheme string) *Syncer { return &Syncer{ db: db, scheme: scheme, @@ -721,7 +721,7 @@ func (s *Syncer) loadSyncStatus() { }, } task.genTrie = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - s.scheme.WriteTrieNode(task.genBatch, owner, path, hash, val) + rawdb.WriteTrieNode(task.genBatch, owner, path, hash, val, s.scheme) }) for accountHash, subtasks := range task.SubTasks { @@ -733,7 +733,7 @@ func (s *Syncer) loadSyncStatus() { }, } subtask.genTrie = trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - s.scheme.WriteTrieNode(subtask.genBatch, owner, path, hash, val) + rawdb.WriteTrieNode(subtask.genBatch, owner, path, hash, val, s.scheme) }, accountHash) } } @@ -789,7 +789,7 @@ func (s *Syncer) loadSyncStatus() { SubTasks: make(map[common.Hash][]*storageTask), genBatch: batch, genTrie: trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - s.scheme.WriteTrieNode(batch, owner, path, hash, val) + rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) }), }) log.Debug("Created account sync task", "from", next, "last", last) @@ -1803,7 +1803,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { } // Check if the account is a contract with an unknown storage trie if account.Root != emptyRoot { - if !s.scheme.HasTrieNode(s.db, res.hashes[i], nil, account.Root) { + if !rawdb.HasTrieNode(s.db, res.hashes[i], nil, account.Root, s.scheme) { // If there was a previous large state retrieval in progress, // don't restart it from scratch. This happens if a sync cycle // is interrupted and resumed later. However, *do* update the @@ -1976,7 +1976,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { root: acc.Root, genBatch: batch, genTrie: trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - s.scheme.WriteTrieNode(batch, owner, path, hash, val) + rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) }, account), }) for r.Next() { @@ -1992,7 +1992,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { root: acc.Root, genBatch: batch, genTrie: trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - s.scheme.WriteTrieNode(batch, owner, path, hash, val) + rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) }, account), }) } @@ -2039,7 +2039,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { if i < len(res.hashes)-1 || res.subTask == nil { tr := trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - s.scheme.WriteTrieNode(batch, owner, path, hash, val) + rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) }, account) for j := 0; j < len(res.hashes[i]); j++ { tr.Update(res.hashes[i][j][:], res.slots[i][j]) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 940357f412..4bddbf7266 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -624,7 +624,7 @@ func TestSyncBloatedProof(t *testing.T) { } } -func setupSyncer(scheme trie.NodeScheme, peers ...*testPeer) *Syncer { +func setupSyncer(scheme string, peers ...*testPeer) *Syncer { stateDb := rawdb.NewMemoryDatabase() syncer := NewSyncer(stateDb, scheme) for _, peer := range peers { @@ -1355,7 +1355,7 @@ func getCodeByHash(hash common.Hash) []byte { } // makeAccountTrieNoStorage spits out a trie, along with the leafs -func makeAccountTrieNoStorage(n int) (trie.NodeScheme, *trie.Trie, entrySlice) { +func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) { // Create emptry Trie var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) @@ -1390,7 +1390,7 @@ func makeAccountTrieNoStorage(n int) (trie.NodeScheme, *trie.Trie, entrySlice) { // makeBoundaryAccountTrie constructs an account trie. Instead of filling // accounts normally, this function will fill a few accounts which have // boundary hash. -func makeBoundaryAccountTrie(n int) (trie.NodeScheme, *trie.Trie, entrySlice) { +func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) { var ( entries entrySlice boundaries []common.Hash @@ -1450,7 +1450,7 @@ func makeBoundaryAccountTrie(n int) (trie.NodeScheme, *trie.Trie, entrySlice) { // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts // has a unique storage set. Code is true when u pass a random code hash to the account -func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (trie.NodeScheme, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) @@ -1503,7 +1503,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) } // makeAccountTrieWithStorage spits out a trie, along with the leafs -func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (trie.NodeScheme, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go index 109406d1e1..ab637529ef 100644 --- a/les/downloader/downloader.go +++ b/les/downloader/downloader.go @@ -229,7 +229,7 @@ func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, headerProcCh: make(chan []*types.Header, 1), quitCh: make(chan struct{}), stateCh: make(chan dataPack), - SnapSyncer: snap.NewSyncer(stateDb, nil), + SnapSyncer: snap.NewSyncer(stateDb, ""), stateSyncStart: make(chan *stateSync), syncStatsState: stateSyncStats{ processed: rawdb.ReadFastTrieProgress(stateDb), diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index b230e8fd98..074e7b1c30 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -151,7 +151,7 @@ func (f *fuzzer) fuzz() int { spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB)) trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - dbB.Scheme().WriteTrieNode(spongeB, owner, path, hash, blob) + rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme()) }) vals kvs useful bool diff --git a/trie/database.go b/trie/database.go index c3add0a8a2..5a2b7dad32 100644 --- a/trie/database.go +++ b/trie/database.go @@ -415,7 +415,7 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { memcacheDirtyMissMeter.Mark(1) // Content unavailable in memory, attempt to retrieve from disk - enc := rawdb.ReadTrieNode(db.diskdb, hash) + enc := rawdb.ReadLegacyTrieNode(db.diskdb, hash) if len(enc) != 0 { if db.cleans != nil { db.cleans.Set(hash[:], enc) @@ -580,7 +580,7 @@ func (db *Database) Cap(limit common.StorageSize) error { for size > limit && oldest != (common.Hash{}) { // Fetch the oldest referenced node and push into the batch node := db.dirties[oldest] - rawdb.WriteTrieNode(batch, oldest, node.rlp()) + rawdb.WriteLegacyTrieNode(batch, oldest, node.rlp()) // If we exceeded the ideal batch size, commit and reset if batch.ValueSize() >= ethdb.IdealBatchSize { @@ -710,7 +710,7 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane return err } // If we've reached an optimal batch size, commit and start over - rawdb.WriteTrieNode(batch, hash, node.rlp()) + rawdb.WriteLegacyTrieNode(batch, hash, node.rlp()) if callback != nil { callback(hash) } @@ -867,6 +867,6 @@ func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, st } // Scheme returns the node scheme used in the database. Right now, we only support hash scheme. -func (db *Database) Scheme() NodeScheme { - return &hashScheme{} +func (db *Database) Scheme() string { + return rawdb.HashScheme } diff --git a/trie/schema.go b/trie/schema.go deleted file mode 100644 index 72b67aa7d9..0000000000 --- a/trie/schema.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package trie - -import ( - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb" -) - -const ( - HashScheme = "hashScheme" // Identifier of hash based node scheme - - // Path-based scheme will be introduced in the following PRs. - // PathScheme = "pathScheme" // Identifier of path based node scheme -) - -// NodeShceme desribes the scheme for interacting nodes in disk. -type NodeScheme interface { - // Name returns the identifier of node scheme. - Name() string - - // HasTrieNode checks the trie node presence with the provided node info and - // the associated node hash. - HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) bool - - // ReadTrieNode retrieves the trie node from database with the provided node - // info and the associated node hash. - ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) []byte - - // WriteTrieNode writes the trie node into database with the provided node - // info and associated node hash. - WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte) - - // DeleteTrieNode deletes the trie node from database with the provided node - // info and associated node hash. - DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash) - - // IsTrieNode returns an indicator if the given database key is the key of - // trie node according to the scheme. - IsTrieNode(key []byte) (bool, []byte) -} - -type hashScheme struct{} - -// Name returns the identifier of hash based scheme. -func (scheme *hashScheme) Name() string { - return HashScheme -} - -// HasTrieNode checks the trie node presence with the provided node info and -// the associated node hash. -func (scheme *hashScheme) HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) bool { - return rawdb.HasTrieNode(db, hash) -} - -// ReadTrieNode retrieves the trie node from database with the provided node info -// and associated node hash. -func (scheme *hashScheme) ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash) []byte { - return rawdb.ReadTrieNode(db, hash) -} - -// WriteTrieNode writes the trie node into database with the provided node info -// and associated node hash. -func (scheme *hashScheme) WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte) { - rawdb.WriteTrieNode(db, hash, node) -} - -// DeleteTrieNode deletes the trie node from database with the provided node info -// and associated node hash. -func (scheme *hashScheme) DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash) { - rawdb.DeleteTrieNode(db, hash) -} - -// IsTrieNode returns an indicator if the given database key is the key of trie -// node according to the scheme. -func (scheme *hashScheme) IsTrieNode(key []byte) (bool, []byte) { - if len(key) == common.HashLength { - return true, key - } - return false, nil -} diff --git a/trie/sync.go b/trie/sync.go index 1ea443e3f7..a406d281e8 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -137,7 +137,7 @@ func (batch *syncMemBatch) hasCode(hash common.Hash) bool { // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. type Sync struct { - scheme NodeScheme // Node scheme descriptor used in database. + scheme string // Node scheme descriptor used in database. database ethdb.KeyValueReader // Persistent database to check for existing entries membatch *syncMemBatch // Memory buffer to avoid frequent database writes nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path @@ -164,7 +164,7 @@ type Sync struct { type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error // NewSync creates a new trie data download scheduler. -func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallback, bloom *SyncBloom, scheme NodeScheme) *Sync { +func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallback, bloom *SyncBloom, scheme string) *Sync { ts := &Sync{ scheme: scheme, database: database, @@ -194,7 +194,8 @@ func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, par // Bloom filter says this might be a duplicate, double check. // If database says yes, then at least the trie node is present // and we hold the assumption that it's NOT legacy contract code. - blob := rawdb.ReadTrieNode(s.database, root) + owner, inner := ResolvePath(path) + blob := rawdb.ReadTrieNode(s.database, owner, inner, root, s.scheme) if len(blob) > 0 { return } @@ -362,7 +363,7 @@ func (s *Sync) Commit(dbw ethdb.Batch) error { // Dump the membatch into a database dbw for path, value := range s.membatch.nodes { owner, inner := ResolvePath([]byte(path)) - s.scheme.WriteTrieNode(dbw, owner, inner, s.membatch.hashes[path], value) + rawdb.WriteTrieNode(dbw, owner, inner, s.membatch.hashes[path], value, s.scheme) hash := s.membatch.hashes[path] if s.bloom != nil { s.bloom.Add(hash[:]) @@ -481,7 +482,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // If database says yes, then at least the trie node is present // and we hold the assumption that it's NOT legacy contract code. owner, inner := ResolvePath(child.path) - if s.scheme.HasTrieNode(s.database, owner, inner, chash) { + if rawdb.HasTrieNode(s.database, owner, inner, chash, s.scheme) { continue } diff --git a/trie/trie_test.go b/trie/trie_test.go index c9533060c1..f065cf8b3a 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -817,7 +817,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} writeFn := func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - db.Scheme().WriteTrieNode(stackTrieSponge, owner, path, hash, blob) + rawdb.WriteTrieNode(stackTrieSponge, owner, path, hash, blob, db.Scheme()) } stTrie := NewStackTrie(writeFn) // Fill the trie with elements, should start 0, otherwise nodes will be nil in the first time. @@ -877,7 +877,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} writeFn := func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - db.Scheme().WriteTrieNode(stackTrieSponge, owner, path, hash, blob) + rawdb.WriteTrieNode(stackTrieSponge, owner, path, hash, blob, db.Scheme()) } stTrie := NewStackTrie(writeFn) // Add a single small-element to the trie(s)