From 15c46fe2b2a26f2783bdbe0e92d3e4f56ab75a4d Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Fri, 10 May 2024 14:18:10 +0800 Subject: [PATCH 01/10] all: introduce state reader interface --- cmd/evm/internal/t8ntool/execution.go | 9 +- cmd/evm/runner.go | 6 +- cmd/evm/staterunner.go | 2 +- cmd/geth/chaincmd.go | 2 +- core/blockchain.go | 14 +- core/blockchain_reader.go | 11 +- core/blockchain_sethead_test.go | 2 +- core/blockchain_test.go | 2 +- core/chain_makers.go | 4 +- core/genesis.go | 6 +- core/state/database.go | 143 +++++----- core/state/iterator_test.go | 2 +- core/state/reader.go | 287 +++++++++++++++++++++ core/state/state_object.go | 46 +--- core/state/state_test.go | 18 +- core/state/statedb.go | 158 ++++-------- core/state/statedb_fuzz_test.go | 3 +- core/state/statedb_test.go | 90 ++++--- core/state/sync_test.go | 8 +- core/state/trie_prefetcher_test.go | 2 +- core/stateless.go | 2 +- core/txpool/blobpool/blobpool_test.go | 12 +- core/txpool/legacypool/legacypool2_test.go | 8 +- core/txpool/legacypool/legacypool_test.go | 40 +-- core/vm/gas_table_test.go | 4 +- core/vm/instructions_test.go | 2 +- core/vm/interpreter_test.go | 2 +- core/vm/runtime/runtime.go | 4 +- core/vm/runtime/runtime_test.go | 10 +- eth/api_debug_test.go | 21 +- eth/state_accessor.go | 13 +- internal/ethapi/api_test.go | 2 +- miner/miner_test.go | 2 +- tests/state_test_util.go | 9 +- 34 files changed, 590 insertions(+), 356 deletions(-) create mode 100644 core/state/reader.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index a4c5f6efcb0a..66f57c7f9ebe 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -372,7 +372,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } // Re-create statedb instance with new root upon the updated database // for accessing latest states. - statedb, err = state.New(root, statedb.Database(), nil) + statedb, err = state.New(root, statedb.Database()) if err != nil { return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err)) } @@ -381,8 +381,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) - statedb, _ := state.New(types.EmptyRootHash, sdb, nil) + tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true}) + sdb := state.NewDatabase(db, tdb, nil) + statedb, _ := state.New(types.EmptyRootHash, sdb) for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) @@ -393,7 +394,7 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc) *state.StateDB } // Commit and re-open to start with a clean state. root, _ := statedb.Commit(0, false) - statedb, _ = state.New(root, sdb, nil) + statedb, _ = state.New(root, sdb) return statedb } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index f179e733e657..7262649aa50b 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -155,8 +155,8 @@ func runCmd(ctx *cli.Context) error { }) defer triedb.Close() genesis := genesisConfig.MustCommit(db, triedb) - sdb := state.NewDatabaseWithNodeDB(db, triedb) - statedb, _ = state.New(genesis.Root(), sdb, nil) + sdb := state.NewDatabase(db, triedb, nil) + statedb, _ = state.New(genesis.Root(), sdb) chainConfig = genesisConfig.Config if ctx.String(SenderFlag.Name) != "" { @@ -277,7 +277,7 @@ func runCmd(ctx *cli.Context) error { fmt.Printf("Failed to commit changes %v\n", err) return err } - dumpdb, err := state.New(root, sdb, nil) + dumpdb, err := state.New(root, sdb) if err != nil { fmt.Printf("Failed to open statedb %v\n", err) return err diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index fc2bf8223f30..4514367e8a45 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -107,7 +107,7 @@ func runStateTest(fname string, cfg vm.Config, dump bool) error { result.Root = &root fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) if dump { // Dump any state to aid debugging - cpy, _ := state.New(root, tstate.StateDB.Database(), nil) + cpy, _ := state.New(root, tstate.StateDB.Database()) dump := cpy.RawDump(nil) result.State = &dump } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 9450c09e7e2b..2301ac2f8a33 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -586,7 +586,7 @@ func dump(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, db, true, true, false) // always enable preimage lookup defer triedb.Close() - state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil) + state, err := state.New(root, state.NewDatabase(db, triedb, nil)) if err != nil { return err } diff --git a/core/blockchain.go b/core/blockchain.go index 05ebfd18b830..23efc04cad61 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -217,7 +217,7 @@ type BlockChain struct { lastWrite uint64 // Last block when the state was flushed flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state triedb *triedb.Database // The database handler for maintaining trie nodes. - stateCache state.Database // State database to reuse between imports (contains state cache) + stateDb *state.CachingDB // State database to reuse between imports (contains state cache) txIndexer *txIndexer // Transaction indexer, might be nil if not enabled hc *HeaderChain @@ -310,7 +310,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit)) bc.forker = NewForkChoice(bc, shouldPreserve) - bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb) + bc.stateDb = state.NewDatabase(bc.db, bc.triedb, nil) bc.validator = NewBlockValidator(chainConfig, bc) bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc) bc.processor = NewStateProcessor(chainConfig, bc.hc) @@ -448,7 +448,13 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis AsyncBuild: !bc.cacheConfig.SnapshotWait, } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) + + // Register the snapshot into statedb. It's an ugly hack as state snapshot + // is constructed after stateDb. TODO(rjl493456442) improve the construction + // logic. + bc.stateDb.SetSnapshot(bc.snaps) } + // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -1800,7 +1806,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) if parent == nil { parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1) } - statedb, err := state.New(parent.Root, bc.stateCache, bc.snaps) + statedb, err := state.New(parent.Root, bc.stateDb) if err != nil { return it.index, err } @@ -1826,7 +1832,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) var followupInterrupt atomic.Bool if !bc.cacheConfig.TrieCleanNoPrefetch { if followup, err := it.peek(); followup != nil && err == nil { - throwaway, _ := state.New(parent.Root, bc.stateCache, bc.snaps) + throwaway, _ := state.New(parent.Root, bc.stateDb) go func(start time.Time, followup *types.Block, throwaway *state.StateDB) { // Disable tracing for prefetcher executions. diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 8a85800dd877..5de9de062b71 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -308,7 +308,7 @@ func (bc *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { - _, err := bc.stateCache.OpenTrie(hash) + _, err := bc.stateDb.OpenTrie(hash) return err == nil } @@ -341,12 +341,9 @@ func (bc *BlockChain) stateRecoverable(root common.Hash) bool { // If the code doesn't exist in the in-memory cache, check the storage with // new code scheme. func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) ([]byte, error) { - type codeReader interface { - ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) - } // TODO(rjl493456442) The associated account address is also required // in Verkle scheme. Fix it once snap-sync is supported for Verkle. - return bc.stateCache.(codeReader).ContractCodeWithPrefix(common.Address{}, hash) + return bc.stateDb.ContractCodeWithPrefix(common.Address{}, hash) } // State returns a new mutable state based on the current HEAD block. @@ -356,7 +353,7 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return state.New(root, bc.stateCache, bc.snaps) + return state.New(root, bc.stateDb) } // Config retrieves the chain's fork configuration. @@ -382,7 +379,7 @@ func (bc *BlockChain) Processor() Processor { // StateCache returns the caching database underpinning the blockchain instance. func (bc *BlockChain) StateCache() state.Database { - return bc.stateCache + return bc.stateDb } // GasLimit returns the gas limit of the current HEAD block. diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 8b77f9f8b20c..b391659a92ac 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -2040,7 +2040,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme dbconfig.HashDB = hashdb.Defaults } chain.triedb = triedb.NewDatabase(chain.db, dbconfig) - chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb) + chain.stateDb = state.NewDatabase(chain.db, chain.triedb, chain.snaps) // Force run a freeze cycle type freezer interface { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 4f28c6f5e681..ff0f67f83314 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -159,7 +159,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateCache, nil) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateDb) if err != nil { return err } diff --git a/core/chain_makers.go b/core/chain_makers.go index 58985347bb31..8406410445d3 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -368,7 +368,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse defer triedb.Close() for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil) + statedb, err := state.New(parent.Root(), state.NewDatabase(db, triedb, nil)) if err != nil { panic(err) } @@ -475,7 +475,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine } for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, trdb), nil) + statedb, err := state.New(parent.Root(), state.NewDatabase(db, trdb, nil)) if err != nil { panic(err) } diff --git a/core/genesis.go b/core/genesis.go index 4ca24807fccd..0d268b2f1e8c 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -127,8 +127,8 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { } // Create an ephemeral in-memory database for computing hash, // all the derived states will be discarded to not pollute disk. - db := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), config) - statedb, err := state.New(types.EmptyRootHash, db, nil) + db := rawdb.NewMemoryDatabase() + statedb, err := state.New(types.EmptyRootHash, state.NewDatabase(db, triedb.NewDatabase(db, config), nil)) if err != nil { return common.Hash{}, err } @@ -149,7 +149,7 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { // states will be persisted into the given database. Also, the genesis state // specification will be flushed as well. func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database, blockhash common.Hash) error { - statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) + statedb, err := state.New(types.EmptyRootHash, state.NewDatabase(db, triedb, nil)) if err != nil { return err } diff --git a/core/state/database.go b/core/state/database.go index d54417d2f91e..3a23add82892 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -45,15 +46,15 @@ const ( // Database wraps access to tries and contract code. type Database interface { + // Reader returns a state reader associated with the specified state root. + Reader(root common.Hash) (Reader, error) + // OpenTrie opens the main account trie. OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) - // CopyTrie returns an independent copy of the given trie. - CopyTrie(Trie) Trie - // ContractCode retrieves a particular contract's code. ContractCode(addr common.Address, codeHash common.Hash) ([]byte, error) @@ -68,6 +69,9 @@ type Database interface { // TrieDB returns the underlying trie database for managing trie nodes. TrieDB() *triedb.Database + + // Snapshot returns the underlying state snapshot. + Snapshot() *snapshot.Tree } // Trie is a Ethereum Merkle Patricia trie. @@ -147,86 +151,81 @@ type Trie interface { IsVerkle() bool } -// NewDatabase creates a backing store for state. The returned database is safe for -// concurrent use, but does not retain any recent trie nodes in memory. To keep some -// historical state in memory, use the NewDatabaseWithConfig constructor. -func NewDatabase(db ethdb.Database) Database { - return NewDatabaseWithConfig(db, nil) +// CachingDB is an implementation of Database interface. It leverages both trie and +// state snapshot to provide functionalities for state access. It's meant to be a +// long-live object and has a few caches inside for sharing between blocks. +type CachingDB struct { + disk ethdb.KeyValueStore + triedb *triedb.Database + snap *snapshot.Tree + codeCache *lru.SizeConstrainedCache[common.Hash, []byte] + codeSizeCache *lru.Cache[common.Hash, int] + pointCache *utils.PointCache } -// NewDatabaseWithConfig creates a backing store for state. The returned database -// is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a -// large memory cache. -func NewDatabaseWithConfig(db ethdb.Database, config *triedb.Config) Database { - return &cachingDB{ - disk: db, - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), +// NewDatabase creates a state database with the provided data sources. +func NewDatabase(disk ethdb.Database, triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { + return &CachingDB{ + disk: disk, + triedb: triedb, + snap: snap, codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: triedb.NewDatabase(db, config), + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), pointCache: utils.NewPointCache(pointCacheSize), } } -// NewDatabaseWithNodeDB creates a state database with an already initialized node database. -func NewDatabaseWithNodeDB(db ethdb.Database, triedb *triedb.Database) Database { - return &cachingDB{ - disk: db, - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - triedb: triedb, - pointCache: utils.NewPointCache(pointCacheSize), - } +// NewDatabaseForTesting is similar to NewDatabase, but it sets up the different +// data sources using the same provided database with default config for testing. +func NewDatabaseForTesting(db ethdb.Database) *CachingDB { + return NewDatabase(db, triedb.NewDatabase(db, nil), nil) } -type cachingDB struct { - disk ethdb.KeyValueStore - codeSizeCache *lru.Cache[common.Hash, int] - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - triedb *triedb.Database - pointCache *utils.PointCache +// Reader returns a state reader associated with the specified state root. +func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { + var readers []Reader + + // Set up the state snapshot reader if available. This feature + // is optional and may be partially useful if it's not fully + // generated. + if db.snap != nil { + sr, err := newStateReader(stateRoot, db.snap) + if err == nil { + readers = append(readers, sr) // snap reader is optional + } + } + // Set up the trie reader, which is expected to always be available + // as the gatekeeper unless the state is corrupted. + tr, err := newTrieReader(stateRoot, db.triedb, db.pointCache) + if err != nil { + return nil, err + } + readers = append(readers, tr) + + return newMultiReader(readers...) } // OpenTrie opens the main account trie at a specific root hash. -func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { +func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { if db.triedb.IsVerkle() { return trie.NewVerkleTrie(root, db.triedb, db.pointCache) } - tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) - if err != nil { - return nil, err - } - return tr, nil + return trie.NewStateTrie(trie.StateTrieID(root), db.triedb) } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { +func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { // In the verkle case, there is only one tree. But the two-tree structure // is hardcoded in the codebase. So we need to return the same trie in this // case. if db.triedb.IsVerkle() { return self, nil } - tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) - if err != nil { - return nil, err - } - return tr, nil -} - -// CopyTrie returns an independent copy of the given trie. -func (db *cachingDB) CopyTrie(t Trie) Trie { - switch t := t.(type) { - case *trie.StateTrie: - return t.Copy() - case *trie.VerkleTrie: - return t.Copy() - default: - panic(fmt.Errorf("unknown trie type %T", t)) - } + return trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) } // ContractCode retrieves a particular contract's code. -func (db *cachingDB) ContractCode(address common.Address, codeHash common.Hash) ([]byte, error) { +func (db *CachingDB) ContractCode(address common.Address, codeHash common.Hash) ([]byte, error) { code, _ := db.codeCache.Get(codeHash) if len(code) > 0 { return code, nil @@ -243,7 +242,7 @@ func (db *cachingDB) ContractCode(address common.Address, codeHash common.Hash) // ContractCodeWithPrefix retrieves a particular contract's code. If the // code can't be found in the cache, then check the existence with **new** // db scheme. -func (db *cachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) { +func (db *CachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) { code, _ := db.codeCache.Get(codeHash) if len(code) > 0 { return code, nil @@ -258,7 +257,7 @@ func (db *cachingDB) ContractCodeWithPrefix(address common.Address, codeHash com } // ContractCodeSize retrieves a particular contracts code's size. -func (db *cachingDB) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { +func (db *CachingDB) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { if cached, ok := db.codeSizeCache.Get(codeHash); ok { return cached, nil } @@ -267,16 +266,38 @@ func (db *cachingDB) ContractCodeSize(addr common.Address, codeHash common.Hash) } // DiskDB returns the underlying key-value disk database. -func (db *cachingDB) DiskDB() ethdb.KeyValueStore { +func (db *CachingDB) DiskDB() ethdb.KeyValueStore { return db.disk } // TrieDB retrieves any intermediate trie-node caching layer. -func (db *cachingDB) TrieDB() *triedb.Database { +func (db *CachingDB) TrieDB() *triedb.Database { return db.triedb } // PointCache returns the cache of evaluated curve points. -func (db *cachingDB) PointCache() *utils.PointCache { +func (db *CachingDB) PointCache() *utils.PointCache { return db.pointCache } + +// Snapshot returns the underlying state snapshot. +func (db *CachingDB) Snapshot() *snapshot.Tree { + return db.snap +} + +// SetSnapshot sets the provided state snapshot. +func (db *CachingDB) SetSnapshot(snap *snapshot.Tree) { + db.snap = snap +} + +// mustCopyTrie returns a deep-copied trie. +func mustCopyTrie(t Trie) Trie { + switch t := t.(type) { + case *trie.StateTrie: + return t.Copy() + case *trie.VerkleTrie: + return t.Copy() + default: + panic(fmt.Errorf("unknown trie type %T", t)) + } +} diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index 73cc22490b6e..26456d7a8934 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -35,7 +35,7 @@ func testNodeIteratorCoverage(t *testing.T, scheme string) { db, sdb, ndb, root, _ := makeTestState(scheme) ndb.Commit(root, false) - state, err := New(root, sdb, nil) + state, err := New(root, sdb) if err != nil { t.Fatalf("failed to create state trie at %x: %v", root, err) } diff --git a/core/state/reader.go b/core/state/reader.go new file mode 100644 index 000000000000..7d50c0ea1b1f --- /dev/null +++ b/core/state/reader.go @@ -0,0 +1,287 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/utils" + "github.com/ethereum/go-ethereum/triedb" +) + +// Reader defines the interface for accessing accounts and storage slots +// associated with a specific state. +type Reader interface { + // Account retrieves the account associated with a particular address. + // + // - Returns a nil account if it does not exist + // - Returns an error only if an unexpected issue occurs + // - The returned account is safe to modify after the call + Account(addr common.Address) (*types.StateAccount, error) + + // Storage retrieves the storage slot associated with a particular account + // address and slot key. + // + // - Returns an empty slot if it does not exist + // - Returns an error only if an unexpected issue occurs + // - The returned storage slot is safe to modify after the call + Storage(addr common.Address, storageRoot common.Hash, slot common.Hash) (common.Hash, error) + + // Copy returns a deep-copied state reader. + Copy() Reader +} + +// stateReader is a wrapper over the state snapshot and implements the Reader +// interface. It provides an efficient way to access flat state. +type stateReader struct { + snap snapshot.Snapshot + buff crypto.KeccakState +} + +// newStateReader constructs a flat state reader with on the specified state root. +func newStateReader(root common.Hash, snaps *snapshot.Tree) (*stateReader, error) { + snap := snaps.Snapshot(root) + if snap == nil { + return nil, errors.New("snapshot is not available") + } + return &stateReader{ + snap: snap, + buff: crypto.NewKeccakState(), + }, nil +} + +// Account implements Reader, retrieving the account specified by the address. +// +// An error will be returned if the associated snapshot is already stale or +// the requested account is not yet covered by the snapshot. +// +// The returned account might be nil if it's not existent. +func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) { + ret, err := r.snap.Account(crypto.HashData(r.buff, addr.Bytes())) + if err != nil { + return nil, err + } + if ret == nil { + return nil, nil + } + acct := &types.StateAccount{ + Nonce: ret.Nonce, + Balance: ret.Balance, + CodeHash: ret.CodeHash, + Root: common.BytesToHash(ret.Root), + } + if len(acct.CodeHash) == 0 { + acct.CodeHash = types.EmptyCodeHash.Bytes() + } + if acct.Root == (common.Hash{}) { + acct.Root = types.EmptyRootHash + } + return acct, nil +} + +// Storage implements Reader, retrieving the storage slot specified by the +// address and slot key. +// +// An error will be returned if the associated snapshot is already stale or +// the requested storage slot is not yet covered by the snapshot. +// +// The returned storage slot might be empty if it's not existent. +func (r *stateReader) Storage(addr common.Address, root common.Hash, key common.Hash) (common.Hash, error) { + addrHash := crypto.HashData(r.buff, addr.Bytes()) + slotHash := crypto.HashData(r.buff, key.Bytes()) + ret, err := r.snap.Storage(addrHash, slotHash) + if err != nil { + return common.Hash{}, err + } + if len(ret) == 0 { + return common.Hash{}, nil + } + _, content, _, err := rlp.Split(ret) + if err != nil { + return common.Hash{}, err + } + var value common.Hash + value.SetBytes(content) + return value, nil +} + +// Copy implements Reader, returning a deep-copied snap reader. +func (r *stateReader) Copy() Reader { + return &stateReader{ + snap: r.snap, + buff: crypto.NewKeccakState(), + } +} + +// trieReader implements the Reader interface, providing functions to access +// state from the referenced trie. +type trieReader struct { + root common.Hash // State root which uniquely represent a state. + db *triedb.Database // Database for loading trie + buff crypto.KeccakState // Buffer for keccak256 hashing. + mainTrie Trie // Main trie, resolved in constructor + subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved. +} + +// trieReader constructs a trie reader of the specific state. An error will be +// returned if the associated trie specified by root is not existent. +func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCache) (*trieReader, error) { + var ( + tr Trie + err error + ) + if !db.IsVerkle() { + tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) + } else { + tr, err = trie.NewVerkleTrie(root, db, cache) + } + if err != nil { + return nil, err + } + return &trieReader{ + root: root, + db: db, + buff: crypto.NewKeccakState(), + mainTrie: tr, + subTries: make(map[common.Address]Trie), + }, nil +} + +// Account implements Reader, retrieving the account specified by the address. +// +// An error will be returned if the trie state is corrupted. An nil account +// will be returned if it's not existent in the trie. +func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { + return r.mainTrie.GetAccount(addr) +} + +// Storage implements Reader, retrieving the storage slot specified by the +// address and slot key. +// +// An error will be returned if the trie state is corrupted. An empty storage +// slot will be returned if it's not existent in the trie. +func (r *trieReader) Storage(addr common.Address, root common.Hash, key common.Hash) (common.Hash, error) { + var ( + tr Trie + found bool + value common.Hash + ) + if r.db.IsVerkle() { + tr = r.mainTrie + } else { + tr, found = r.subTries[addr] + if !found { + var err error + tr, err = trie.NewStateTrie(trie.StorageTrieID(r.root, crypto.HashData(r.buff, addr.Bytes()), root), r.db) + if err != nil { + return common.Hash{}, err + } + r.subTries[addr] = tr + } + } + ret, err := tr.GetStorage(addr, key.Bytes()) + if err != nil { + return common.Hash{}, err + } + value.SetBytes(ret) + return value, nil +} + +// Copy implements Reader, returning a deep-copied trie reader. +func (r *trieReader) Copy() Reader { + tries := make(map[common.Address]Trie) + for addr, tr := range r.subTries { + tries[addr] = mustCopyTrie(tr) + } + return &trieReader{ + root: r.root, + db: r.db, + buff: crypto.NewKeccakState(), + mainTrie: mustCopyTrie(r.mainTrie), + subTries: tries, + } +} + +// multiReader is the aggregation of a list of Reader interface, providing state +// access by leveraging all readers. The checking priority is determined by the +// position in the reader list. +type multiReader struct { + readers []Reader // List of readers, sorted by checking priority +} + +// newMultiReader constructs a multiReader instance with the given readers. The +// priority among readers is assumed to be sorted. Note, it must contain at least +// one reader for constructing a multiReader. +func newMultiReader(readers ...Reader) (*multiReader, error) { + if len(readers) == 0 { + return nil, errors.New("empty reader set") + } + return &multiReader{ + readers: readers, + }, nil +} + +// Account implementing Reader interface, retrieving the account associated with +// a particular address. +// +// - Returns a nil account if it does not exist +// - Returns an error only if an unexpected issue occurs +// - The returned account is safe to modify after the call +func (r *multiReader) Account(addr common.Address) (*types.StateAccount, error) { + var errs []error + for _, reader := range r.readers { + acct, err := reader.Account(addr) + if err == nil { + return acct, nil + } + errs = append(errs, err) + } + return nil, errors.Join(errs...) +} + +// Storage implementing Reader interface, retrieving the storage slot associated +// with a particular account address and slot key. +// +// - Returns an empty slot if it does not exist +// - Returns an error only if an unexpected issue occurs +// - The returned storage slot is safe to modify after the call +func (r *multiReader) Storage(addr common.Address, storageRoot common.Hash, slot common.Hash) (common.Hash, error) { + var errs []error + for _, reader := range r.readers { + slot, err := reader.Storage(addr, storageRoot, slot) + if err == nil { + return slot, nil + } + errs = append(errs, err) + } + return common.Hash{}, errors.Join(errs...) +} + +// Copy implementing Reader interface, returning a deep-copied state reader. +func (r *multiReader) Copy() Reader { + var readers []Reader + for _, reader := range r.readers { + readers = append(readers, reader.Copy()) + } + return &multiReader{readers: readers} +} diff --git a/core/state/state_object.go b/core/state/state_object.go index 880b715b4b37..fa890fa10061 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "maps" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" @@ -194,45 +193,12 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { s.originStorage[key] = common.Hash{} // track the empty slot as origin value return common.Hash{} } - // If no live objects are available, attempt to use snapshots - var ( - enc []byte - err error - value common.Hash - ) - if s.db.snap != nil { - start := time.Now() - enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes())) - s.db.SnapshotStorageReads += time.Since(start) - - if len(enc) > 0 { - _, content, _, err := rlp.Split(enc) - if err != nil { - s.db.setError(err) - } - value.SetBytes(content) - } - } - // If the snapshot is unavailable or reading from it fails, load from the database. - if s.db.snap == nil || err != nil { - start := time.Now() - tr, err := s.getTrie() - if err != nil { - s.db.setError(err) - return common.Hash{} - } - val, err := tr.GetStorage(s.address, key.Bytes()) - s.db.StorageReads += time.Since(start) - - if err != nil { - s.db.setError(err) - return common.Hash{} - } - value.SetBytes(val) + value, err := s.db.reader.Storage(s.address, s.data.Root, key) + if err != nil { + s.db.setError(err) + return common.Hash{} } - // Independent of where we loaded the data from, add it to the prefetcher. - // Whilst this would be a bit weird if snapshots are disabled, but we still - // want the trie nodes to end up in the prefetcher too, so just push through. + // Schedule the resolved storage slots for prefetching if it's enabled. if s.db.prefetcher != nil && s.data.Root != types.EmptyRootHash { if err = s.db.prefetcher.prefetch(s.addrHash, s.origin.Root, s.address, [][]byte{key[:]}, true); err != nil { log.Error("Failed to prefetch storage slot", "addr", s.address, "key", key, "err", err) @@ -541,7 +507,7 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { newContract: s.newContract, } if s.trie != nil { - obj.trie = db.db.CopyTrie(s.trie) + obj.trie = mustCopyTrie(s.trie) } return obj } diff --git a/core/state/state_test.go b/core/state/state_test.go index 9200e4abe9f9..9e6c0ca5c503 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -38,14 +38,15 @@ type stateEnv struct { func newStateEnv() *stateEnv { db := rawdb.NewMemoryDatabase() - sdb, _ := New(types.EmptyRootHash, NewDatabase(db), nil) + sdb, _ := New(types.EmptyRootHash, NewDatabaseForTesting(db)) return &stateEnv{db: db, state: sdb} } func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - tdb := NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) - sdb, _ := New(types.EmptyRootHash, tdb, nil) + triedb := triedb.NewDatabase(db, &triedb.Config{Preimages: true}) + tdb := NewDatabase(db, triedb, nil) + sdb, _ := New(types.EmptyRootHash, tdb) s := &stateEnv{db: db, state: sdb} // generate a few entries @@ -62,7 +63,7 @@ func TestDump(t *testing.T) { root, _ := s.state.Commit(0, false) // check that DumpToCollector contains the state objects that are in trie - s.state, _ = New(root, tdb, nil) + s.state, _ = New(root, tdb) got := string(s.state.Dump(nil)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", @@ -101,8 +102,9 @@ func TestDump(t *testing.T) { func TestIterativeDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - tdb := NewDatabaseWithConfig(db, &triedb.Config{Preimages: true}) - sdb, _ := New(types.EmptyRootHash, tdb, nil) + triedb := triedb.NewDatabase(db, &triedb.Config{Preimages: true}) + tdb := NewDatabase(db, triedb, nil) + sdb, _ := New(types.EmptyRootHash, tdb) s := &stateEnv{db: db, state: sdb} // generate a few entries @@ -119,7 +121,7 @@ func TestIterativeDump(t *testing.T) { s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) root, _ := s.state.Commit(0, false) - s.state, _ = New(root, tdb, nil) + s.state, _ = New(root, tdb) b := &bytes.Buffer{} s.state.IterativeDump(nil, json.NewEncoder(b)) @@ -195,7 +197,7 @@ func TestSnapshotEmpty(t *testing.T) { } func TestCreateObjectRevert(t *testing.T) { - state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ := New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) addr := common.BytesToAddress([]byte("so0")) snap := state.Snapshot() diff --git a/core/state/statedb.go b/core/state/statedb.go index 80a53dbb1772..4e38e77db0a0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -88,10 +88,8 @@ type StateDB struct { db Database prefetcher *triePrefetcher trie Trie - hasher crypto.KeccakState logger *tracing.Hooks - snaps *snapshot.Tree // Nil if snapshot is not available - snap snapshot.Snapshot // Nil if snapshot is not available + reader Reader // originalRoot is the pre-state root, before any changes were made. // It will be updated when the Commit is called. @@ -170,16 +168,20 @@ type StateDB struct { } // New creates a new state from a given trie. -func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) { +func New(root common.Hash, db Database) (*StateDB, error) { tr, err := db.OpenTrie(root) if err != nil { return nil, err } - sdb := &StateDB{ + reader, err := db.Reader(root) + if err != nil { + return nil, err + } + return &StateDB{ db: db, trie: tr, originalRoot: root, - snaps: snaps, + reader: reader, stateObjects: make(map[common.Address]*stateObject), stateObjectsDestruct: make(map[common.Address]*stateObject), mutations: make(map[common.Address]*mutation), @@ -188,12 +190,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) journal: newJournal(), accessList: newAccessList(), transientStorage: newTransientStorage(), - hasher: crypto.NewKeccakState(), - } - if sdb.snaps != nil { - sdb.snap = sdb.snaps.Snapshot(root) - } - return sdb, nil + }, nil } // SetLogger sets the logger for account update hooks. @@ -206,30 +203,23 @@ func (s *StateDB) SetLogger(l *tracing.Hooks) { // commit phase, most of the needed data is already hot. func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) { // Terminate any previously running prefetcher - if s.prefetcher != nil { - s.prefetcher.terminate(false) - s.prefetcher.report() - s.prefetcher = nil - } + s.StopPrefetcher() + // Enable witness collection if requested s.witness = witness - // If snapshots are enabled, start prefethers explicitly - if s.snap != nil { - s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace, witness == nil) - - // With the switch to the Proof-of-Stake consensus algorithm, block production - // rewards are now handled at the consensus layer. Consequently, a block may - // have no state transitions if it contains no transactions and no withdrawals. - // In such cases, the account trie won't be scheduled for prefetching, leading - // to unnecessary error logs. - // - // To prevent this, the account trie is always scheduled for prefetching once - // the prefetcher is constructed. For more details, see: - // https://github.com/ethereum/go-ethereum/issues/29880 - if err := s.prefetcher.prefetch(common.Hash{}, s.originalRoot, common.Address{}, nil, false); err != nil { - log.Error("Failed to prefetch account trie", "root", s.originalRoot, "err", err) - } + // With the switch to the Proof-of-Stake consensus algorithm, block production + // rewards are now handled at the consensus layer. Consequently, a block may + // have no state transitions if it contains no transactions and no withdrawals. + // In such cases, the account trie won't be scheduled for prefetching, leading + // to unnecessary error logs. + // + // To prevent this, the account trie is always scheduled for prefetching once + // the prefetcher is constructed. For more details, see: + // https://github.com/ethereum/go-ethereum/issues/29880 + s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace, witness == nil) + if err := s.prefetcher.prefetch(common.Hash{}, s.originalRoot, common.Address{}, nil, false); err != nil { + log.Error("Failed to prefetch account trie", "root", s.originalRoot, "err", err) } } @@ -593,55 +583,22 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject { if _, ok := s.stateObjectsDestruct[addr]; ok { return nil } - // If no live objects are available, attempt to use snapshots - var data *types.StateAccount - if s.snap != nil { - start := time.Now() - acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes())) - s.SnapshotAccountReads += time.Since(start) - if err == nil { - if acc == nil { - return nil - } - data = &types.StateAccount{ - Nonce: acc.Nonce, - Balance: acc.Balance, - CodeHash: acc.CodeHash, - Root: common.BytesToHash(acc.Root), - } - if len(data.CodeHash) == 0 { - data.CodeHash = types.EmptyCodeHash.Bytes() - } - if data.Root == (common.Hash{}) { - data.Root = types.EmptyRootHash - } - } + acct, err := s.reader.Account(addr) + if err != nil { + s.setError(fmt.Errorf("getStateObject (%x) error: %w", addr.Bytes(), err)) + return nil } - // If snapshot unavailable or reading from it failed, load from the database - if data == nil { - start := time.Now() - var err error - data, err = s.trie.GetAccount(addr) - s.AccountReads += time.Since(start) - - if err != nil { - s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) - return nil - } - if data == nil { - return nil - } + if acct == nil { + return nil } - // Independent of where we loaded the data from, add it to the prefetcher. - // Whilst this would be a bit weird if snapshots are disabled, but we still - // want the trie nodes to end up in the prefetcher too, so just push through. + // Schedule the resolved account for prefetching if it's enabled. if s.prefetcher != nil { if err := s.prefetcher.prefetch(common.Hash{}, s.originalRoot, common.Address{}, [][]byte{addr[:]}, true); err != nil { log.Error("Failed to prefetch account", "addr", addr, "err", err) } } // Insert into the live set - obj := newObject(s, addr, data) + obj := newObject(s, addr, acct) s.setStateObject(obj) return obj } @@ -695,8 +652,8 @@ func (s *StateDB) Copy() *StateDB { // Copy all the basic fields, initialize the memory ones state := &StateDB{ db: s.db, - trie: s.db.CopyTrie(s.trie), - hasher: crypto.NewKeccakState(), + trie: mustCopyTrie(s.trie), + reader: s.reader.Copy(), originalRoot: s.originalRoot, stateObjects: make(map[common.Address]*stateObject, len(s.stateObjects)), stateObjectsDestruct: make(map[common.Address]*stateObject, len(s.stateObjectsDestruct)), @@ -708,16 +665,18 @@ func (s *StateDB) Copy() *StateDB { logs: make(map[common.Hash][]*types.Log, len(s.logs)), logSize: s.logSize, preimages: maps.Clone(s.preimages), - journal: s.journal.copy(), - validRevisions: slices.Clone(s.validRevisions), - nextRevisionId: s.nextRevisionId, - // In order for the block producer to be able to use and make additions - // to the snapshot tree, we need to copy that as well. Otherwise, any - // block mined by ourselves will cause gaps in the tree, and force the - // miner to operate trie-backed only. - snaps: s.snaps, - snap: s.snap, + // Do we need to copy the access list and transient storage? + // In practice: No. At the start of a transaction, these two lists are empty. + // In practice, we only ever copy state _between_ transactions/blocks, never + // in the middle of a transaction. However, it doesn't cost us much to copy + // empty lists, so we do it anyway to not blow up if we ever decide copy them + // in the middle of a transaction. + accessList: s.accessList.Copy(), + transientStorage: s.transientStorage.Copy(), + journal: s.journal.copy(), + validRevisions: slices.Clone(s.validRevisions), + nextRevisionId: s.nextRevisionId, } if s.witness != nil { state.witness = s.witness.Copy() @@ -743,14 +702,6 @@ func (s *StateDB) Copy() *StateDB { } state.logs[hash] = cpy } - // Do we need to copy the access list and transient storage? - // In practice: No. At the start of a transaction, these two lists are empty. - // In practice, we only ever copy state _between_ transactions/blocks, never - // in the middle of a transaction. However, it doesn't cost us much to copy - // empty lists, so we do it anyway to not blow up if we ever decide copy them - // in the middle of a transaction. - state.accessList = s.accessList.Copy() - state.transientStorage = s.transientStorage.Copy() return state } @@ -1000,8 +951,8 @@ func (s *StateDB) clearJournalAndRefund() { // of a specific account. It leverages the associated state snapshot for fast // storage iteration and constructs trie node deletion markers by creating // stack trie with iterated slots. -func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { - iter, err := s.snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) +func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { + iter, err := snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{}) if err != nil { return nil, nil, err } @@ -1079,10 +1030,11 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root // The fast approach can be failed if the snapshot is not fully // generated, or it's internally corrupted. Fallback to the slow // one just in case. - if s.snap != nil { - slots, nodes, err = s.fastDeleteStorage(addrHash, root) + snaps := s.db.Snapshot() + if snaps != nil { + slots, nodes, err = s.fastDeleteStorage(snaps, addrHash, root) } - if s.snap == nil || err != nil { + if snaps == nil || err != nil { slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root) } if err != nil { @@ -1326,18 +1278,16 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU } if !ret.empty() { // If snapshotting is enabled, update the snapshot tree with this new version - if s.snap != nil { - s.snap = nil - + if snap := s.db.Snapshot(); snap != nil { start := time.Now() - if err := s.snaps.Update(ret.root, ret.originRoot, ret.destructs, ret.accounts, ret.storages); err != nil { + if err := snap.Update(ret.root, ret.originRoot, ret.destructs, ret.accounts, ret.storages); err != nil { log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state - if err := s.snaps.Cap(ret.root, TriesInMemory); err != nil { + if err := snap.Cap(ret.root, TriesInMemory); err != nil { log.Warn("Failed to cap snapshot tree", "root", ret.root, "layers", TriesInMemory, "err", err) } s.SnapshotCommits += time.Since(start) @@ -1352,6 +1302,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU s.TrieDBCommits += time.Since(start) } } + s.reader, _ = s.db.Reader(s.originalRoot) return ret, err } @@ -1470,6 +1421,7 @@ func (s *StateDB) markUpdate(addr common.Address) { s.mutations[addr].typ = update } +// PointCache returns the point cache used by verkle tree. func (s *StateDB) PointCache() *utils.PointCache { return s.db.PointCache() } diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 40b079cd8a43..038ec503d463 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -197,7 +197,6 @@ func (test *stateTest) run() bool { } disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) - sdb = NewDatabaseWithNodeDB(disk, tdb) byzantium = rand.Intn(2) == 0 ) defer disk.Close() @@ -217,7 +216,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[len(roots)-1] } - state, err := New(root, sdb, snaps) + state, err := New(root, NewDatabase(disk, tdb, snaps)) if err != nil { panic(err) } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 2ce2b868faa9..8ce861629185 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -19,7 +19,6 @@ package state import ( "bytes" "encoding/binary" - "errors" "fmt" "maps" "math" @@ -53,8 +52,9 @@ func TestUpdateLeaks(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(db, nil) + sdb = NewDatabase(db, tdb, nil) ) - state, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(db, tdb), nil) + state, _ := New(types.EmptyRootHash, sdb) // Update it with some accounts for i := byte(0); i < 255; i++ { @@ -90,8 +90,8 @@ func TestIntermediateLeaks(t *testing.T) { finalDb := rawdb.NewMemoryDatabase() transNdb := triedb.NewDatabase(transDb, nil) finalNdb := triedb.NewDatabase(finalDb, nil) - transState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(transDb, transNdb), nil) - finalState, _ := New(types.EmptyRootHash, NewDatabaseWithNodeDB(finalDb, finalNdb), nil) + transState, _ := New(types.EmptyRootHash, NewDatabase(transDb, transNdb, nil)) + finalState, _ := New(types.EmptyRootHash, NewDatabase(finalDb, finalNdb, nil)) modify := func(state *StateDB, addr common.Address, i, tweak byte) { state.SetBalance(addr, uint256.NewInt(uint64(11*i)+uint64(tweak)), tracing.BalanceChangeUnspecified) @@ -166,7 +166,7 @@ func TestIntermediateLeaks(t *testing.T) { // https://github.com/ethereum/go-ethereum/pull/15549. func TestCopy(t *testing.T) { // Create a random state test to copy and modify "independently" - orig, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + orig, _ := New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) for i := byte(0); i < 255; i++ { obj := orig.getOrNewStateObject(common.BytesToAddress([]byte{i})) @@ -230,8 +230,8 @@ func TestCopy(t *testing.T) { // TestCopyWithDirtyJournal tests if Copy can correct create a equal copied // stateDB with dirty journal present. func TestCopyWithDirtyJournal(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - orig, _ := New(types.EmptyRootHash, db, nil) + db := NewDatabaseForTesting(rawdb.NewMemoryDatabase()) + orig, _ := New(types.EmptyRootHash, db) // Fill up the initial states for i := byte(0); i < 255; i++ { @@ -241,7 +241,7 @@ func TestCopyWithDirtyJournal(t *testing.T) { orig.updateStateObject(obj) } root, _ := orig.Commit(0, true) - orig, _ = New(root, db, nil) + orig, _ = New(root, db) // modify all in memory without finalizing for i := byte(0); i < 255; i++ { @@ -274,8 +274,8 @@ func TestCopyWithDirtyJournal(t *testing.T) { // It then proceeds to make changes to S1. Those changes are _not_ supposed // to affect S2. This test checks that the copy properly deep-copies the objectstate func TestCopyObjectState(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - orig, _ := New(types.EmptyRootHash, db, nil) + db := NewDatabaseForTesting(rawdb.NewMemoryDatabase()) + orig, _ := New(types.EmptyRootHash, db) // Fill up the initial states for i := byte(0); i < 5; i++ { @@ -521,7 +521,7 @@ func (test *snapshotTest) String() string { func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( - state, _ = New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ = New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 checkstates = make([]*StateDB, len(test.snapshots)) @@ -693,7 +693,7 @@ func TestTouchDelete(t *testing.T) { s := newStateEnv() s.state.getOrNewStateObject(common.Address{}) root, _ := s.state.Commit(0, false) - s.state, _ = New(root, s.state.db, s.state.snaps) + s.state, _ = New(root, s.state.db) snapshot := s.state.Snapshot() s.state.AddBalance(common.Address{}, new(uint256.Int), tracing.BalanceChangeUnspecified) @@ -710,7 +710,7 @@ func TestTouchDelete(t *testing.T) { // TestCopyOfCopy tests that modified objects are carried over to the copy, and the copy of the copy. // See https://github.com/ethereum/go-ethereum/pull/15225#issuecomment-380191512 func TestCopyOfCopy(t *testing.T) { - state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ := New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) addr := common.HexToAddress("aaaa") state.SetBalance(addr, uint256.NewInt(42), tracing.BalanceChangeUnspecified) @@ -727,8 +727,8 @@ func TestCopyOfCopy(t *testing.T) { // // See https://github.com/ethereum/go-ethereum/issues/20106. func TestCopyCommitCopy(t *testing.T) { - tdb := NewDatabase(rawdb.NewMemoryDatabase()) - state, _ := New(types.EmptyRootHash, tdb, nil) + tdb := NewDatabaseForTesting(rawdb.NewMemoryDatabase()) + state, _ := New(types.EmptyRootHash, tdb) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -781,7 +781,7 @@ func TestCopyCommitCopy(t *testing.T) { } // Commit state, ensure states can be loaded from disk root, _ := state.Commit(0, false) - state, _ = New(root, tdb, nil) + state, _ = New(root, tdb) if balance := state.GetBalance(addr); balance.Cmp(uint256.NewInt(42)) != 0 { t.Fatalf("state post-commit balance mismatch: have %v, want %v", balance, 42) } @@ -801,7 +801,7 @@ func TestCopyCommitCopy(t *testing.T) { // // See https://github.com/ethereum/go-ethereum/issues/20106. func TestCopyCopyCommitCopy(t *testing.T) { - state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ := New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -870,8 +870,8 @@ func TestCopyCopyCommitCopy(t *testing.T) { // TestCommitCopy tests the copy from a committed state is not fully functional. func TestCommitCopy(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - state, _ := New(types.EmptyRootHash, db, nil) + db := NewDatabaseForTesting(rawdb.NewMemoryDatabase()) + state, _ := New(types.EmptyRootHash, db) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -896,7 +896,7 @@ func TestCommitCopy(t *testing.T) { } root, _ := state.Commit(0, true) - state, _ = New(root, db, nil) + state, _ = New(root, db) state.SetState(addr, skey2, sval2) state.Commit(1, true) @@ -909,10 +909,10 @@ func TestCommitCopy(t *testing.T) { t.Fatalf("unexpected code: have %x", code) } // Miss slots because of non-functional trie after commit - if val := copied.GetState(addr, skey1); val != (common.Hash{}) { - t.Fatalf("unexpected storage slot: have %x", sval1) + if val := copied.GetState(addr, skey1); val != sval1 { + t.Fatalf("unexpected storage slot: have %x", val) } - if val := copied.GetCommittedState(addr, skey1); val != (common.Hash{}) { + if val := copied.GetCommittedState(addr, skey1); val != sval1 { t.Fatalf("unexpected storage slot: have %x", val) } // Slots cached in the stateDB, available after commit @@ -922,9 +922,6 @@ func TestCommitCopy(t *testing.T) { if val := copied.GetCommittedState(addr, skey2); val != sval2 { t.Fatalf("unexpected storage slot: have %x", val) } - if !errors.Is(copied.Error(), trie.ErrCommitted) { - t.Fatalf("unexpected state error, %v", copied.Error()) - } } // TestDeleteCreateRevert tests a weird state transition corner case that we hit @@ -937,13 +934,13 @@ func TestCommitCopy(t *testing.T) { // first, but the journal wiped the entire state object on create-revert. func TestDeleteCreateRevert(t *testing.T) { // Create an initial state with a single contract - state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ := New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) addr := common.BytesToAddress([]byte("so")) state.SetBalance(addr, uint256.NewInt(1), tracing.BalanceChangeUnspecified) root, _ := state.Commit(0, false) - state, _ = New(root, state.db, state.snaps) + state, _ = New(root, state.db) // Simulate self-destructing in one transaction, then create-reverting in another state.SelfDestruct(addr) @@ -955,7 +952,7 @@ func TestDeleteCreateRevert(t *testing.T) { // Commit the entire state and make sure we don't crash and have the correct state root, _ = state.Commit(0, true) - state, _ = New(root, state.db, state.snaps) + state, _ = New(root, state.db) if state.getStateObject(addr) != nil { t.Fatalf("self-destructed contract came alive") @@ -986,10 +983,10 @@ func testMissingTrieNodes(t *testing.T, scheme string) { CleanCacheSize: 0, }}) // disable caching } - db := NewDatabaseWithNodeDB(memDb, tdb) + db := NewDatabase(memDb, tdb, nil) var root common.Hash - state, _ := New(types.EmptyRootHash, db, nil) + state, _ := New(types.EmptyRootHash, db) addr := common.BytesToAddress([]byte("so")) { state.SetBalance(addr, uint256.NewInt(1), tracing.BalanceChangeUnspecified) @@ -1003,7 +1000,7 @@ func testMissingTrieNodes(t *testing.T, scheme string) { tdb.Commit(root, false) } // Create a new state on the old root - state, _ = New(root, db, nil) + state, _ = New(root, db) // Now we clear out the memdb it := memDb.NewIterator(nil, nil) for it.Next() { @@ -1037,8 +1034,8 @@ func TestStateDBAccessList(t *testing.T) { } memDb := rawdb.NewMemoryDatabase() - db := NewDatabase(memDb) - state, _ := New(types.EmptyRootHash, db, nil) + db := NewDatabaseForTesting(memDb) + state, _ := New(types.EmptyRootHash, db) state.accessList = newAccessList() verifyAddrs := func(astrings ...string) { @@ -1207,9 +1204,9 @@ func TestFlushOrderDataLoss(t *testing.T) { // Create a state trie with many accounts and slots var ( memdb = rawdb.NewMemoryDatabase() - triedb = triedb.NewDatabase(memdb, nil) - statedb = NewDatabaseWithNodeDB(memdb, triedb) - state, _ = New(types.EmptyRootHash, statedb, nil) + triedb = triedb.NewDatabase(memdb, triedb.HashDefaults) + statedb = NewDatabase(memdb, triedb, nil) + state, _ = New(types.EmptyRootHash, statedb) ) for a := byte(0); a < 10; a++ { state.CreateAccount(common.Address{a}) @@ -1229,7 +1226,7 @@ func TestFlushOrderDataLoss(t *testing.T) { t.Fatalf("failed to commit state trie: %v", err) } // Reopen the state trie from flushed disk and verify it - state, err = New(root, NewDatabase(memdb), nil) + state, err = New(root, NewDatabaseForTesting(memdb)) if err != nil { t.Fatalf("failed to reopen state trie: %v", err) } @@ -1244,8 +1241,8 @@ func TestFlushOrderDataLoss(t *testing.T) { func TestStateDBTransientStorage(t *testing.T) { memDb := rawdb.NewMemoryDatabase() - db := NewDatabase(memDb) - state, _ := New(types.EmptyRootHash, db, nil) + db := NewDatabaseForTesting(memDb) + state, _ := New(types.EmptyRootHash, db) key := common.Hash{0x01} value := common.Hash{0x02} @@ -1280,9 +1277,9 @@ func TestDeleteStorage(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, nil) - db = NewDatabaseWithNodeDB(disk, tdb) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) - state, _ = New(types.EmptyRootHash, db, snaps) + db = NewDatabase(disk, tdb, snaps) + state, _ = New(types.EmptyRootHash, db) addr = common.HexToAddress("0x1") ) // Initialize account and populate storage @@ -1294,9 +1291,10 @@ func TestDeleteStorage(t *testing.T) { state.SetState(addr, slot, value) } root, _ := state.Commit(0, true) + // Init phase done, create two states, one with snap and one without - fastState, _ := New(root, db, snaps) - slowState, _ := New(root, db, nil) + fastState, _ := New(root, NewDatabase(disk, tdb, snaps)) + slowState, _ := New(root, NewDatabase(disk, tdb, nil)) obj := fastState.getOrNewStateObject(addr) storageRoot := obj.data.Root @@ -1334,8 +1332,8 @@ func TestStorageDirtiness(t *testing.T) { var ( disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, nil) - db = NewDatabaseWithNodeDB(disk, tdb) - state, _ = New(types.EmptyRootHash, db, nil) + db = NewDatabase(disk, tdb, nil) + state, _ = New(types.EmptyRootHash, db) addr = common.HexToAddress("0x1") checkDirty = func(key common.Hash, value common.Hash, dirty bool) { obj := state.getStateObject(addr) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index b7039c9e1cb7..1d5220a80223 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -53,8 +53,8 @@ func makeTestState(scheme string) (ethdb.Database, Database, *triedb.Database, c } db := rawdb.NewMemoryDatabase() nodeDb := triedb.NewDatabase(db, config) - sdb := NewDatabaseWithNodeDB(db, nodeDb) - state, _ := New(types.EmptyRootHash, sdb, nil) + sdb := NewDatabase(db, nodeDb, nil) + state, _ := New(types.EmptyRootHash, sdb) // Fill it with some arbitrary data var accounts []*testAccount @@ -94,7 +94,7 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, scheme string, root com config.PathDB = pathdb.Defaults } // Check root availability and state contents - state, err := New(root, NewDatabaseWithConfig(db, &config), nil) + state, err := New(root, NewDatabase(db, triedb.NewDatabase(db, &config), nil)) if err != nil { t.Fatalf("failed to create state trie at %x: %v", root, err) } @@ -120,7 +120,7 @@ func checkStateConsistency(db ethdb.Database, scheme string, root common.Hash) e if scheme == rawdb.PathScheme { config.PathDB = pathdb.Defaults } - state, err := New(root, NewDatabaseWithConfig(db, config), nil) + state, err := New(root, NewDatabase(db, triedb.NewDatabase(db, config), nil)) if err != nil { return err } diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index 8f01acd2214d..e973cdec1c1a 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -28,7 +28,7 @@ import ( ) func filledStateDB() *StateDB { - state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ := New(types.EmptyRootHash, NewDatabaseForTesting(rawdb.NewMemoryDatabase())) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") diff --git a/core/stateless.go b/core/stateless.go index 4c7e6f31027f..5519db908641 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -43,7 +43,7 @@ func ExecuteStateless(config *params.ChainConfig, witness *stateless.Witness) (c // Create and populate the state database to serve as the stateless backend memdb := witness.MakeHashDB() - db, err := state.New(witness.Root(), state.NewDatabaseWithConfig(memdb, triedb.HashDefaults), nil) + db, err := state.New(witness.Root(), state.NewDatabase(memdb, triedb.NewDatabase(memdb, triedb.HashDefaults), nil)) if err != nil { return common.Hash{}, common.Hash{}, err } diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index d658a6daf44a..55380b432fd4 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -545,7 +545,7 @@ func TestOpenDrops(t *testing.T) { store.Close() // Create a blob pool out of the pre-seeded data - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewDatabase(memorydb.New()))) statedb.AddBalance(crypto.PubkeyToAddress(gapper.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) statedb.AddBalance(crypto.PubkeyToAddress(dangler.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) statedb.AddBalance(crypto.PubkeyToAddress(filler.PublicKey), uint256.NewInt(1000000), tracing.BalanceChangeUnspecified) @@ -676,7 +676,7 @@ func TestOpenIndex(t *testing.T) { store.Close() // Create a blob pool out of the pre-seeded data - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewDatabase(memorydb.New()))) statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.Commit(0, true) @@ -776,7 +776,7 @@ func TestOpenHeap(t *testing.T) { store.Close() // Create a blob pool out of the pre-seeded data - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewDatabase(memorydb.New()))) statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) @@ -856,7 +856,7 @@ func TestOpenCap(t *testing.T) { // with a high cap to ensure everything was persisted previously for _, datacap := range []uint64{2 * (txAvgSize + blobSize), 100 * (txAvgSize + blobSize)} { // Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewDatabase(memorydb.New()))) statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) @@ -1266,7 +1266,7 @@ func TestAdd(t *testing.T) { keys = make(map[string]*ecdsa.PrivateKey) addrs = make(map[string]common.Address) ) - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewDatabase(memorydb.New()))) for acc, seed := range tt.seeds { // Generate a new random key/address for the seed account keys[acc], _ = crypto.GenerateKey() @@ -1328,7 +1328,7 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { basefee = uint64(1050) blobfee = uint64(105) signer = types.LatestSigner(testChainConfig) - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewDatabase(memorydb.New()))) chain = &testBlockChain{ config: testChainConfig, basefee: uint256.NewInt(basefee), diff --git a/core/txpool/legacypool/legacypool2_test.go b/core/txpool/legacypool/legacypool2_test.go index fd961d1d925c..cf7f3075469c 100644 --- a/core/txpool/legacypool/legacypool2_test.go +++ b/core/txpool/legacypool/legacypool2_test.go @@ -80,7 +80,7 @@ func TestTransactionFutureAttack(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalQueue = 100 @@ -117,7 +117,7 @@ func TestTransactionFutureAttack(t *testing.T) { func TestTransactionFuture1559(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) @@ -150,7 +150,7 @@ func TestTransactionFuture1559(t *testing.T) { func TestTransactionZAttack(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) @@ -218,7 +218,7 @@ func TestTransactionZAttack(t *testing.T) { func BenchmarkFutureAttack(b *testing.B) { // Create the pool to test the limit enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalQueue = 100 diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index c86991c942da..4c413db1164e 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -160,7 +160,7 @@ func setupPool() (*LegacyPool, *ecdsa.PrivateKey) { } func setupPoolWithConfig(config *params.ChainConfig) (*LegacyPool, *ecdsa.PrivateKey) { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(config, 10000000, statedb, new(event.Feed)) key, _ := crypto.GenerateKey() @@ -251,7 +251,7 @@ func (c *testChain) State() (*state.StateDB, error) { // a state change between those fetches. stdb := c.statedb if *c.trigger { - c.statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + c.statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) // simulate that the new head block included tx0 and tx1 c.statedb.SetNonce(c.address, 2) c.statedb.SetBalance(c.address, new(uint256.Int).SetUint64(params.Ether), tracing.BalanceChangeUnspecified) @@ -269,7 +269,7 @@ func TestStateChangeDuringReset(t *testing.T) { var ( key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) trigger = false ) @@ -468,7 +468,7 @@ func TestChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) statedb.AddBalance(addr, uint256.NewInt(100000000000000), tracing.BalanceChangeUnspecified) pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) @@ -497,7 +497,7 @@ func TestDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) statedb.AddBalance(addr, uint256.NewInt(100000000000000), tracing.BalanceChangeUnspecified) pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) @@ -697,7 +697,7 @@ func TestPostponing(t *testing.T) { t.Parallel() // Create the pool to test the postponing with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) @@ -910,7 +910,7 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -1003,7 +1003,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { evictionInterval = time.Millisecond * 100 // Create the pool to test the non-expiration enforcement - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -1189,7 +1189,7 @@ func TestPendingGlobalLimiting(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -1291,7 +1291,7 @@ func TestCapClearsFromAll(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -1326,7 +1326,7 @@ func TestPendingMinimumAllowance(t *testing.T) { t.Parallel() // Create the pool to test the limit enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -1375,7 +1375,7 @@ func TestRepricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) @@ -1495,7 +1495,7 @@ func TestMinGasPriceEnforced(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(eip1559Config, 10000000, statedb, new(event.Feed)) txPoolConfig := DefaultConfig @@ -1668,7 +1668,7 @@ func TestRepricingKeepsLocals(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) @@ -1742,7 +1742,7 @@ func TestUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -1857,7 +1857,7 @@ func TestStableUnderpricing(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -2090,7 +2090,7 @@ func TestDeduplication(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) @@ -2157,7 +2157,7 @@ func TestReplacement(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) @@ -2363,7 +2363,7 @@ func testJournaling(t *testing.T, nolocals bool) { os.Remove(journal) // Create the original pool to inject transaction into the journal - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig @@ -2464,7 +2464,7 @@ func TestStatusCheck(t *testing.T) { t.Parallel() // Create the pool to test the status retrievals with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 02fc94840d60..4e3ccf6992b6 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -86,7 +86,7 @@ func TestEIP2200(t *testing.T) { for i, tt := range eip2200Tests { address := common.BytesToAddress([]byte("contract")) - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) statedb.CreateAccount(address) statedb.SetCode(address, hexutil.MustDecode(tt.input)) statedb.SetState(address, common.Hash{}, common.BytesToHash([]byte{tt.original})) @@ -138,7 +138,7 @@ func TestCreateGas(t *testing.T) { var gasUsed = uint64(0) doCheck := func(testGas int) bool { address := common.BytesToAddress([]byte("contract")) - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) statedb.CreateAccount(address) statedb.SetCode(address, hexutil.MustDecode(tt.code)) statedb.Finalise(true) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index e17e913aa3ce..ccb41df232ee 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -582,7 +582,7 @@ func BenchmarkOpMstore(bench *testing.B) { func TestOpTstore(t *testing.T) { var ( - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) env = NewEVM(BlockContext{}, TxContext{}, statedb, params.TestChainConfig, Config{}) stack = newstack() mem = NewMemory() diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index ff4977d728ed..bd6080f6d847 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -43,7 +43,7 @@ func TestLoopInterrupt(t *testing.T) { } for i, tt := range loopInterruptTests { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) statedb.CreateAccount(address) statedb.SetCode(address, common.Hex2Bytes(tt)) statedb.Finalise(true) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index 1181e5fccdc3..b9a429a093c0 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -128,7 +128,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { setDefaults(cfg) if cfg.State == nil { - cfg.State, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + cfg.State, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) } var ( address = common.BytesToAddress([]byte("contract")) @@ -165,7 +165,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { setDefaults(cfg) if cfg.State == nil { - cfg.State, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + cfg.State, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) } var ( vmenv = NewEnv(cfg) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 04abc5480eac..19158b76b254 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -104,7 +104,7 @@ func TestExecute(t *testing.T) { } func TestCall(t *testing.T) { - state, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + state, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) address := common.HexToAddress("0xaa") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, @@ -160,7 +160,7 @@ func BenchmarkCall(b *testing.B) { } func benchmarkEVM_Create(bench *testing.B, code string) { var ( - statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) sender = common.BytesToAddress([]byte("sender")) receiver = common.BytesToAddress([]byte("receiver")) ) @@ -328,7 +328,7 @@ func TestBlockhash(t *testing.T) { func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode string, b *testing.B) { cfg := new(Config) setDefaults(cfg) - cfg.State, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + cfg.State, _ = state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) cfg.GasLimit = gas if len(tracerCode) > 0 { tracer, err := tracers.DefaultDirectory.New(tracerCode, new(tracers.Context), nil) @@ -819,7 +819,7 @@ func TestRuntimeJSTracer(t *testing.T) { main := common.HexToAddress("0xaa") for i, jsTracer := range jsTracers { for j, tc := range tests { - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) statedb.SetCode(main, tc.code) statedb.SetCode(common.HexToAddress("0xbb"), calleeCode) statedb.SetCode(common.HexToAddress("0xcc"), calleeCode) @@ -861,7 +861,7 @@ func TestJSTracerCreateTx(t *testing.T) { exit: function(res) { this.exits++ }}` code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting(rawdb.NewMemoryDatabase())) tracer, err := tracers.DefaultDirectory.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 750cee5e44e8..34aa5c52bede 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -64,8 +64,9 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &triedb.Config{Preimages: true}) - sdb, _ = state.New(types.EmptyRootHash, statedb, nil) + mdb = rawdb.NewMemoryDatabase() + statedb = state.NewDatabase(mdb, triedb.NewDatabase(mdb, &triedb.Config{Preimages: true}), nil) + sdb, _ = state.New(types.EmptyRootHash, statedb) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} ) @@ -82,7 +83,7 @@ func TestAccountRange(t *testing.T) { } } root, _ := sdb.Commit(0, true) - sdb, _ = state.New(root, statedb, nil) + sdb, _ = state.New(root, statedb) trie, err := statedb.OpenTrie(root) if err != nil { @@ -135,12 +136,12 @@ func TestEmptyAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) - st, _ = state.New(types.EmptyRootHash, statedb, nil) + statedb = state.NewDatabaseForTesting(rawdb.NewMemoryDatabase()) + st, _ = state.New(types.EmptyRootHash, statedb) ) // Commit(although nothing to flush) and re-init the statedb st.Commit(0, true) - st, _ = state.New(types.EmptyRootHash, statedb, nil) + st, _ = state.New(types.EmptyRootHash, statedb) results := st.RawDump(&state.DumpConfig{ SkipCode: true, @@ -161,8 +162,10 @@ func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( - db = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &triedb.Config{Preimages: true}) - sdb, _ = state.New(types.EmptyRootHash, db, nil) + mdb = rawdb.NewMemoryDatabase() + tdb = triedb.NewDatabase(mdb, &triedb.Config{Preimages: true}) + db = state.NewDatabase(mdb, tdb, nil) + sdb, _ = state.New(types.EmptyRootHash, db) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), @@ -181,7 +184,7 @@ func TestStorageRangeAt(t *testing.T) { sdb.SetState(addr, *entry.Key, entry.Value) } root, _ := sdb.Commit(0, false) - sdb, _ = state.New(root, db, nil) + sdb, _ = state.New(root, db) // Check a few combinations of limit and start/end. tests := []struct { diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 372c76f49692..6dfb2a13e65d 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -68,8 +68,9 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u // the internal junks created by tracing will be persisted into the disk. // TODO(rjl493456442), clean cache is disabled to prevent memory leak, // please re-enable it for better performance. - database = state.NewDatabaseWithConfig(eth.chainDb, triedb.HashDefaults) - if statedb, err = state.New(block.Root(), database, nil); err == nil { + tdb := triedb.NewDatabase(eth.chainDb, triedb.HashDefaults) + database = state.NewDatabase(eth.chainDb, tdb, nil) + if statedb, err = state.New(block.Root(), database); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) return statedb, noopReleaser, nil } @@ -86,13 +87,13 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u // TODO(rjl493456442), clean cache is disabled to prevent memory leak, // please re-enable it for better performance. tdb = triedb.NewDatabase(eth.chainDb, triedb.HashDefaults) - database = state.NewDatabaseWithNodeDB(eth.chainDb, tdb) + database = state.NewDatabase(eth.chainDb, tdb, nil) // If we didn't check the live database, do check state over ephemeral database, // otherwise we would rewind past a persisted block (specific corner case is // chain tracing from the genesis). if !readOnly { - statedb, err = state.New(current.Root(), database, nil) + statedb, err = state.New(current.Root(), database) if err == nil { return statedb, noopReleaser, nil } @@ -111,7 +112,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u } current = parent - statedb, err = state.New(current.Root(), database, nil) + statedb, err = state.New(current.Root(), database) if err == nil { break } @@ -156,7 +157,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u return nil, nil, fmt.Errorf("stateAtBlock commit failed, number %d root %v: %w", current.NumberU64(), current.Root().Hex(), err) } - statedb, err = state.New(root, database, nil) + statedb, err = state.New(root, database) if err != nil { return nil, nil, fmt.Errorf("state reset after block %d failed: %v", current.NumberU64(), err) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index e867f572847a..7465fb55295f 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -979,7 +979,7 @@ func TestCall(t *testing.T) { }, overrides: StateOverride{ dad: OverrideAccount{ - State: &map[common.Hash]common.Hash{}, + State: map[common.Hash]common.Hash{}, }, }, want: "0x0000000000000000000000000000000000000000000000000000000000000000", diff --git a/miner/miner_test.go b/miner/miner_test.go index da133ad8d0b6..07e013d203b7 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -156,7 +156,7 @@ func createMiner(t *testing.T) *Miner { if err != nil { t.Fatalf("can't create new chain %v", err) } - statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache(), nil) + statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache()) blockchain := &testBlockChain{bc.Genesis().Root(), chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 416bab947264..a7e4f24f3246 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -222,7 +222,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if logs := rlpHash(st.StateDB.Logs()); logs != common.Hash(post.Logs) { return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - st.StateDB, _ = state.New(root, st.StateDB.Database(), st.Snapshots) + st.StateDB, _ = state.New(root, st.StateDB.Database()) return nil } @@ -460,8 +460,8 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bo tconf.PathDB = pathdb.Defaults } triedb := triedb.NewDatabase(db, tconf) - sdb := state.NewDatabaseWithNodeDB(db, triedb) - statedb, _ := state.New(types.EmptyRootHash, sdb, nil) + sdb := state.NewDatabase(db, triedb, nil) + statedb, _ := state.New(types.EmptyRootHash, sdb) for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) @@ -484,7 +484,8 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bo } snaps, _ = snapshot.New(snapconfig, db, triedb, root) } - statedb, _ = state.New(root, sdb, snaps) + sdb = state.NewDatabase(db, triedb, snaps) + statedb, _ = state.New(root, sdb) return StateTestState{statedb, triedb, snaps} } From bd06430b7d85cd90c83f706dd13b5e2b0f73ecca Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 18 Jul 2024 09:48:49 +0800 Subject: [PATCH 02/10] core: address comments from the review meeting --- core/blockchain.go | 6 ++--- core/state/reader.go | 46 +++++++++++++++++++++++++++++--------- core/state/state_object.go | 2 +- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 23efc04cad61..40ea78e01e7c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -449,10 +449,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) - // Register the snapshot into statedb. It's an ugly hack as state snapshot - // is constructed after stateDb. TODO(rjl493456442) improve the construction - // logic. - bc.stateDb.SetSnapshot(bc.snaps) + // Re-initialize the state database with snapshot + bc.stateDb = state.NewDatabase(bc.db, bc.triedb, bc.snaps) } // Rewind the chain in case of an incompatible config upgrade. diff --git a/core/state/reader.go b/core/state/reader.go index 7d50c0ea1b1f..8efe72807f06 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -18,6 +18,7 @@ package state import ( "errors" + "maps" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -45,7 +46,7 @@ type Reader interface { // - Returns an empty slot if it does not exist // - Returns an error only if an unexpected issue occurs // - The returned storage slot is safe to modify after the call - Storage(addr common.Address, storageRoot common.Hash, slot common.Hash) (common.Hash, error) + Storage(addr common.Address, slot common.Hash) (common.Hash, error) // Copy returns a deep-copied state reader. Copy() Reader @@ -106,7 +107,7 @@ func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) // the requested storage slot is not yet covered by the snapshot. // // The returned storage slot might be empty if it's not existent. -func (r *stateReader) Storage(addr common.Address, root common.Hash, key common.Hash) (common.Hash, error) { +func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { addrHash := crypto.HashData(r.buff, addr.Bytes()) slotHash := crypto.HashData(r.buff, key.Bytes()) ret, err := r.snap.Storage(addrHash, slotHash) @@ -136,11 +137,12 @@ func (r *stateReader) Copy() Reader { // trieReader implements the Reader interface, providing functions to access // state from the referenced trie. type trieReader struct { - root common.Hash // State root which uniquely represent a state. - db *triedb.Database // Database for loading trie - buff crypto.KeccakState // Buffer for keccak256 hashing. - mainTrie Trie // Main trie, resolved in constructor - subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved. + root common.Hash // State root which uniquely represent a state + db *triedb.Database // Database for loading trie + buff crypto.KeccakState // Buffer for keccak256 hashing + mainTrie Trie // Main trie, resolved in constructor + subRoots map[common.Address]common.Hash // Set of storage roots, cached when the account is resolved + subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved } // trieReader constructs a trie reader of the specific state. An error will be @@ -163,6 +165,7 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach db: db, buff: crypto.NewKeccakState(), mainTrie: tr, + subRoots: make(map[common.Address]common.Hash), subTries: make(map[common.Address]Trie), }, nil } @@ -172,7 +175,16 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach // An error will be returned if the trie state is corrupted. An nil account // will be returned if it's not existent in the trie. func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { - return r.mainTrie.GetAccount(addr) + account, err := r.mainTrie.GetAccount(addr) + if err != nil { + return nil, err + } + if account == nil { + r.subRoots[addr] = types.EmptyRootHash + } else { + r.subRoots[addr] = account.Root + } + return account, nil } // Storage implements Reader, retrieving the storage slot specified by the @@ -180,7 +192,7 @@ func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { // // An error will be returned if the trie state is corrupted. An empty storage // slot will be returned if it's not existent in the trie. -func (r *trieReader) Storage(addr common.Address, root common.Hash, key common.Hash) (common.Hash, error) { +func (r *trieReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { var ( tr Trie found bool @@ -191,6 +203,17 @@ func (r *trieReader) Storage(addr common.Address, root common.Hash, key common.H } else { tr, found = r.subTries[addr] if !found { + root, ok := r.subRoots[addr] + + // The storage slot is accessed without account caching. It's unexpected + // behavior but try to resolve the account first anyway. + if !ok { + _, err := r.Account(addr) + if err != nil { + return common.Hash{}, err + } + root = r.subRoots[addr] + } var err error tr, err = trie.NewStateTrie(trie.StorageTrieID(r.root, crypto.HashData(r.buff, addr.Bytes()), root), r.db) if err != nil { @@ -218,6 +241,7 @@ func (r *trieReader) Copy() Reader { db: r.db, buff: crypto.NewKeccakState(), mainTrie: mustCopyTrie(r.mainTrie), + subRoots: maps.Clone(r.subRoots), subTries: tries, } } @@ -265,10 +289,10 @@ func (r *multiReader) Account(addr common.Address) (*types.StateAccount, error) // - Returns an empty slot if it does not exist // - Returns an error only if an unexpected issue occurs // - The returned storage slot is safe to modify after the call -func (r *multiReader) Storage(addr common.Address, storageRoot common.Hash, slot common.Hash) (common.Hash, error) { +func (r *multiReader) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { var errs []error for _, reader := range r.readers { - slot, err := reader.Storage(addr, storageRoot, slot) + slot, err := reader.Storage(addr, slot) if err == nil { return slot, nil } diff --git a/core/state/state_object.go b/core/state/state_object.go index fa890fa10061..7ddee9c28dcb 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -193,7 +193,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { s.originStorage[key] = common.Hash{} // track the empty slot as origin value return common.Hash{} } - value, err := s.db.reader.Storage(s.address, s.data.Root, key) + value, err := s.db.reader.Storage(s.address, key) if err != nil { s.db.setError(err) return common.Hash{} From deaf017466d19f2daa8b9ed86826b8472496f70b Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 18 Jul 2024 10:21:11 +0800 Subject: [PATCH 03/10] core: fix metrics --- core/blockchain.go | 30 ++++++++------------ core/state/metrics.go | 5 ++++ core/state/reader.go | 64 +++++++++++++++++++++++++++++++++++++++++++ core/state/statedb.go | 26 ++++++++++-------- 4 files changed, 96 insertions(+), 29 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 40ea78e01e7c..29ad6aada757 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -72,11 +72,8 @@ var ( storageUpdateTimer = metrics.NewRegisteredResettingTimer("chain/storage/updates", nil) storageCommitTimer = metrics.NewRegisteredResettingTimer("chain/storage/commits", nil) - snapshotAccountReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/account/reads", nil) - snapshotStorageReadTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/storage/reads", nil) - snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) - - triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) + snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) blockInsertTimer = metrics.NewRegisteredResettingTimer("chain/inserts", nil) blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil) @@ -1951,19 +1948,16 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s proctime := time.Since(start) // processing + validation // Update the metrics touched during block processing and validation - accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) - storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) - snapshotAccountReadTimer.Update(statedb.SnapshotAccountReads) // Account reads are complete(in processing) - snapshotStorageReadTimer.Update(statedb.SnapshotStorageReads) // Storage reads are complete(in processing) - accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) - storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) - accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) - triehash := statedb.AccountHashes // The time spent on tries hashing - trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates // The time spent on tries update - trieRead := statedb.SnapshotAccountReads + statedb.AccountReads // The time spent on account read - trieRead += statedb.SnapshotStorageReads + statedb.StorageReads // The time spent on storage read - blockExecutionTimer.Update(ptime - trieRead) // The time spent on EVM processing - blockValidationTimer.Update(vtime - (triehash + trieUpdate)) // The time spent on block validation + accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) + storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) + accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) + storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) + accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) + triehash := statedb.AccountHashes // The time spent on tries hashing + trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates // The time spent on tries update + trieRead := statedb.AccountReads + statedb.StorageReads // The time spent on account read and storage read + blockExecutionTimer.Update(ptime - trieRead) // The time spent on EVM processing + blockValidationTimer.Update(vtime - (triehash + trieUpdate)) // The time spent on block validation // Write the block to the chain and get the status. var ( diff --git a/core/state/metrics.go b/core/state/metrics.go index e702ef3a81a6..9e90ff1b0026 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -27,4 +27,9 @@ var ( storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) + + trieAccountReadTimer = metrics.NewRegisteredResettingTimer("state/trie/account/reads", nil) + trieStorageReadTimer = metrics.NewRegisteredResettingTimer("state/trie/storage/reads", nil) + snapshotAccountReadTimer = metrics.NewRegisteredResettingTimer("state/snapshot/account/reads", nil) + snapshotStorageReadTimer = metrics.NewRegisteredResettingTimer("state/snapshot/storage/reads", nil) ) diff --git a/core/state/reader.go b/core/state/reader.go index 8efe72807f06..c3e5f9e8edf2 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -19,6 +19,7 @@ package state import ( "errors" "maps" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -48,6 +49,10 @@ type Reader interface { // - The returned storage slot is safe to modify after the call Storage(addr common.Address, slot common.Hash) (common.Hash, error) + // Stats returns the statistics of the reader, specifically detailing the time + // spent on account reading and storage reading. + Stats() (time.Duration, time.Duration) + // Copy returns a deep-copied state reader. Copy() Reader } @@ -57,6 +62,9 @@ type Reader interface { type stateReader struct { snap snapshot.Snapshot buff crypto.KeccakState + + accountTime time.Duration + storageTime time.Duration } // newStateReader constructs a flat state reader with on the specified state root. @@ -78,6 +86,11 @@ func newStateReader(root common.Hash, snaps *snapshot.Tree) (*stateReader, error // // The returned account might be nil if it's not existent. func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) { + defer func(start time.Time) { + r.accountTime += time.Since(start) + snapshotAccountReadTimer.UpdateSince(start) + }(time.Now()) + ret, err := r.snap.Account(crypto.HashData(r.buff, addr.Bytes())) if err != nil { return nil, err @@ -108,6 +121,11 @@ func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) // // The returned storage slot might be empty if it's not existent. func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + defer func(start time.Time) { + r.storageTime += time.Since(start) + snapshotStorageReadTimer.UpdateSince(start) + }(time.Now()) + addrHash := crypto.HashData(r.buff, addr.Bytes()) slotHash := crypto.HashData(r.buff, key.Bytes()) ret, err := r.snap.Storage(addrHash, slotHash) @@ -126,11 +144,20 @@ func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash return value, nil } +// Stats implements Reader, returning the time spent on account reading and +// storage reading from the snapshot. +func (r *stateReader) Stats() (time.Duration, time.Duration) { + return r.accountTime, r.storageTime +} + // Copy implements Reader, returning a deep-copied snap reader. func (r *stateReader) Copy() Reader { return &stateReader{ snap: r.snap, buff: crypto.NewKeccakState(), + + // statistics (accountTime and storageTime) are not copied, as they + // only belong to current reader instance. } } @@ -143,6 +170,9 @@ type trieReader struct { mainTrie Trie // Main trie, resolved in constructor subRoots map[common.Address]common.Hash // Set of storage roots, cached when the account is resolved subTries map[common.Address]Trie // Group of storage tries, cached when it's resolved + + accountTime time.Duration // Time spent on the account reading + storageTime time.Duration // Time spent on the storage reading } // trieReader constructs a trie reader of the specific state. An error will be @@ -175,6 +205,11 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach // An error will be returned if the trie state is corrupted. An nil account // will be returned if it's not existent in the trie. func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { + defer func(start time.Time) { + r.accountTime += time.Since(start) + trieAccountReadTimer.UpdateSince(start) + }(time.Now()) + account, err := r.mainTrie.GetAccount(addr) if err != nil { return nil, err @@ -193,6 +228,11 @@ func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { // An error will be returned if the trie state is corrupted. An empty storage // slot will be returned if it's not existent in the trie. func (r *trieReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { + defer func(start time.Time) { + r.storageTime += time.Since(start) + trieStorageReadTimer.UpdateSince(start) + }(time.Now()) + var ( tr Trie found bool @@ -230,6 +270,12 @@ func (r *trieReader) Storage(addr common.Address, key common.Hash) (common.Hash, return value, nil } +// Stats implements Reader, returning the time spent on account reading and +// storage reading from the trie. +func (r *trieReader) Stats() (time.Duration, time.Duration) { + return r.accountTime, r.storageTime +} + // Copy implements Reader, returning a deep-copied trie reader. func (r *trieReader) Copy() Reader { tries := make(map[common.Address]Trie) @@ -243,6 +289,9 @@ func (r *trieReader) Copy() Reader { mainTrie: mustCopyTrie(r.mainTrie), subRoots: maps.Clone(r.subRoots), subTries: tries, + + // statistics (accountTime and storageTime) are not copied, as they + // only belong to current reader instance. } } @@ -301,6 +350,21 @@ func (r *multiReader) Storage(addr common.Address, slot common.Hash) (common.Has return common.Hash{}, errors.Join(errs...) } +// Stats implements Reader, returning the time spent on account reading and +// storage reading from the reader. +func (r *multiReader) Stats() (time.Duration, time.Duration) { + var ( + accountTime time.Duration + storageTime time.Duration + ) + for _, reader := range r.readers { + aTime, sTime := reader.Stats() + accountTime += aTime + storageTime += sTime + } + return accountTime, storageTime +} + // Copy implementing Reader interface, returning a deep-copied state reader. func (r *multiReader) Copy() Reader { var readers []Reader diff --git a/core/state/statedb.go b/core/state/statedb.go index 4e38e77db0a0..85cc2b7189b8 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -149,17 +149,16 @@ type StateDB struct { witness *stateless.Witness // Measurements gathered during execution for debugging purposes - AccountReads time.Duration - AccountHashes time.Duration - AccountUpdates time.Duration - AccountCommits time.Duration - StorageReads time.Duration - StorageUpdates time.Duration - StorageCommits time.Duration - SnapshotAccountReads time.Duration - SnapshotStorageReads time.Duration - SnapshotCommits time.Duration - TrieDBCommits time.Duration + AccountReads time.Duration + AccountHashes time.Duration + AccountUpdates time.Duration + AccountCommits time.Duration + StorageReads time.Duration + StorageUpdates time.Duration + StorageCommits time.Duration + + SnapshotCommits time.Duration + TrieDBCommits time.Duration AccountUpdated int StorageUpdated atomic.Int64 @@ -1302,6 +1301,11 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU s.TrieDBCommits += time.Since(start) } } + // Submit the statistics of reader to metric system and reset it with new root + aTime, sTime := s.reader.Stats() + s.AccountReads += aTime + s.AccountReads += sTime + s.reader, _ = s.db.Reader(s.originalRoot) return ret, err } From 30b0fd842d701a99ed3a49e02e24cb4b530dd9e2 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 18 Jul 2024 11:29:18 +0800 Subject: [PATCH 04/10] core: fix metrics --- core/blockchain.go | 24 ++++++++++++------------ core/state/metrics.go | 5 ----- core/state/reader.go | 4 ---- core/state/statedb.go | 2 +- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 29ad6aada757..2fe8f9a836fe 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1947,18 +1947,6 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s } proctime := time.Since(start) // processing + validation - // Update the metrics touched during block processing and validation - accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) - storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) - accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) - storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) - accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) - triehash := statedb.AccountHashes // The time spent on tries hashing - trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates // The time spent on tries update - trieRead := statedb.AccountReads + statedb.StorageReads // The time spent on account read and storage read - blockExecutionTimer.Update(ptime - trieRead) // The time spent on EVM processing - blockValidationTimer.Update(vtime - (triehash + trieUpdate)) // The time spent on block validation - // Write the block to the chain and get the status. var ( wstart = time.Now() @@ -1973,6 +1961,18 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s if err != nil { return nil, err } + // Update the metrics touched during block processing and validation + accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing) + storageReadTimer.Update(statedb.StorageReads) // Storage reads are complete(in processing) + accountUpdateTimer.Update(statedb.AccountUpdates) // Account updates are complete(in validation) + storageUpdateTimer.Update(statedb.StorageUpdates) // Storage updates are complete(in validation) + accountHashTimer.Update(statedb.AccountHashes) // Account hashes are complete(in validation) + triehash := statedb.AccountHashes // The time spent on account trie hashing + trieUpdate := statedb.AccountUpdates + statedb.StorageUpdates // The time spent on account trie update + storage tries update and hashing + trieRead := statedb.AccountReads + statedb.StorageReads // The time spent on account read and storage read + blockExecutionTimer.Update(ptime - trieRead) // The time spent on EVM processing + blockValidationTimer.Update(vtime - (triehash + trieUpdate)) // The time spent on block validation + // Update the metrics touched during block commit accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them diff --git a/core/state/metrics.go b/core/state/metrics.go index 9e90ff1b0026..e702ef3a81a6 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -27,9 +27,4 @@ var ( storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) - - trieAccountReadTimer = metrics.NewRegisteredResettingTimer("state/trie/account/reads", nil) - trieStorageReadTimer = metrics.NewRegisteredResettingTimer("state/trie/storage/reads", nil) - snapshotAccountReadTimer = metrics.NewRegisteredResettingTimer("state/snapshot/account/reads", nil) - snapshotStorageReadTimer = metrics.NewRegisteredResettingTimer("state/snapshot/storage/reads", nil) ) diff --git a/core/state/reader.go b/core/state/reader.go index c3e5f9e8edf2..f4526308fb8a 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -88,7 +88,6 @@ func newStateReader(root common.Hash, snaps *snapshot.Tree) (*stateReader, error func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) { defer func(start time.Time) { r.accountTime += time.Since(start) - snapshotAccountReadTimer.UpdateSince(start) }(time.Now()) ret, err := r.snap.Account(crypto.HashData(r.buff, addr.Bytes())) @@ -123,7 +122,6 @@ func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { defer func(start time.Time) { r.storageTime += time.Since(start) - snapshotStorageReadTimer.UpdateSince(start) }(time.Now()) addrHash := crypto.HashData(r.buff, addr.Bytes()) @@ -207,7 +205,6 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { defer func(start time.Time) { r.accountTime += time.Since(start) - trieAccountReadTimer.UpdateSince(start) }(time.Now()) account, err := r.mainTrie.GetAccount(addr) @@ -230,7 +227,6 @@ func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { func (r *trieReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { defer func(start time.Time) { r.storageTime += time.Since(start) - trieStorageReadTimer.UpdateSince(start) }(time.Now()) var ( diff --git a/core/state/statedb.go b/core/state/statedb.go index 85cc2b7189b8..03d2ebe82ded 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1304,7 +1304,7 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateU // Submit the statistics of reader to metric system and reset it with new root aTime, sTime := s.reader.Stats() s.AccountReads += aTime - s.AccountReads += sTime + s.StorageReads += sTime s.reader, _ = s.db.Reader(s.originalRoot) return ret, err From 575a3be8527e4dc75a7d41f9efa0657e4cefcef2 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 25 Jul 2024 10:41:09 +0800 Subject: [PATCH 05/10] core, trie: extend state reader interface with StorageExists --- core/rawdb/accessors_snapshot.go | 7 +++ core/state/database.go | 5 ++ core/state/reader.go | 69 +++++++++++++++++++++++++++ core/state/snapshot/difflayer.go | 31 ++++++++++++ core/state/snapshot/disklayer.go | 31 ++++++++++++ core/state/snapshot/disklayer_test.go | 48 +++++++++++++++++++ core/state/snapshot/snapshot.go | 4 ++ trie/secure_trie.go | 10 ++++ trie/secure_trie_test.go | 33 +++++++++++++ trie/verkle.go | 11 +++++ 10 files changed, 249 insertions(+) diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go index 5cea581fcda3..4d9a8c61aad4 100644 --- a/core/rawdb/accessors_snapshot.go +++ b/core/rawdb/accessors_snapshot.go @@ -98,6 +98,13 @@ func ReadStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash commo return data } +// HasStorageSnapshot returns a flag indicating whether the requested storage +// slot exists. +func HasStorageSnapshot(db ethdb.KeyValueReader, accountHash, storageHash common.Hash) bool { + exists, _ := db.Has(storageSnapshotKey(accountHash, storageHash)) + return exists +} + // WriteStorageSnapshot stores the snapshot entry of a storage trie leaf. func WriteStorageSnapshot(db ethdb.KeyValueWriter, accountHash, storageHash common.Hash, entry []byte) { if err := db.Put(storageSnapshotKey(accountHash, storageHash), entry); err != nil { diff --git a/core/state/database.go b/core/state/database.go index 3a23add82892..afb032f4f3ad 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -95,6 +95,11 @@ type Trie interface { // a trie.MissingNodeError is returned. GetStorage(addr common.Address, key []byte) ([]byte, error) + // StorageExists returns a flag indicating whether the requested storage slot + // is existent in the trie or not. An error should be returned if the trie + // state is internally corrupted. + StorageExists(addr common.Address, key []byte) (bool, error) + // UpdateAccount abstracts an account write to the trie. It encodes the // provided account object with associated algorithm and then updates it // in the trie with provided address. diff --git a/core/state/reader.go b/core/state/reader.go index f4526308fb8a..da609b3f2f97 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -49,6 +49,10 @@ type Reader interface { // - The returned storage slot is safe to modify after the call Storage(addr common.Address, slot common.Hash) (common.Hash, error) + // StorageExists implements Reader, returning a flag indicating whether the + // requested storage slot exists. + StorageExists(addr common.Address, slot common.Hash) (bool, error) + // Stats returns the statistics of the reader, specifically detailing the time // spent on account reading and storage reading. Stats() (time.Duration, time.Duration) @@ -142,6 +146,18 @@ func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash return value, nil } +// StorageExists implements Reader, returning a flag indicating whether the +// requested storage slot exists. +func (r *stateReader) StorageExists(addr common.Address, key common.Hash) (bool, error) { + defer func(start time.Time) { + r.storageTime += time.Since(start) + }(time.Now()) + + addrHash := crypto.HashData(r.buff, addr.Bytes()) + slotHash := crypto.HashData(r.buff, key.Bytes()) + return r.snap.StorageExists(addrHash, slotHash) +} + // Stats implements Reader, returning the time spent on account reading and // storage reading from the snapshot. func (r *stateReader) Stats() (time.Duration, time.Duration) { @@ -266,6 +282,45 @@ func (r *trieReader) Storage(addr common.Address, key common.Hash) (common.Hash, return value, nil } +// StorageExists implements Reader, returning a flag indicating whether the +// requested storage slot exists. An error will be returned if the trie data +// is internally corrupted. +func (r *trieReader) StorageExists(addr common.Address, key common.Hash) (bool, error) { + defer func(start time.Time) { + r.storageTime += time.Since(start) + }(time.Now()) + + var ( + tr Trie + found bool + ) + if r.db.IsVerkle() { + tr = r.mainTrie + } else { + tr, found = r.subTries[addr] + if !found { + root, ok := r.subRoots[addr] + + // The storage slot is accessed without account caching. It's unexpected + // behavior but try to resolve the account first anyway. + if !ok { + _, err := r.Account(addr) + if err != nil { + return false, err + } + root = r.subRoots[addr] + } + var err error + tr, err = trie.NewStateTrie(trie.StorageTrieID(r.root, crypto.HashData(r.buff, addr.Bytes()), root), r.db) + if err != nil { + return false, err + } + r.subTries[addr] = tr + } + } + return tr.StorageExists(addr, key.Bytes()) +} + // Stats implements Reader, returning the time spent on account reading and // storage reading from the trie. func (r *trieReader) Stats() (time.Duration, time.Duration) { @@ -346,6 +401,20 @@ func (r *multiReader) Storage(addr common.Address, slot common.Hash) (common.Has return common.Hash{}, errors.Join(errs...) } +// StorageExists implements Reader, returning a flag indicating whether the +// requested storage slot exists. +func (r *multiReader) StorageExists(addr common.Address, slot common.Hash) (bool, error) { + var errs []error + for _, reader := range r.readers { + exists, err := reader.StorageExists(addr, slot) + if err == nil { + return exists, nil + } + errs = append(errs, err) + } + return false, errors.Join(errs...) +} + // Stats implements Reader, returning the time spent on account reading and // storage reading from the reader. func (r *multiReader) Stats() (time.Duration, time.Duration) { diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 779c1ea98c2f..7e62f37726b2 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -365,6 +365,37 @@ func (dl *diffLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro return dl.storage(accountHash, storageHash, 0) } +// StorageExists returns a flag indicating whether the requested storage slot is +// existent or not. +func (dl *diffLayer) StorageExists(accountHash, storageHash common.Hash) (bool, error) { + // Check the bloom filter first whether there's even a point in reaching into + // all the maps in all the layers below + dl.lock.RLock() + // Check staleness before reaching further. + if dl.Stale() { + dl.lock.RUnlock() + return false, ErrSnapshotStale + } + hit := dl.diffed.ContainsHash(storageBloomHash(accountHash, storageHash)) + if !hit { + hit = dl.diffed.ContainsHash(destructBloomHash(accountHash)) + } + var origin *diskLayer + if !hit { + origin = dl.origin // extract origin while holding the lock + } + dl.lock.RUnlock() + + // If the bloom filter misses, don't even bother with traversing the memory + // diff layers, reach straight into the bottom persistent disk layer + if origin != nil { + snapshotBloomStorageMissMeter.Mark(1) + return origin.StorageExists(accountHash, storageHash) + } + // The bloom filter hit, start poking in the internal maps + return dl.StorageExists(accountHash, storageHash) +} + // storage is an internal version of Storage that skips the bloom filter checks // and uses the internal maps to try and retrieve the data. It's meant to be // used if a higher layer's bloom filter hit already. diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index f5518a204ca1..e1aaef367a7a 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -169,6 +169,37 @@ func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, erro return blob, nil } +// StorageExists returns a flag indicating whether the requested storage slot is +// existent or not. +func (dl *diskLayer) StorageExists(accountHash, storageHash common.Hash) (bool, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // If the layer was flattened into, consider it invalid (any live reference to + // the original should be marked as unusable). + if dl.stale { + return false, ErrSnapshotStale + } + key := append(accountHash[:], storageHash[:]...) + + // If the layer is being generated, ensure the requested hash has already been + // covered by the generator. + if dl.genMarker != nil && bytes.Compare(key, dl.genMarker) > 0 { + return false, ErrNotCoveredYet + } + // If we're in the disk layer, all diff layers missed + snapshotDirtyStorageMissMeter.Mark(1) + + // Try to retrieve the storage slot from the memory cache + if blob, found := dl.cache.HasGet(nil, key); found { + snapshotCleanStorageHitMeter.Mark(1) + snapshotCleanStorageReadMeter.Mark(int64(len(blob))) + return true, nil + } + snapshotCleanStorageMissMeter.Mark(1) + return rawdb.HasStorageSnapshot(dl.diskdb, accountHash, storageHash), nil +} + // Update creates a new layer on top of the existing snapshot diff tree with // the specified data items. Note, the maps are retained by the method to avoid // copying everything. diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 168458c40519..5623e51b66d2 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -572,3 +572,51 @@ func TestDiskSeek(t *testing.T) { } } } + +func TestDiskStorageExists(t *testing.T) { + // Create some accounts in the disk layer + db := rawdb.NewMemoryDatabase() + defer db.Close() + + // fill storage slot with zero size + rawdb.WriteStorageSnapshot(db, common.Hash{0x1}, common.Hash{0x1}, []byte{}) + rawdb.WriteStorageSnapshot(db, common.Hash{0x1}, common.Hash{0x2}, nil) + rawdb.WriteStorageSnapshot(db, common.Hash{0x1}, common.Hash{0x3}, []byte{0x1}) + + dl := &diskLayer{ + diskdb: db, + cache: fastcache.New(500 * 1024), + root: randomHash(), + } + // Test some different seek positions + type testcase struct { + key common.Hash + expect bool + } + var cases = []testcase{ + { + common.Hash{0x0}, false, + }, + { + common.Hash{0x1}, true, + }, + { + common.Hash{0x2}, true, + }, + { + common.Hash{0x3}, true, + }, + { + common.Hash{0x4}, false, + }, + } + for i, tc := range cases { + result, err := dl.StorageExists(common.Hash{0x1}, tc.key) + if err != nil { + t.Fatalf("Failed to query disk layer: %v", err) + } + if result != tc.expect { + t.Fatalf("%d, unexpected result, want %t, got: %t", i, tc.expect, result) + } + } +} diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 752f4359fb85..638d15ec4059 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -112,6 +112,10 @@ type Snapshot interface { // Storage directly retrieves the storage data associated with a particular hash, // within a particular account. Storage(accountHash, storageHash common.Hash) ([]byte, error) + + // StorageExists returns a flag indicating whether the requested storage slot is + // existent or not. + StorageExists(accountHash, storageHash common.Hash) (bool, error) } // snapshot is the internal version of the snapshot data layer that supports some diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 6eb6defa45eb..7a7198ee9e25 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -113,6 +113,16 @@ func (t *StateTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { return content, err } +// StorageExists implements state.Trie, returning a flag indicating whether the +// requested storage slot is existent or not. +func (t *StateTrie) StorageExists(addr common.Address, key []byte) (bool, error) { + data, err := t.GetStorage(addr, key) + if err != nil { + return false, nil + } + return len(data) != 0, nil +} + // GetAccount attempts to retrieve an account with provided account address. // If the specified account is not in the trie, nil will be returned. // If a trie node is not found in the database, a MissingNodeError is returned. diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 59958d33f4cf..5d950b5de16c 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -147,3 +147,36 @@ func TestStateTrieConcurrency(t *testing.T) { // Wait for all threads to finish pend.Wait() } + +func TestSecureStorageExists(t *testing.T) { + trie := newEmptySecure() + + // Zero size value + trie.MustUpdate([]byte("foo"), []byte("")) + exists, err := trie.StorageExists(common.Address{}, []byte("foo")) + if err != nil { + t.Fatalf("trie is corrupted: %v", err) + } + if exists { + t.Fatal("Unexpected trie element") + } + + // Non-existent value + exists, err = trie.StorageExists(common.Address{}, []byte("dead")) + if err != nil { + t.Fatalf("trie is corrupted: %v", err) + } + if exists { + t.Fatal("Unexpected trie element") + } + + // Non-zero size value + trie.MustUpdate([]byte("foo"), []byte("bar")) + exists, err = trie.StorageExists(common.Address{}, []byte("foo")) + if err != nil { + t.Fatalf("trie is corrupted: %v", err) + } + if !exists { + t.Fatal("Trie element is missing") + } +} diff --git a/trie/verkle.go b/trie/verkle.go index fb4d81281cbd..a4260648f6df 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -127,6 +127,17 @@ func (t *VerkleTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) return common.TrimLeftZeroes(val), nil } +// StorageExists implements state.Trie, returning a flag indicating whether the +// requested storage slot is existent or not. +func (t *VerkleTrie) StorageExists(addr common.Address, key []byte) (bool, error) { + k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) + val, err := t.root.Get(k, t.nodeResolver) + if err != nil { + return false, err + } + return len(val) != 0, nil +} + // UpdateAccount implements state.Trie, writing the provided account into the tree. // If the tree is corrupted, an error will be returned. func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount) error { From 34089b905e7c62ffd5c740fc0f608de275dacc4b Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:17:59 +0200 Subject: [PATCH 06/10] implement FILL_COST --- core/state/access_events.go | 93 ++++++++++++++++++++---------------- core/state/statedb.go | 12 +++++ core/vm/interface.go | 2 + core/vm/operations_verkle.go | 20 ++++---- ethdb/memorydb/memorydb.go | 6 +-- 5 files changed, 81 insertions(+), 52 deletions(-) diff --git a/core/state/access_events.go b/core/state/access_events.go index 4b6c7c7e69bb..a8946fe931cd 100644 --- a/core/state/access_events.go +++ b/core/state/access_events.go @@ -44,14 +44,16 @@ var zeroTreeIndex uint256.Int type AccessEvents struct { branches map[branchAccessKey]mode chunks map[chunkAccessKey]mode + fills map[chunkAccessKey]struct{} pointCache *utils.PointCache } -func NewAccessEvents(pointCache *utils.PointCache) *AccessEvents { +func NewAccessEvents(pointCache *utils.PointCache, fills map[chunkAccessKey]struct{}) *AccessEvents { return &AccessEvents{ branches: make(map[branchAccessKey]mode), chunks: make(map[chunkAccessKey]mode), + fills: fills, pointCache: pointCache, } } @@ -66,6 +68,9 @@ func (ae *AccessEvents) Merge(other *AccessEvents) { for k, chunk := range other.chunks { ae.chunks[k] |= chunk } + for k := range other.fills { + ae.fills[k] = struct{}{} + } } // Keys returns, predictably, the list of keys that were touched during the @@ -85,6 +90,7 @@ func (ae *AccessEvents) Copy() *AccessEvents { cpy := &AccessEvents{ branches: maps.Clone(ae.branches), chunks: maps.Clone(ae.chunks), + fills: maps.Clone(ae.fills), pointCache: ae.pointCache, } return cpy @@ -92,13 +98,13 @@ func (ae *AccessEvents) Copy() *AccessEvents { // AddAccount returns the gas to be charged for each of the currently cold // member fields of an account. -func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool) uint64 { +func (ae *AccessEvents) AddAccount(addr common.Address, isWrite, isFill bool) uint64 { var gas uint64 - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, isWrite) - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, isWrite) - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, isWrite) - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeKeccakLeafKey, isWrite) - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeSizeLeafKey, isWrite) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, isWrite, isFill) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, isWrite, isFill) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, isWrite, isFill) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeKeccakLeafKey, isWrite, isFill) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeSizeLeafKey, isWrite, isFill) return gas } @@ -107,28 +113,28 @@ func (ae *AccessEvents) AddAccount(addr common.Address, isWrite bool) uint64 { // call to that account. func (ae *AccessEvents) MessageCallGas(destination common.Address) uint64 { var gas uint64 - gas += ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.VersionLeafKey, false) - gas += ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.CodeSizeLeafKey, false) + gas += ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.VersionLeafKey, false, false) + gas += ae.touchAddressAndChargeGas(destination, zeroTreeIndex, utils.CodeSizeLeafKey, false, false) return gas } // ValueTransferGas returns the gas to be charged for each of the currently // cold balance member fields of the caller and the callee accounts. -func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address) uint64 { +func (ae *AccessEvents) ValueTransferGas(callerAddr, targetAddr common.Address, isFill bool) uint64 { var gas uint64 - gas += ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, utils.BalanceLeafKey, true) - gas += ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, utils.BalanceLeafKey, true) + gas += ae.touchAddressAndChargeGas(callerAddr, zeroTreeIndex, utils.BalanceLeafKey, true, false) + gas += ae.touchAddressAndChargeGas(targetAddr, zeroTreeIndex, utils.BalanceLeafKey, true, isFill) return gas } // ContractCreateInitGas returns the access gas costs for the initialization of // a contract creation. -func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, createSendsValue bool) uint64 { +func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, createSendsValue, isFill bool) uint64 { var gas uint64 - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, true) - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, true) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, true, true) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, true, true) if createSendsValue { - gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, true) + gas += ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, true, isFill) } return gas } @@ -136,33 +142,33 @@ func (ae *AccessEvents) ContractCreateInitGas(addr common.Address, createSendsVa // AddTxOrigin adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. func (ae *AccessEvents) AddTxOrigin(originAddr common.Address) { - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.VersionLeafKey, false) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.BalanceLeafKey, true) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.NonceLeafKey, true) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeKeccakLeafKey, false) - ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeSizeLeafKey, false) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.VersionLeafKey, false, false) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.BalanceLeafKey, true, false) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.NonceLeafKey, true, false) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeKeccakLeafKey, false, false) + ae.touchAddressAndChargeGas(originAddr, zeroTreeIndex, utils.CodeSizeLeafKey, false, false) } // AddTxDestination adds the member fields of the sender account to the access event list, // so that cold accesses are not charged, since they are covered by the 21000 gas. -func (ae *AccessEvents) AddTxDestination(addr common.Address, sendsValue bool) { - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, false) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, sendsValue) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, false) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeKeccakLeafKey, false) - ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeSizeLeafKey, false) +func (ae *AccessEvents) AddTxDestination(addr common.Address, sendsValue, isFill bool) { + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, false, false) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, sendsValue, isFill) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, false, false) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeKeccakLeafKey, false, false) + ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeSizeLeafKey, false, false) } // SlotGas returns the amount of gas to be charged for a cold storage access. -func (ae *AccessEvents) SlotGas(addr common.Address, slot common.Hash, isWrite bool) uint64 { +func (ae *AccessEvents) SlotGas(addr common.Address, slot common.Hash, isWrite, isFill bool) uint64 { treeIndex, subIndex := utils.StorageIndex(slot.Bytes()) - return ae.touchAddressAndChargeGas(addr, *treeIndex, subIndex, isWrite) + return ae.touchAddressAndChargeGas(addr, *treeIndex, subIndex, isWrite, isFill) } // touchAddressAndChargeGas adds any missing access event to the access event list, and returns the cold // access cost to be charged, if need be. -func (ae *AccessEvents) touchAddressAndChargeGas(addr common.Address, treeIndex uint256.Int, subIndex byte, isWrite bool) uint64 { - stemRead, selectorRead, stemWrite, selectorWrite, selectorFill := ae.touchAddress(addr, treeIndex, subIndex, isWrite) +func (ae *AccessEvents) touchAddressAndChargeGas(addr common.Address, treeIndex uint256.Int, subIndex byte, isWrite, isFill bool) uint64 { + stemRead, selectorRead, stemWrite, selectorWrite, selectorFill := ae.touchAddress(addr, treeIndex, subIndex, isWrite, isFill) var gas uint64 if stemRead { @@ -184,7 +190,7 @@ func (ae *AccessEvents) touchAddressAndChargeGas(addr common.Address, treeIndex } // touchAddress adds any missing access event to the access event list. -func (ae *AccessEvents) touchAddress(addr common.Address, treeIndex uint256.Int, subIndex byte, isWrite bool) (bool, bool, bool, bool, bool) { +func (ae *AccessEvents) touchAddress(addr common.Address, treeIndex uint256.Int, subIndex byte, isWrite, isFill bool) (bool, bool, bool, bool, bool) { branchKey := newBranchAccessKey(addr, treeIndex) chunkKey := newChunkAccessKey(branchKey, subIndex) @@ -212,7 +218,12 @@ func (ae *AccessEvents) touchAddress(addr common.Address, treeIndex uint256.Int, chunkWrite = true ae.chunks[chunkKey] |= AccessWitnessWriteFlag } - // TODO: charge chunk filling costs if the leaf was previously empty in the state + _, ok := ae.fills[chunkKey] + if isFill && !ok { + chunkFill = true + ae.fills[chunkKey] = struct{}{} + } + } return branchRead, chunkRead, branchWrite, chunkWrite, chunkFill } @@ -242,7 +253,7 @@ func newChunkAccessKey(branchKey branchAccessKey, leafKey byte) chunkAccessKey { } // CodeChunksRangeGas is a helper function to touch every chunk in a code range and charge witness gas costs -func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, size uint64, codeLen uint64, isWrite bool) uint64 { +func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, size uint64, codeLen uint64, isWrite, isFill bool) uint64 { // note that in the case where the copied code is outside the range of the // contract code but touches the last leaf with contract code in it, // we don't include the last leaf of code in the AccessWitness. The @@ -265,7 +276,7 @@ func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, for chunkNumber := startPC / 31; chunkNumber <= endPC/31; chunkNumber++ { treeIndex := *uint256.NewInt((chunkNumber + 128) / 256) subIndex := byte((chunkNumber + 128) % 256) - gas := ae.touchAddressAndChargeGas(contractAddr, treeIndex, subIndex, isWrite) + gas := ae.touchAddressAndChargeGas(contractAddr, treeIndex, subIndex, isWrite, isFill) var overflow bool statelessGasCharged, overflow = math.SafeAdd(statelessGasCharged, gas) if overflow { @@ -280,7 +291,7 @@ func (ae *AccessEvents) CodeChunksRangeGas(contractAddr common.Address, startPC, // Note that an access in write mode implies an access in read mode, whereas an // access in read mode does not imply an access in write mode. func (ae *AccessEvents) VersionGas(addr common.Address, isWrite bool) uint64 { - return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, isWrite) + return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.VersionLeafKey, isWrite, false) } // BalanceGas adds the account's balance to the accessed data, and returns the @@ -288,8 +299,8 @@ func (ae *AccessEvents) VersionGas(addr common.Address, isWrite bool) uint64 { // in write mode. If false, the charged gas corresponds to an access in read mode. // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. -func (ae *AccessEvents) BalanceGas(addr common.Address, isWrite bool) uint64 { - return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, isWrite) +func (ae *AccessEvents) BalanceGas(addr common.Address, isWrite, isFill bool) uint64 { + return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.BalanceLeafKey, isWrite, isFill) } // NonceGas adds the account's nonce to the accessed data, and returns the @@ -298,7 +309,7 @@ func (ae *AccessEvents) BalanceGas(addr common.Address, isWrite bool) uint64 { // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. func (ae *AccessEvents) NonceGas(addr common.Address, isWrite bool) uint64 { - return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, isWrite) + return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.NonceLeafKey, isWrite, false) } // CodeSizeGas adds the account's code size to the accessed data, and returns the @@ -307,7 +318,7 @@ func (ae *AccessEvents) NonceGas(addr common.Address, isWrite bool) uint64 { // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. func (ae *AccessEvents) CodeSizeGas(addr common.Address, isWrite bool) uint64 { - return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeSizeLeafKey, isWrite) + return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeSizeLeafKey, isWrite, false) } // CodeHashGas adds the account's code hash to the accessed data, and returns the @@ -316,5 +327,5 @@ func (ae *AccessEvents) CodeSizeGas(addr common.Address, isWrite bool) uint64 { // Note that an access in write mode implies an access in read mode, whereas an access in // read mode does not imply an access in write mode. func (ae *AccessEvents) CodeHashGas(addr common.Address, isWrite bool) uint64 { - return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeKeccakLeafKey, isWrite) + return ae.touchAddressAndChargeGas(addr, zeroTreeIndex, utils.CodeKeccakLeafKey, isWrite, false) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 03d2ebe82ded..70ece087c82c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,6 +28,7 @@ import ( "sync/atomic" "time" + "github.com/cockroachdb/pebble" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -1434,3 +1436,13 @@ func (s *StateDB) PointCache() *utils.PointCache { func (s *StateDB) Witness() *stateless.Witness { return s.witness } + +func (s *StateDB) IsSlotFilled(addr common.Address, slot common.Hash) bool { + // The snapshot can not be used, because it uses the old encoding where + // no difference is made between 0 and no data. + _, err := s.db.DiskDB().Get(utils.StorageSlotKeyWithEvaluatedAddress(s.accessList.pointCache.GetTreeKeyHeader(addr[:]), slot[:])) + // The error needs to be checked because we want to be future-proof + // and not rely on the length of the encoding, in case 0-values are + // somehow compressed later. + return errors.Is(pebble.ErrNotFound, err) || errors.Is(memorydb.ErrMemorydbNotFound, err) +} diff --git a/core/vm/interface.go b/core/vm/interface.go index 5f426435650d..d280a3f91931 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -90,6 +90,8 @@ type StateDB interface { AddPreimage(common.Hash, []byte) Witness() *stateless.Witness + + IsSlotFilled(common.Address, common.Hash) bool } // CallContext provides a basic interface for the EVM calling conventions. The EVM diff --git a/core/vm/operations_verkle.go b/core/vm/operations_verkle.go index 73eb05974dc0..5a0ef73a7117 100644 --- a/core/vm/operations_verkle.go +++ b/core/vm/operations_verkle.go @@ -23,7 +23,11 @@ import ( ) func gasSStore4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { - gas := evm.AccessEvents.SlotGas(contract.Address(), stack.peek().Bytes32(), true) + var ( + addr = contract.Address() + slot = common.Hash(stack.peek().Bytes32()) + ) + gas := evm.AccessEvents.SlotGas(addr, slot, true, evm.StateDB.IsSlotFilled(addr, slot)) if gas == 0 { gas = params.WarmStorageReadCostEIP2929 } @@ -31,7 +35,7 @@ func gasSStore4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memo } func gasSLoad4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { - gas := evm.AccessEvents.SlotGas(contract.Address(), stack.peek().Bytes32(), false) + gas := evm.AccessEvents.SlotGas(contract.Address(), stack.peek().Bytes32(), false, false) if gas == 0 { gas = params.WarmStorageReadCostEIP2929 } @@ -40,7 +44,7 @@ func gasSLoad4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memor func gasBalance4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { address := stack.peek().Bytes20() - gas := evm.AccessEvents.BalanceGas(address, false) + gas := evm.AccessEvents.BalanceGas(address, false, false) if gas == 0 { gas = params.WarmStorageReadCostEIP2929 } @@ -104,15 +108,15 @@ func gasSelfdestructEIP4762(evm *EVM, contract *Contract, stack *Stack, mem *Mem contractAddr := contract.Address() statelessGas := evm.AccessEvents.VersionGas(contractAddr, false) statelessGas += evm.AccessEvents.CodeSizeGas(contractAddr, false) - statelessGas += evm.AccessEvents.BalanceGas(contractAddr, false) + statelessGas += evm.AccessEvents.BalanceGas(contractAddr, false, false) if contractAddr != beneficiaryAddr { - statelessGas += evm.AccessEvents.BalanceGas(beneficiaryAddr, false) + statelessGas += evm.AccessEvents.BalanceGas(beneficiaryAddr, false, false) } // Charge write costs if it transfers value if evm.StateDB.GetBalance(contractAddr).Sign() != 0 { - statelessGas += evm.AccessEvents.BalanceGas(contractAddr, true) + statelessGas += evm.AccessEvents.BalanceGas(contractAddr, true, false) if contractAddr != beneficiaryAddr { - statelessGas += evm.AccessEvents.BalanceGas(beneficiaryAddr, true) + statelessGas += evm.AccessEvents.BalanceGas(beneficiaryAddr, true, !evm.StateDB.Exist(beneficiaryAddr)) } } return statelessGas, nil @@ -133,7 +137,7 @@ func gasCodeCopyEip4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, } _, copyOffset, nonPaddedCopyLength := getDataAndAdjustedBounds(contract.Code, uint64CodeOffset, length.Uint64()) if !contract.IsDeployment { - gas += evm.AccessEvents.CodeChunksRangeGas(contract.Address(), copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false) + gas += evm.AccessEvents.CodeChunksRangeGas(contract.Address(), copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false, false) } return gas, nil } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 532e0dfe3f36..0fbe6545f296 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -32,9 +32,9 @@ var ( // invocation of a data access operation. errMemorydbClosed = errors.New("database closed") - // errMemorydbNotFound is returned if a key is requested that is not found in + // ErrMemorydbNotFound is returned if a key is requested that is not found in // the provided memory database. - errMemorydbNotFound = errors.New("not found") + ErrMemorydbNotFound = errors.New("not found") ) // Database is an ephemeral key-value store. Apart from basic data storage @@ -94,7 +94,7 @@ func (db *Database) Get(key []byte) ([]byte, error) { if entry, ok := db.db[string(key)]; ok { return common.CopyBytes(entry), nil } - return nil, errMemorydbNotFound + return nil, ErrMemorydbNotFound } // Put inserts the given value into the key-value store. From 1a358f66e2b4a3ad3a97aae737cf34757e7b1067 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:42:15 +0200 Subject: [PATCH 07/10] rebase FILL_COST --- core/state/access_events.go | 10 +++- core/state/access_events_test.go | 12 ++--- core/state/statedb.go | 30 ++++++----- core/state_processor.go | 2 + core/state_processor_test.go | 93 +++++++++++++++++++++++++++++++- core/state_transition.go | 6 ++- core/vm/eips.go | 6 +-- core/vm/evm.go | 25 ++++++--- core/vm/gas_table.go | 4 +- core/vm/interface.go | 2 + core/vm/interpreter.go | 2 +- 11 files changed, 157 insertions(+), 35 deletions(-) diff --git a/core/state/access_events.go b/core/state/access_events.go index a8946fe931cd..8eef37caf712 100644 --- a/core/state/access_events.go +++ b/core/state/access_events.go @@ -49,11 +49,11 @@ type AccessEvents struct { pointCache *utils.PointCache } -func NewAccessEvents(pointCache *utils.PointCache, fills map[chunkAccessKey]struct{}) *AccessEvents { +func NewAccessEvents(pointCache *utils.PointCache) *AccessEvents { return &AccessEvents{ branches: make(map[branchAccessKey]mode), chunks: make(map[chunkAccessKey]mode), - fills: fills, + fills: make(map[chunkAccessKey]struct{}), pointCache: pointCache, } } @@ -96,6 +96,12 @@ func (ae *AccessEvents) Copy() *AccessEvents { return cpy } +// Reset resets all values of an access event, except its `fills` and point cache fields +func (ae *AccessEvents) Reset() { + ae.branches = make(map[branchAccessKey]mode) + ae.chunks = make(map[chunkAccessKey]mode) +} + // AddAccount returns the gas to be charged for each of the currently cold // member fields of an account. func (ae *AccessEvents) AddAccount(addr common.Address, isWrite, isFill bool) uint64 { diff --git a/core/state/access_events_test.go b/core/state/access_events_test.go index c8c93accfdfe..bb67953ebba2 100644 --- a/core/state/access_events_test.go +++ b/core/state/access_events_test.go @@ -52,7 +52,7 @@ func TestAccountHeaderGas(t *testing.T) { } // Check cold read costs in the same group no longer incur the branch read cost - gas = ae.BalanceGas(testAddr, false) + gas = ae.BalanceGas(testAddr, false, false) if gas != params.WitnessChunkReadCost { t.Fatalf("incorrect gas computed, got %d, want %d", gas, params.WitnessChunkReadCost) } @@ -82,20 +82,20 @@ func TestAccountHeaderGas(t *testing.T) { } // Check a write without a read charges both read and write costs - gas = ae.BalanceGas(testAddr2, true) + gas = ae.BalanceGas(testAddr2, true, false) if want := params.WitnessBranchReadCost + params.WitnessBranchWriteCost + params.WitnessChunkWriteCost + params.WitnessChunkReadCost; gas != want { t.Fatalf("incorrect gas computed, got %d, want %d", gas, want) } // Check that a write followed by a read charges nothing - gas = ae.BalanceGas(testAddr2, false) + gas = ae.BalanceGas(testAddr2, false, false) if gas != 0 { t.Fatalf("incorrect gas computed, got %d, want %d", gas, 0) } // Check that reading a slot from the account header only charges the // chunk read cost. - gas = ae.SlotGas(testAddr, common.Hash{}, false) + gas = ae.SlotGas(testAddr, common.Hash{}, false, false) if gas != params.WitnessChunkReadCost { t.Fatalf("incorrect gas computed, got %d, want %d", gas, params.WitnessChunkReadCost) } @@ -112,13 +112,13 @@ func TestContractCreateInitGas(t *testing.T) { } // Check cold read cost, without a value - gas := ae.ContractCreateInitGas(testAddr, false) + gas := ae.ContractCreateInitGas(testAddr, false, false) if want := params.WitnessBranchWriteCost + params.WitnessBranchReadCost + params.WitnessChunkWriteCost*2 + params.WitnessChunkReadCost*2; gas != want { t.Fatalf("incorrect gas computed, got %d, want %d", gas, want) } // Check warm read cost - gas = ae.ContractCreateInitGas(testAddr, false) + gas = ae.ContractCreateInitGas(testAddr, false, false) if gas != 0 { t.Fatalf("incorrect gas computed, got %d, want %d", gas, 0) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 70ece087c82c..9b8977893ed5 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,7 +28,6 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/pebble" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -36,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -136,7 +134,8 @@ type StateDB struct { preimages map[common.Hash][]byte // Per-transaction access list - accessList *accessList + accessList *accessList + accessEvents *AccessEvents // Transient storage transientStorage transientStorage @@ -178,7 +177,7 @@ func New(root common.Hash, db Database) (*StateDB, error) { if err != nil { return nil, err } - return &StateDB{ + sdb := &StateDB{ db: db, trie: tr, originalRoot: root, @@ -191,7 +190,12 @@ func New(root common.Hash, db Database) (*StateDB, error) { journal: newJournal(), accessList: newAccessList(), transientStorage: newTransientStorage(), - }, nil + } + if db.TrieDB().IsVerkle() { + sdb.accessEvents = NewAccessEvents(db.(*CachingDB).pointCache) + } + + return sdb, nil } // SetLogger sets the logger for account update hooks. @@ -682,6 +686,9 @@ func (s *StateDB) Copy() *StateDB { if s.witness != nil { state.witness = s.witness.Copy() } + if s.accessEvents != nil { + state.accessEvents = s.accessEvents.Copy() + } // Deep copy cached state objects. for addr, obj := range s.stateObjects { state.stateObjects[addr] = obj.deepCopy(state) @@ -1438,11 +1445,10 @@ func (s *StateDB) Witness() *stateless.Witness { } func (s *StateDB) IsSlotFilled(addr common.Address, slot common.Hash) bool { - // The snapshot can not be used, because it uses the old encoding where - // no difference is made between 0 and no data. - _, err := s.db.DiskDB().Get(utils.StorageSlotKeyWithEvaluatedAddress(s.accessList.pointCache.GetTreeKeyHeader(addr[:]), slot[:])) - // The error needs to be checked because we want to be future-proof - // and not rely on the length of the encoding, in case 0-values are - // somehow compressed later. - return errors.Is(pebble.ErrNotFound, err) || errors.Is(memorydb.ErrMemorydbNotFound, err) + ok, err := s.GetTrie().StorageExists(addr, slot[:]) + return err == nil && ok +} + +func (s *StateDB) AccessEvents() *AccessEvents { + return s.accessEvents } diff --git a/core/state_processor.go b/core/state_processor.go index c21f644f9851..7215e3f8e0e5 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -156,6 +156,8 @@ func ApplyTransactionWithEVM(msg *Message, config *params.ChainConfig, gp *GasPo receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce()) } + statedb.AccessEvents().Merge(evm.TxContext.AccessEvents) + // Set the receipt logs and create the bloom filter. receipt.Logs = statedb.GetLogs(tx.Hash(), blockNumber.Uint64(), blockHash) receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index af4d29b604da..ef463d62356c 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -19,6 +19,7 @@ package core import ( "crypto/ecdsa" "math/big" + "os" "testing" "github.com/ethereum/go-ethereum/common" @@ -32,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/holiman/uint256" @@ -473,7 +475,6 @@ func TestProcessVerkle(t *testing.T) { ) // Verkle trees use the snapshot, which must be enabled before the // data is saved into the tree+database. - // genesis := gspec.MustCommit(bcdb, triedb) cacheConfig := DefaultCacheConfigWithScheme("path") cacheConfig.SnapshotLimit = 0 blockchain, _ := NewBlockChain(bcdb, cacheConfig, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil, nil) @@ -528,3 +529,93 @@ func TestProcessVerkle(t *testing.T) { } } } + +// TestProcessVerkleFillThenNoFill tests the case where a slot is filled in a first transaction, for which the +// CHUNK_EDIT costs are filled, and then the same slot is written to in a second transaction, for which the CHUNK_EDIT +// costs are not charged, provided the two transactions occur in the same block. +func TestProcessVerkleFillThenNoFill(t *testing.T) { + var ( + config = ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), + ShanghaiTime: u64(0), + PragueTime: u64(0), + TerminalTotalDifficulty: common.Big0, + TerminalTotalDifficultyPassed: true, + } + signer = types.LatestSigner(config) + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + bcdb = rawdb.NewMemoryDatabase() // Database for the blockchain + coinbase = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") + gspec = &Genesis{ + Config: config, + Alloc: GenesisAlloc{ + coinbase: GenesisAccount{ + Balance: big.NewInt(1000000000000000000), // 1 ether + }, + common.Address{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}: GenesisAccount{ + // PUSH1 0, CALLDATALOAD, PUSH1 2, SSTORE + Code: []byte{0x60, 0, 0x35, 0x60, 2, 0x55}, + Balance: big.NewInt(1000000000000000000), + }, + }, + } + loggerCfg = &logger.Config{} + ) + os.MkdirAll("output", 0755) + traceFile, err := os.Create("./output/traces.jsonl") + if err != nil { + t.Fatal(err) + } + + // Verkle trees use the snapshot, which must be enabled before the + // data is saved into the tree+database. + blockchain, _ := NewBlockChain(bcdb, nil, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{Tracer: logger.NewJSONLogger(loggerCfg, traceFile)}, nil, nil) + defer blockchain.Stop() + + blockGasUsagesExpected := params.TxGas + 216 /* intrinsic gas */ + 3*vm.GasFastestStep + params.WitnessChunkReadCost + params.WitnessChunkWriteCost + _, chain, _, _, _ := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { + gen.SetPoS() + + // fill slot + tx, _ := types.SignTx(types.NewTransaction(0, common.Address{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, big.NewInt(999), blockGasUsagesExpected+params.WitnessChunkFillCost, big.NewInt(875000000), []byte{1}), signer, testKey) + gen.AddTx(tx) + // write but no fill + tx, _ = types.SignTx(types.NewTransaction(1, common.Address{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, big.NewInt(999), blockGasUsagesExpected, big.NewInt(875000000), []byte{2}), signer, testKey) + gen.AddTx(tx) + }) + + // check the proof for the last block + // err = trie.DeserializeAndVerifyVerkleProof(proofs[0], genesis.Root().Bytes(), chain[0].Root().Bytes(), keyvals[0]) + // if err != nil { + // t.Fatal(err) + // } + // t.Log("verfied verkle proof") + + endnum, err := blockchain.InsertChain(chain) + if err != nil { + t.Fatalf("block %d imported with error: %v", endnum, err) + } + + b := blockchain.GetBlockByNumber(1) + if b == nil { + t.Fatalf("expected block %d to be present in chain", 1) + } + if b.Hash() != chain[0].Hash() { + t.Fatalf("block #%d not found at expected height", b.NumberU64()) + } + if b.GasUsed() != 2*blockGasUsagesExpected+params.WitnessChunkFillCost { + t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), 2*blockGasUsagesExpected, b.GasUsed()) + } +} diff --git a/core/state_transition.go b/core/state_transition.go index 1a6a66a2fc14..eb6f7ff0a4b0 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -410,7 +410,9 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { st.evm.AccessEvents.AddTxOrigin(msg.From) if targetAddr := msg.To; targetAddr != nil { - st.evm.AccessEvents.AddTxDestination(*targetAddr, msg.Value.Sign() != 0) + // the fill is always set to true, as it will be covered by the intrinsic gas + // if needed. + st.evm.AccessEvents.AddTxDestination(*targetAddr, msg.Value.Sign() != 0, true) } } @@ -470,7 +472,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { // add the coinbase to the witness iff the fee is greater than 0 if rules.IsEIP4762 && fee.Sign() != 0 { - st.evm.AccessEvents.BalanceGas(st.evm.Context.Coinbase, true) + st.evm.AccessEvents.BalanceGas(st.evm.Context.Coinbase, true, true) } } diff --git a/core/vm/eips.go b/core/vm/eips.go index edd6ec8d0a2c..e9faf6277815 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -342,7 +342,7 @@ func opExtCodeCopyEIP4762(pc *uint64, interpreter *EVMInterpreter, scope *ScopeC self: AccountRef(addr), } paddedCodeCopy, copyOffset, nonPaddedCopyLength := getDataAndAdjustedBounds(code, uint64CodeOffset, length.Uint64()) - statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(addr, copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false) + statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(addr, copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false, false) if !scope.Contract.UseGas(statelessGas, interpreter.evm.Config.Tracer, tracing.GasChangeUnspecified) { scope.Contract.Gas = 0 return nil, ErrOutOfGas @@ -368,7 +368,7 @@ func opPush1EIP4762(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext // touch next chunk if PUSH1 is at the boundary. if so, *pc has // advanced past this boundary. contractAddr := scope.Contract.Address() - statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(contractAddr, *pc+1, uint64(1), uint64(len(scope.Contract.Code)), false) + statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(contractAddr, *pc+1, uint64(1), uint64(len(scope.Contract.Code)), false, false) if !scope.Contract.UseGas(statelessGas, interpreter.evm.Config.Tracer, tracing.GasChangeUnspecified) { scope.Contract.Gas = 0 return nil, ErrOutOfGas @@ -396,7 +396,7 @@ func makePushEIP4762(size uint64, pushByteSize int) executionFunc { if !scope.Contract.IsDeployment { contractAddr := scope.Contract.Address() - statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(contractAddr, uint64(start), uint64(pushByteSize), uint64(len(scope.Contract.Code)), false) + statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(contractAddr, uint64(start), uint64(pushByteSize), uint64(len(scope.Contract.Code)), false, false) if !scope.Contract.UseGas(statelessGas, interpreter.evm.Config.Tracer, tracing.GasChangeUnspecified) { scope.Contract.Gas = 0 return nil, ErrOutOfGas diff --git a/core/vm/evm.go b/core/vm/evm.go index 1944189b5da2..355275c5f950 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -161,7 +161,8 @@ func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig // This is not threadsafe and should only be done very cautiously. func (evm *EVM) Reset(txCtx TxContext, statedb StateDB) { if evm.chainRules.IsEIP4762 { - txCtx.AccessEvents = state.NewAccessEvents(statedb.PointCache()) + txCtx.AccessEvents = statedb.AccessEvents().Copy() + txCtx.AccessEvents.Reset() } evm.TxContext = txCtx evm.StateDB = statedb @@ -209,7 +210,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas if !evm.StateDB.Exist(addr) { if !isPrecompile && evm.chainRules.IsEIP4762 { // add proof of absence to witness - wgas := evm.AccessEvents.AddAccount(addr, false) + wgas := evm.AccessEvents.AddAccount(addr, false, false) if gas < wgas { evm.StateDB.RevertToSnapshot(snapshot) return nil, 0, ErrOutOfGas @@ -221,6 +222,15 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // Calling a non-existing account, don't do anything. return nil, gas, nil } + if !isPrecompile && evm.chainRules.IsEIP4762 && !value.IsZero() { + // Charge the fill+write costs if the account is created + wgas := evm.AccessEvents.AddAccount(addr, true, true) + if gas < wgas { + evm.StateDB.RevertToSnapshot(snapshot) + return nil, 0, ErrOutOfGas + } + gas -= wgas + } evm.StateDB.CreateAccount(addr) } evm.Context.Transfer(evm.StateDB, caller.Address(), addr, value) @@ -490,7 +500,8 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // It might be possible the contract code is deployed to a pre-existent // account with non-zero balance. snapshot := evm.StateDB.Snapshot() - if !evm.StateDB.Exist(address) { + addressExisted := evm.StateDB.Exist(address) + if !addressExisted { evm.StateDB.CreateAccount(address) } // CreateContract means that regardless of whether the account previously existed @@ -512,7 +523,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // Charge the contract creation init gas in verkle mode if evm.chainRules.IsEIP4762 { - if !contract.UseGas(evm.AccessEvents.ContractCreateInitGas(address, value.Sign() != 0), evm.Config.Tracer, tracing.GasChangeWitnessContractInit) { + if !contract.UseGas(evm.AccessEvents.ContractCreateInitGas(address, value.Sign() != 0, !addressExisted), evm.Config.Tracer, tracing.GasChangeWitnessContractInit) { err = ErrOutOfGas } } @@ -543,11 +554,13 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } } else { // Contract creation completed, touch the missing fields in the contract - if !contract.UseGas(evm.AccessEvents.AddAccount(address, true), evm.Config.Tracer, tracing.GasChangeWitnessContractCreation) { + // The isFill argument is always set to true, but if it has been charged, + // e.g. when performing the top-level call, it won't be charged twice. + if !contract.UseGas(evm.AccessEvents.AddAccount(address, true, true), evm.Config.Tracer, tracing.GasChangeWitnessContractCreation) { err = ErrCodeStoreOutOfGas } - if err == nil && len(ret) > 0 && !contract.UseGas(evm.AccessEvents.CodeChunksRangeGas(address, 0, uint64(len(ret)), uint64(len(ret)), true), evm.Config.Tracer, tracing.GasChangeWitnessCodeChunk) { + if err == nil && len(ret) > 0 && !contract.UseGas(evm.AccessEvents.CodeChunksRangeGas(address, 0, uint64(len(ret)), uint64(len(ret)), true, true), evm.Config.Tracer, tracing.GasChangeWitnessCodeChunk) { err = ErrCodeStoreOutOfGas } } diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index d294324b08c2..7c09f378453e 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -396,7 +396,7 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize } if evm.chainRules.IsEIP4762 { if transfersValue { - gas, overflow = math.SafeAdd(gas, evm.AccessEvents.ValueTransferGas(contract.Address(), address)) + gas, overflow = math.SafeAdd(gas, evm.AccessEvents.ValueTransferGas(contract.Address(), address, !evm.StateDB.Exist(address))) if overflow { return 0, ErrGasUintOverflow } @@ -432,7 +432,7 @@ func gasCallCode(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memory address := common.Address(stack.Back(1).Bytes20()) transfersValue := !stack.Back(2).IsZero() if transfersValue { - gas, overflow = math.SafeAdd(gas, evm.AccessEvents.ValueTransferGas(contract.Address(), address)) + gas, overflow = math.SafeAdd(gas, evm.AccessEvents.ValueTransferGas(contract.Address(), address, !evm.StateDB.Exist(address))) if overflow { return 0, ErrGasUintOverflow } diff --git a/core/vm/interface.go b/core/vm/interface.go index d280a3f91931..5aac4132790d 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -20,6 +20,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" @@ -90,6 +91,7 @@ type StateDB interface { AddPreimage(common.Hash, []byte) Witness() *stateless.Witness + AccessEvents() *state.AccessEvents IsSlotFilled(common.Address, common.Hash) bool } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 2b1ea3848352..9762ab533f52 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -228,7 +228,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( // if the PC ends up in a new "chunk" of verkleized code, charge the // associated costs. contractAddr := contract.Address() - contract.Gas -= in.evm.TxContext.AccessEvents.CodeChunksRangeGas(contractAddr, pc, 1, uint64(len(contract.Code)), false) + contract.Gas -= in.evm.TxContext.AccessEvents.CodeChunksRangeGas(contractAddr, pc, 1, uint64(len(contract.Code)), false, false) } // Get the operation from the jump table and validate the stack to ensure there are From 22ece67c2bde1da8746991bdf3fbab437aa7a39b Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:39:17 +0200 Subject: [PATCH 08/10] neuter DeleteAccount and rework gas costs --- core/state_processor_test.go | 4 ++-- trie/verkle.go | 16 ---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index ef463d62356c..ea04e7c30ec7 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -482,8 +482,8 @@ func TestProcessVerkle(t *testing.T) { txCost1 := params.TxGas txCost2 := params.TxGas - contractCreationCost := intrinsicContractCreationGas + uint64(2039 /* execution costs */) - codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + uint64(57444 /* execution costs */) + contractCreationCost := intrinsicContractCreationGas + 7*params.WitnessChunkFillCost + uint64(2939 /* execution costs */) + codeWithExtCodeCopyGas := intrinsicCodeWithExtCodeCopyGas + 41*params.WitnessChunkFillCost + uint64(56544 /* execution costs */) blockGasUsagesExpected := []uint64{ txCost1*2 + txCost2, txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, diff --git a/trie/verkle.go b/trie/verkle.go index a4260648f6df..e13d1c9d34d7 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -191,22 +191,6 @@ func (t *VerkleTrie) UpdateStorage(address common.Address, key, value []byte) er // trie. If the account was not existent in the trie, no error will be returned. // If the trie is corrupted, an error will be returned. func (t *VerkleTrie) DeleteAccount(addr common.Address) error { - var ( - err error - values = make([][]byte, verkle.NodeWidth) - ) - for i := 0; i < verkle.NodeWidth; i++ { - values[i] = zero[:] - } - switch n := t.root.(type) { - case *verkle.InternalNode: - err = n.InsertValuesAtStem(t.cache.GetStem(addr.Bytes()), values, t.nodeResolver) - if err != nil { - return fmt.Errorf("DeleteAccount (%x) error: %v", addr, err) - } - default: - return errInvalidRootType - } return nil } From 61d4eec67fbb421170edbd18a7d69eb8ebcda81c Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:28:47 +0200 Subject: [PATCH 09/10] fix some of the tests --- core/state_processor_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index ea04e7c30ec7..9108d784e124 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -551,6 +551,7 @@ func TestProcessVerkleFillThenNoFill(t *testing.T) { Ethash: new(params.EthashConfig), ShanghaiTime: u64(0), PragueTime: u64(0), + VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, TerminalTotalDifficultyPassed: true, } @@ -581,7 +582,9 @@ func TestProcessVerkleFillThenNoFill(t *testing.T) { // Verkle trees use the snapshot, which must be enabled before the // data is saved into the tree+database. - blockchain, _ := NewBlockChain(bcdb, nil, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{Tracer: logger.NewJSONLogger(loggerCfg, traceFile)}, nil, nil) + cacheConfig := DefaultCacheConfigWithScheme("path") + cacheConfig.SnapshotLimit = 0 + blockchain, _ := NewBlockChain(bcdb, cacheConfig, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{Tracer: logger.NewJSONLogger(loggerCfg, traceFile)}, nil, nil) defer blockchain.Stop() blockGasUsagesExpected := params.TxGas + 216 /* intrinsic gas */ + 3*vm.GasFastestStep + params.WitnessChunkReadCost + params.WitnessChunkWriteCost From 8e7bffa017f3935280784634f9ce2e9be12412ab Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:33:18 +0200 Subject: [PATCH 10/10] fix last issues in tests --- core/state/trie_prefetcher.go | 4 +++- core/state_processor_test.go | 2 +- core/vm/operations_verkle.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 491b3807c8d3..8ff898cea99e 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -376,7 +376,9 @@ func (sf *subfetcher) loop() { if len(task.key) == common.AddressLength { sf.trie.GetAccount(common.BytesToAddress(task.key)) } else { - sf.trie.GetStorage(sf.addr, task.key) + if sf.trie != nil { + sf.trie.GetStorage(sf.addr, task.key) + } } if task.read { sf.seenRead[key] = struct{}{} diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 9108d784e124..4c7ca6dff32a 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -619,6 +619,6 @@ func TestProcessVerkleFillThenNoFill(t *testing.T) { t.Fatalf("block #%d not found at expected height", b.NumberU64()) } if b.GasUsed() != 2*blockGasUsagesExpected+params.WitnessChunkFillCost { - t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), 2*blockGasUsagesExpected, b.GasUsed()) + t.Fatalf("expected block #%d txs to use %d, got %d\n", b.NumberU64(), 2*blockGasUsagesExpected+params.WitnessChunkFillCost, b.GasUsed()) } } diff --git a/core/vm/operations_verkle.go b/core/vm/operations_verkle.go index 5a0ef73a7117..6b08703af3ea 100644 --- a/core/vm/operations_verkle.go +++ b/core/vm/operations_verkle.go @@ -27,7 +27,7 @@ func gasSStore4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memo addr = contract.Address() slot = common.Hash(stack.peek().Bytes32()) ) - gas := evm.AccessEvents.SlotGas(addr, slot, true, evm.StateDB.IsSlotFilled(addr, slot)) + gas := evm.AccessEvents.SlotGas(addr, slot, true, !evm.StateDB.IsSlotFilled(addr, slot)) if gas == 0 { gas = params.WarmStorageReadCostEIP2929 }