From 5b74de52ef1986244bae0f982bb751c70293599a Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 16 Nov 2021 14:27:57 +0800 Subject: [PATCH 01/32] Support customization consensus engine --- eth/backend.go | 2 +- eth/ethconfig/config.go | 7 ++++++- les/client.go | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 793d3b81f1b1..65e4b3f01473 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -145,7 +145,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), + engine: config.ConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 89cdb75597e0..ad988ca0ca13 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -91,6 +91,7 @@ var Defaults = Config{ RPCGasCap: 50000000, GPO: FullNodeGPO, RPCTxFeeCap: 1, // 1 ether + ConsensusEngine:CreateDefaultConsensusEngine, } func init() { @@ -114,6 +115,8 @@ func init() { } } +type CreateConsensusEngine func (*node.Node, *params.ChainConfig, *ethash.Config, []string, bool, ethdb.Database) consensus.Engine + //go:generate gencodec -type Config -formats toml -out gen_config.go // Config contains configuration options for of the ETH and LES protocols. @@ -200,10 +203,12 @@ type Config struct { // Berlin block override (TODO: remove after the fork) OverrideLondon *big.Int `toml:",omitempty"` + + ConsensusEngine CreateConsensusEngine } // CreateConsensusEngine creates a consensus engine for the given chain configuration. -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { +func CreateDefaultConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up if chainConfig.Clique != nil { return clique.New(chainConfig.Clique, db) diff --git a/les/client.go b/les/client.go index 5d07c783e99d..df6ed9aa5904 100644 --- a/les/client.go +++ b/les/client.go @@ -109,7 +109,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { eventMux: stack.EventMux(), reqDist: newRequestDistributor(peers, &mclock.System{}), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), + engine: config.ConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), p2pServer: stack.Server(), From 37caa547780a07ed84d970033d4962950f86de16 Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Sun, 12 Dec 2021 12:31:14 +0800 Subject: [PATCH 02/32] go.mod : fix go-duktape warning --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5f4330af3ea9..58e17940c11f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea github.com/deepmap/oapi-codegen v1.8.2 // indirect + github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 @@ -70,7 +71,6 @@ require ( golang.org/x/text v0.3.6 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce - gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools v2.2.0+incompatible // indirect From c2e721e02fac407def52e594a5fc1d32c41188bf Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Sun, 12 Dec 2021 17:25:20 +0800 Subject: [PATCH 03/32] eth/tracers: fix go-duktape warning --- eth/tracers/tracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index fdeffb8705da..10f9abdedd8a 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "gopkg.in/olebedev/go-duktape.v3" + "github.com/dindinw/go-duktape" ) // bigIntegerJS is the minified version of https://github.com/peterolson/BigInteger.js. From 13b32c05a220491515180b3b28661b87470375cd Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Sun, 12 Dec 2021 17:31:30 +0800 Subject: [PATCH 04/32] go.sum done go.mod tidy --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 0890b8c5e04c..0485733b2f3b 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 h1:zqe534a6faNWAHtQQw9Dz0evsn/3P9GTphvxGukL8aQ= +github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288/go.mod h1:2tBqBHv6UMUEDS9cXrAOJv6A+okIYUNoc910JjZvf3Q= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= @@ -634,8 +636,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= From 5a818d0bc663e8064cc06e8b76346620b68c484d Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 4 Jan 2022 20:54:46 +0800 Subject: [PATCH 05/32] Support RemoveTx for pool --- core/tx_pool.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/tx_pool.go b/core/tx_pool.go index 5c70cf170578..5b569d40e70c 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -986,6 +986,10 @@ func (pool *TxPool) Has(hash common.Hash) bool { return pool.all.Get(hash) != nil } +func (pool *TxPool) RemoveTx(hash common.Hash, outofbound bool) { + pool.removeTx(hash,outofbound) +} + // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { From ca2e57390fb19ae224d86415386902f4cf7550e8 Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 10 Feb 2022 12:15:05 +0800 Subject: [PATCH 06/32] Expose web3 interface --- expose.go | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 expose.go diff --git a/expose.go b/expose.go new file mode 100644 index 000000000000..9f8e2fe8294b --- /dev/null +++ b/expose.go @@ -0,0 +1,5 @@ +package ethereum + +import "github.com/ethereum/go-ethereum/internal/web3ext" + +var Modules = web3ext.Modules From 29cefce14256e92977bdb1074ec6fa37116edc2d Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 10 Feb 2022 12:18:12 +0800 Subject: [PATCH 07/32] optimize c --- expose.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/expose.go b/expose.go index 9f8e2fe8294b..f63ffafaae92 100644 --- a/expose.go +++ b/expose.go @@ -1,3 +1,5 @@ +// Copyright (c) 2017-2022 The qitmeer developers + package ethereum import "github.com/ethereum/go-ethereum/internal/web3ext" From f092f204e8b120dad3abe6e502599585762df539 Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 17 Feb 2022 10:37:49 +0800 Subject: [PATCH 08/32] Delete dependency about docker --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 58e17940c11f..f6f62ec68923 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 github.com/dlclark/regexp2 v1.2.0 // indirect - github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 github.com/edsrzf/mmap-go v1.0.0 github.com/fatih/color v1.7.0 diff --git a/go.sum b/go.sum index 0485733b2f3b..142fbfac73a1 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,6 @@ github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 h1:zqe534a6faNW github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288/go.mod h1:2tBqBHv6UMUEDS9cXrAOJv6A+okIYUNoc910JjZvf3Q= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= From 07a4b97a266ffd95564adf9127b915e7c722fa5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 8 Mar 2022 12:00:29 +0200 Subject: [PATCH 09/32] core/statedb: always clear out access list when setting a new one --- core/state/statedb.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index e3541339eaa5..cf46e1f174c6 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -885,7 +885,6 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { func (s *StateDB) Prepare(thash common.Hash, ti int) { s.thash = thash s.txIndex = ti - s.accessList = newAccessList() } func (s *StateDB) clearJournalAndRefund() { @@ -995,6 +994,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // // This method should only be called if Berlin/2929+2930 is applicable at the current number. func (s *StateDB) PrepareAccessList(sender common.Address, dst *common.Address, precompiles []common.Address, list types.AccessList) { + // Clear out any leftover from previous executions + s.accessList = newAccessList() + s.AddAddressToAccessList(sender) if dst != nil { s.AddAddressToAccessList(*dst) From ba548988e4e37965e7659e475d03ed2b2384207f Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 22 Mar 2022 10:42:46 +0800 Subject: [PATCH 10/32] BUG:tx pool multi thread lock conflict --- core/tx_pool.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/tx_pool.go b/core/tx_pool.go index 5b569d40e70c..29c880859a6b 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -987,6 +987,8 @@ func (pool *TxPool) Has(hash common.Hash) bool { } func (pool *TxPool) RemoveTx(hash common.Hash, outofbound bool) { + pool.mu.Lock() + defer pool.mu.Unlock() pool.removeTx(hash,outofbound) } From 3f851b29304a225547d9b96c0c4c5f26b005004a Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 22 Mar 2022 20:08:21 +0800 Subject: [PATCH 11/32] Support external proxy for miner --- eth/api_backend.go | 2 +- eth/backend.go | 11 ++++++++--- miner/interface.go | 27 +++++++++++++++++++++++++++ miner/miner.go | 1 + 4 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 miner/interface.go diff --git a/eth/api_backend.go b/eth/api_backend.go index 1af33414cda4..9aa2ab0b9308 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -339,7 +339,7 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *EthAPIBackend) Miner() *miner.Miner { +func (b *EthAPIBackend) Miner() miner.IMiner { return b.eth.Miner() } diff --git a/eth/backend.go b/eth/backend.go index 65e4b3f01473..5d127c0f8a01 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -85,7 +85,7 @@ type Ethereum struct { APIBackend *EthAPIBackend - miner *miner.Miner + miner miner.IMiner gasPrice *big.Int etherbase common.Address @@ -225,7 +225,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) + if config.Miner.External == nil { + eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) + }else{ + eth.miner = config.Miner.External + } + eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -497,7 +502,7 @@ func (s *Ethereum) StopMining() { } func (s *Ethereum) IsMining() bool { return s.miner.Mining() } -func (s *Ethereum) Miner() *miner.Miner { return s.miner } +func (s *Ethereum) Miner() miner.IMiner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } diff --git a/miner/interface.go b/miner/interface.go new file mode 100644 index 000000000000..a29fc9c3715e --- /dev/null +++ b/miner/interface.go @@ -0,0 +1,27 @@ +package miner + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "time" +) + +type IMiner interface { + Start(coinbase common.Address) + Stop() + Close() + Mining() bool + Hashrate() uint64 + SetExtra(extra []byte) error + SetRecommitInterval(interval time.Duration) + Pending() (*types.Block, *state.StateDB) + PendingBlock() *types.Block + PendingBlockAndReceipts() (*types.Block, types.Receipts) + SetEtherbase(addr common.Address) + SetGasCeil(ceil uint64) + EnablePreseal() + DisablePreseal() + SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription +} diff --git a/miner/miner.go b/miner/miner.go index a4a01b9f4ff7..1a6c36c1edb6 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -51,6 +51,7 @@ type Config struct { GasPrice *big.Int // Minimum gas price for mining a transaction Recommit time.Duration // The time interval for miner to re-create mining work. Noverify bool // Disable remote mining solution verification(only useful in ethash). + External IMiner // External miner } // Miner creates blocks and searches for proof-of-work values. From e3fc7f6b629974f16855e898fd948b7c817d9d87 Mon Sep 17 00:00:00 2001 From: Jin Date: Fri, 10 Jun 2022 19:30:30 +0800 Subject: [PATCH 12/32] optimize miner interface for new version --- cmd/utils/flags.go | 2 +- miner/interface.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4ce16ef90031..839fafa53d8a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2018,7 +2018,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai if ctx.GlobalBool(FakePoWFlag.Name) { ethashConf.PowMode = ethash.ModeFake } - engine = ethconfig.CreateConsensusEngine(stack, config, ðashConf, nil, false, chainDb) + engine = ethconfig.CreateDefaultConsensusEngine(stack, config, ðashConf, nil, false, chainDb) if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } diff --git a/miner/interface.go b/miner/interface.go index a29fc9c3715e..e67ed4f66145 100644 --- a/miner/interface.go +++ b/miner/interface.go @@ -24,4 +24,6 @@ type IMiner interface { EnablePreseal() DisablePreseal() SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription + GetSealingBlockAsync(parent common.Hash, timestamp uint64, coinbase common.Address, random common.Hash, noTxs bool) (chan *types.Block, error) + GetSealingBlockSync(parent common.Hash, timestamp uint64, coinbase common.Address, random common.Hash, noTxs bool) (*types.Block, error) } From f20bb40c07b166106e8c251dadd0f1386815d5ca Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 18 Aug 2022 16:25:58 +0800 Subject: [PATCH 13/32] Remove fork info and optimize ethcatalyst --- cmd/utils/flags.go | 12 ++++++++---- eth/backend.go | 17 ++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f9584340b29a..0d971e8de1e0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1984,8 +1984,10 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - if err := lescatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the Engine API service: %v", err) + if backend.BlockChain().Config().TerminalTotalDifficulty != nil { + if err := lescatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) + } } return backend.ApiBackend, nil } @@ -1999,8 +2001,10 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } - if err := ethcatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the Engine API service: %v", err) + if backend.BlockChain().Config().TerminalTotalDifficulty != nil { + if err := ethcatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) + } } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend diff --git a/eth/backend.go b/eth/backend.go index 73589211a75b..cdcb3b19b0f5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,7 +22,6 @@ import ( "fmt" "math/big" "runtime" - "strings" "sync" "sync/atomic" "time" @@ -141,13 +140,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } - log.Info("") - log.Info(strings.Repeat("-", 153)) - for _, line := range strings.Split(chainConfig.String(), "\n") { - log.Info(line) - } - log.Info(strings.Repeat("-", 153)) - log.Info("") + //log.Info("") + //log.Info(strings.Repeat("-", 153)) + //for _, line := range strings.Split(chainConfig.String(), "\n") { + // log.Info(line) + //} + //log.Info(strings.Repeat("-", 153)) + //log.Info("") if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) @@ -243,7 +242,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.Miner.External == nil { eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) - }else{ + } else { eth.miner = config.Miner.External } From d0dc349fd36bd79f94516c866251783641ed12f1 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Wed, 31 Aug 2022 16:14:53 +0200 Subject: [PATCH 14/32] graphql: return correct logs for tx (#25612) * graphql: fix tx logs * minor * Use optimized search for selecting tx logs --- graphql/graphql.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 97b460c205ce..66c25841db98 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "sort" "strconv" "github.com/ethereum/go-ethereum" @@ -478,13 +479,16 @@ func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { if err != nil { return nil, err } - ret := make([]*Log, 0, len(logs)) - for _, log := range logs { + var ret []*Log + // Select tx logs from all block logs + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) == t.index }) + for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ r: t.r, transaction: t, - log: log, + log: logs[ix], }) + ix++ } return &ret, nil } From 3b41be695e6e12829abf6e5edec0606ee37f14d9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 13 Sep 2022 21:49:52 +0200 Subject: [PATCH 15/32] graphql: fixes missing tx logs (#25745) * graphql: fix tx logs * graphql: refactor test service setup * graphql: add test for tx logs --- graphql/graphql.go | 2 +- graphql/graphql_test.go | 227 +++++++++++++++++++++------------------- graphql/service.go | 9 +- 3 files changed, 124 insertions(+), 114 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 66c25841db98..356ff669fb16 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -481,7 +481,7 @@ func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { } var ret []*Log // Select tx logs from all block logs - ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) == t.index }) + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) >= t.index }) for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ r: t.r, diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index d55f4e063486..491c73152113 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -17,6 +17,8 @@ package graphql import ( + "context" + "encoding/json" "fmt" "io" "math/big" @@ -51,15 +53,21 @@ func TestBuildSchema(t *testing.T) { } defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { + if _, err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint func TestGraphQLBlockSerialization(t *testing.T) { - stack := createNode(t, true, false) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + } + newGQLService(t, stack, genesis, 10, func(i int, gen *core.BlockGen) {}) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -161,8 +169,55 @@ func TestGraphQLBlockSerialization(t *testing.T) { } func TestGraphQLBlockSerializationEIP2718(t *testing.T) { - stack := createNode(t, true, true) + // Account for signing txes + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + dad = common.HexToAddress("0x0000000000000000000000000000000000000dad") + ) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + address: {Balance: funds}, + // The address 0xdad sloads 0x00 and 0x01 + dad: { + Code: []byte{byte(vm.PC), byte(vm.PC), byte(vm.SLOAD), byte(vm.SLOAD)}, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer := types.LatestSigner(genesis.Config) + newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(common.Address{1}) + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: uint64(0), + To: &dad, + Value: big.NewInt(100), + Gas: 50000, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(1), + To: &dad, + Gas: 30000, + GasPrice: big.NewInt(params.InitialBaseFee), + Value: big.NewInt(50), + AccessList: types.AccessList{{ + Address: dad, + StorageKeys: []common.Hash{{0}}, + }}, + }) + gen.AddTx(tx) + }) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -198,7 +253,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { - stack := createNode(t, false, false) + stack := createNode(t) defer stack.Close() if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -212,7 +267,59 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } -func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { +func TestGraphQLTransactionLogs(t *testing.T) { + var ( + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + dadStr = "0x0000000000000000000000000000000000000dad" + dad = common.HexToAddress(dadStr) + genesis = &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(params.Ether)}, + dad: { + // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) + Code: common.Hex2Bytes("60006000a060006000a060006000f3"), + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + signer = types.LatestSigner(genesis.Config) + stack = createNode(t) + ) + defer stack.Close() + + handler := newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 1, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 2, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + }) + // start node + if err := stack.Start(); err != nil { + t.Fatalf("could not start node: %v", err) + } + query := `{block { transactions { logs { account { address } } } } }` + res := handler.Schema.Exec(context.Background(), query, "", map[string]interface{}{}) + if res.Errors != nil { + t.Fatalf("graphql query failed: %v", res.Errors) + } + have, err := json.Marshal(res.Data) + if err != nil { + t.Fatalf("failed to encode graphql response: %s", err) + } + want := fmt.Sprintf(`{"block":{"transactions":[{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]}]}}`, dadStr, dadStr, dadStr, dadStr, dadStr, dadStr) + if string(have) != want { + t.Errorf("response unmatch. expected %s, got %s", want, have) + } +} + +func createNode(t *testing.T) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", HTTPPort: 0, @@ -222,83 +329,12 @@ func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { if err != nil { t.Fatalf("could not create node: %v", err) } - if !gqlEnabled { - return stack - } - if !txEnabled { - createGQLService(t, stack) - } else { - createGQLServiceWithTransactions(t, stack) - } return stack } -func createGQLService(t *testing.T, stack *node.Node) { - // create backend - ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - }, - Ethash: ethash.Config{ - PowMode: ethash.ModeFake, - }, - NetworkId: 1337, - TrieCleanCache: 5, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 5, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 5, - } - ethBackend, err := eth.New(stack, ethConf) - if err != nil { - t.Fatalf("could not create eth backend: %v", err) - } - // Create some blocks and import them - chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 10, func(i int, gen *core.BlockGen) {}) - _, err = ethBackend.BlockChain().InsertChain(chain) - if err != nil { - t.Fatalf("could not create import blocks: %v", err) - } - // create gql service - filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) - err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) - if err != nil { - t.Fatalf("could not create graphql service: %v", err) - } -} - -func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { - // create backend - key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address := crypto.PubkeyToAddress(key.PublicKey) - funds := big.NewInt(1000000000000000) - dad := common.HexToAddress("0x0000000000000000000000000000000000000dad") - +func newGQLService(t *testing.T, stack *node.Node, gspec *core.Genesis, genBlocks int, genfunc func(i int, gen *core.BlockGen)) *handler { ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ - address: {Balance: funds}, - // The address 0xdad sloads 0x00 and 0x01 - dad: { - Code: []byte{ - byte(vm.PC), - byte(vm.PC), - byte(vm.SLOAD), - byte(vm.SLOAD), - }, - Nonce: 0, - Balance: big.NewInt(0), - }, - }, - BaseFee: big.NewInt(params.InitialBaseFee), - }, + Genesis: gspec, Ethash: ethash.Config{ PowMode: ethash.ModeFake, }, @@ -310,49 +346,22 @@ func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { TrieTimeout: 60 * time.Minute, SnapshotCache: 5, } - ethBackend, err := eth.New(stack, ethConf) if err != nil { t.Fatalf("could not create eth backend: %v", err) } - signer := types.LatestSigner(ethConf.Genesis.Config) - - legacyTx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: uint64(0), - To: &dad, - Value: big.NewInt(100), - Gas: 50000, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - envelopTx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ - ChainID: ethConf.Genesis.Config.ChainID, - Nonce: uint64(1), - To: &dad, - Gas: 30000, - GasPrice: big.NewInt(params.InitialBaseFee), - Value: big.NewInt(50), - AccessList: types.AccessList{{ - Address: dad, - StorageKeys: []common.Hash{{0}}, - }}, - }) - // Create some blocks and import them chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 1, func(i int, b *core.BlockGen) { - b.SetCoinbase(common.Address{1}) - b.AddTx(legacyTx) - b.AddTx(envelopTx) - }) - + ethash.NewFaker(), ethBackend.ChainDb(), genBlocks, genfunc) _, err = ethBackend.BlockChain().InsertChain(chain) if err != nil { t.Fatalf("could not create import blocks: %v", err) } - // create gql service + // Set up handler filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) - err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) + handler, err := newHandler(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } + return handler } diff --git a/graphql/service.go b/graphql/service.go index 019026bc7ea7..6f6e58335991 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -57,17 +57,18 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // New constructs a new GraphQL service instance. func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { - return newHandler(stack, backend, filterSystem, cors, vhosts) + _, err := newHandler(stack, backend, filterSystem, cors, vhosts) + return err } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) (*handler, error) { q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { - return err + return nil, err } h := handler{Schema: s} handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) @@ -76,5 +77,5 @@ func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters. stack.RegisterHandler("GraphQL", "/graphql", handler) stack.RegisterHandler("GraphQL", "/graphql/", handler) - return nil + return &h, nil } From 972007a517c49ee9e2a359950d81c74467492ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Sep 2022 11:07:10 +0200 Subject: [PATCH 16/32] Release Geth v1.10.24 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 5f24b41f2852..349bdf1c18d1 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 23 // Patch version component of the current release + VersionPatch = 24 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From 8f61fc8b73565ff15d27fd9731b945d9d1878ec3 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 15 Sep 2022 17:50:54 +0200 Subject: [PATCH 17/32] params: set TerminalTotalDifficultyPassed to true (#25769) * params: set TerminalTotalDifficultyPassed to true * Update params/config.go Co-authored-by: Martin Holst Swende --- params/config.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/params/config.go b/params/config.go index 4d6eec8939bc..80e671f9bf41 100644 --- a/params/config.go +++ b/params/config.go @@ -59,25 +59,26 @@ var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - GrayGlacierBlock: big.NewInt(15_050_000), - TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. From 69568c554880b3567bace64f8848ff1be27d084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 15 Sep 2022 17:55:58 +0200 Subject: [PATCH 18/32] params: release Geth v1.10.25 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 349bdf1c18d1..3b5978e849cd 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 24 // Patch version component of the current release + VersionPatch = 25 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From cf51cba21b91e770f2ea29bf734f2a235ec987d6 Mon Sep 17 00:00:00 2001 From: Jin Date: Wed, 5 Oct 2022 21:39:14 +0800 Subject: [PATCH 19/32] usbwallet support Ledger Nano S Plus --- accounts/usbwallet/hub.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index 23be98a08483..0705bad1eb01 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -75,14 +75,17 @@ func NewLedgerHub() (*Hub, error) { 0x0000, /* Ledger Blue */ 0x0001, /* Ledger Nano S */ 0x0004, /* Ledger Nano X */ + 0x0005, /* Ledger Nano S Plus */ // Upcoming product IDs: https://www.ledger.com/2019/05/17/windows-10-update-sunsetting-u2f-tunnel-transport-for-ledger-devices/ 0x0015, /* HID + U2F + WebUSB Ledger Blue */ 0x1015, /* HID + U2F + WebUSB Ledger Nano S */ 0x4015, /* HID + U2F + WebUSB Ledger Nano X */ + 0x5015, /* HID + U2F + WebUSB Ledger Nano S Plus */ 0x0011, /* HID + WebUSB Ledger Blue */ 0x1011, /* HID + WebUSB Ledger Nano S */ 0x4011, /* HID + WebUSB Ledger Nano X */ + 0x5011, /* HID + WebUSB Ledger Nano S Plus */ }, 0xffa0, 0, newLedgerDriver) } From 85e469f787be53192fcbcee86ce67635020a4ad5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 31 Aug 2022 17:58:18 +0200 Subject: [PATCH 20/32] eth/protocols/snap: fix problems due to idle-but-busy peers (#25651) --- eth/protocols/snap/sync.go | 90 +++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index deaa4456e0fd..1455bacbcb88 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2248,14 +2248,18 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.accountIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.accountIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.accountReqs[id] if !ok { @@ -2360,14 +2364,18 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeReqs[id] if !ok { @@ -2469,14 +2477,18 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.storageIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.storageIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.storageReqs[id] if !ok { @@ -2596,14 +2608,18 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.trienodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.trienodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.trienodeHealReqs[id] if !ok { @@ -2691,14 +2707,18 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeHealReqs[id] if !ok { From 937ea491f95e5676d2ce3e0364ba9f05452bc351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 9 Sep 2022 11:42:57 +0300 Subject: [PATCH 21/32] eth/protocols/snap: throttle trie heal requests when peers DoS us (#25666) * eth/protocols/snap: throttle trie heal requests when peers DoS us * eth/protocols/snap: lower heal throttle log to debug Co-authored-by: Martin Holst Swende * eth/protocols/snap: fix comment Co-authored-by: Martin Holst Swende --- eth/protocols/snap/sync.go | 107 ++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 1455bacbcb88..eb8260bf7c97 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -21,10 +21,12 @@ import ( "encoding/json" "errors" "fmt" + gomath "math" "math/big" "math/rand" "sort" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -78,6 +80,29 @@ const ( // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. maxTrieRequestCount = maxRequestSize / 512 + + // trienodeHealRateMeasurementImpact is the impact a single measurement has on + // the local node's trienode processing capacity. A value closer to 0 reacts + // slower to sudden changes, but it is also more stable against temporary hiccups. + trienodeHealRateMeasurementImpact = 0.005 + + // minTrienodeHealThrottle is the minimum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + minTrienodeHealThrottle = 1 + + // maxTrienodeHealThrottle is the maximum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + maxTrienodeHealThrottle = maxTrieRequestCount + + // trienodeHealThrottleIncrease is the multiplier for the throttle when the + // rate of arriving data is higher than the rate of processing it. + trienodeHealThrottleIncrease = 1.33 + + // trienodeHealThrottleDecrease is the divisor for the throttle when the + // rate of arriving data is lower than the rate of processing it. + trienodeHealThrottleDecrease = 1.25 ) var ( @@ -431,6 +456,11 @@ type Syncer struct { trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running + trienodeHealRate float64 // Average heal rate for processing trie node data + trienodeHealPend uint64 // Number of trie nodes currently pending for processing + trienodeHealThrottle float64 // Divisor for throttling the amount of trienode heal data requested + trienodeHealThrottled time.Time // Timestamp the last time the throttle was updated + trienodeHealSynced uint64 // Number of state trie nodes downloaded trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk trienodeHealDups uint64 // Number of state trie nodes already processed @@ -476,9 +506,10 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { trienodeHealIdlers: make(map[string]struct{}), bytecodeHealIdlers: make(map[string]struct{}), - trienodeHealReqs: make(map[uint64]*trienodeHealRequest), - bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), - stateWriter: db.NewBatch(), + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + trienodeHealThrottle: maxTrienodeHealThrottle, // Tune downward instead of insta-filling with junk + stateWriter: db.NewBatch(), extProgress: new(SyncProgress), } @@ -1321,6 +1352,10 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai if cap > maxTrieRequestCount { cap = maxTrieRequestCount } + cap = int(float64(cap) / s.trienodeHealThrottle) + if cap <= 0 { + cap = 1 + } var ( hashes = make([]common.Hash, 0, cap) paths = make([]string, 0, cap) @@ -2090,6 +2125,10 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // processTrienodeHealResponse integrates an already validated trienode response // into the healer tasks. func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { + var ( + start = time.Now() + fills int + ) for i, hash := range res.hashes { node := res.nodes[i] @@ -2098,6 +2137,8 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { res.task.trieTasks[res.paths[i]] = res.hashes[i] continue } + fills++ + // Push the trie node into the state syncer s.trienodeHealSynced++ s.trienodeHealBytes += common.StorageSize(len(node)) @@ -2121,6 +2162,50 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { log.Crit("Failed to persist healing data", "err", err) } log.Debug("Persisted set of healing data", "type", "trienodes", "bytes", common.StorageSize(batch.ValueSize())) + + // Calculate the processing rate of one filled trie node + rate := float64(fills) / (float64(time.Since(start)) / float64(time.Second)) + + // Update the currently measured trienode queueing and processing throughput. + // + // The processing rate needs to be updated uniformly independent if we've + // processed 1x100 trie nodes or 100x1 to keep the rate consistent even in + // the face of varying network packets. As such, we cannot just measure the + // time it took to process N trie nodes and update once, we need one update + // per trie node. + // + // Naively, that would be: + // + // for i:=0; i time.Second { + // Periodically adjust the trie node throttler + if float64(pending) > 2*s.trienodeHealRate { + s.trienodeHealThrottle *= trienodeHealThrottleIncrease + } else { + s.trienodeHealThrottle /= trienodeHealThrottleDecrease + } + if s.trienodeHealThrottle > maxTrienodeHealThrottle { + s.trienodeHealThrottle = maxTrienodeHealThrottle + } else if s.trienodeHealThrottle < minTrienodeHealThrottle { + s.trienodeHealThrottle = minTrienodeHealThrottle + } + s.trienodeHealThrottled = time.Now() + + log.Debug("Updated trie node heal throttler", "rate", s.trienodeHealRate, "pending", pending, "throttle", s.trienodeHealThrottle) + } } // processBytecodeHealResponse integrates an already validated bytecode response @@ -2655,10 +2740,12 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Cross reference the requested trienodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) - hash := make([]byte, 32) - - nodes := make([][]byte, len(req.hashes)) + var ( + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + nodes = make([][]byte, len(req.hashes)) + fills uint64 + ) for i, j := 0, 0; i < len(trienodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() @@ -2670,16 +2757,22 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error } if j < len(req.hashes) { nodes[j] = trienodes[i] + fills++ j++ continue } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + // Signal this request as failed, and ready for rescheduling s.scheduleRevertTrienodeHealRequest(req) return errors.New("unexpected healing trienode") } // Response validated, send it to the scheduler for filling + atomic.AddUint64(&s.trienodeHealPend, fills) + defer func() { + atomic.AddUint64(&s.trienodeHealPend, ^(fills - 1)) + }() response := &trienodeHealResponse{ paths: req.paths, task: req.task, From a32e69a28c2abef6a6b96e8d2466f1f27fd693ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 6 Sep 2022 12:57:03 +0300 Subject: [PATCH 22/32] trie: check childrens' existence concurrently for snap heal (#25694) --- trie/sync.go | 57 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/trie/sync.go b/trie/sync.go index 303fcbfa22e2..862ce7e16e6c 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -19,6 +19,7 @@ package trie import ( "errors" "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" @@ -381,11 +382,11 @@ func (s *Sync) scheduleCodeRequest(req *codeRequest) { // retrieval scheduling. func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // Gather all the children of the node, irrelevant whether known or not - type child struct { + type childNode struct { path []byte node node } - var children []child + var children []childNode switch node := (object).(type) { case *shortNode: @@ -393,14 +394,14 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if hasTerm(key) { key = key[:len(key)-1] } - children = []child{{ + children = []childNode{{ node: node.Val, path: append(append([]byte(nil), req.path...), key...), }} case *fullNode: for i := 0; i < 17; i++ { if node.Children[i] != nil { - children = append(children, child{ + children = append(children, childNode{ node: node.Children[i], path: append(append([]byte(nil), req.path...), byte(i)), }) @@ -410,7 +411,10 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { panic(fmt.Sprintf("unknown node: %+v", node)) } // Iterate over the children, and request all unknown ones - requests := make([]*nodeRequest, 0, len(children)) + var ( + missing = make(chan *nodeRequest, len(children)) + pending sync.WaitGroup + ) for _, child := range children { // Notify any external watcher of a new key/value node if req.callback != nil { @@ -433,19 +437,36 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if s.membatch.hasNode(child.path) { continue } - // If database says duplicate, then at least the trie node is present - // and we hold the assumption that it's NOT legacy contract code. - chash := common.BytesToHash(node) - if rawdb.HasTrieNode(s.database, chash) { - continue - } - // Locally unknown node, schedule for retrieval - requests = append(requests, &nodeRequest{ - path: child.path, - hash: chash, - parent: req, - callback: req.callback, - }) + // Check the presence of children concurrently + pending.Add(1) + go func(child childNode) { + defer pending.Done() + + // If database says duplicate, then at least the trie node is present + // and we hold the assumption that it's NOT legacy contract code. + chash := common.BytesToHash(node) + if rawdb.HasTrieNode(s.database, chash) { + return + } + // Locally unknown node, schedule for retrieval + missing <- &nodeRequest{ + path: child.path, + hash: chash, + parent: req, + callback: req.callback, + } + }(child) + } + } + pending.Wait() + + requests := make([]*nodeRequest, 0, len(children)) + for done := false; !done; { + select { + case miss := <-missing: + requests = append(requests, miss) + default: + done = true } } return requests, nil From 99bbb337011ef614fc5d25c130aa1cef92915366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 20 Sep 2022 14:14:24 +0300 Subject: [PATCH 23/32] eth: fix a rare datarace on CHT challenge reply / shutdown (#25831) --- eth/handler.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 4224a9f33a84..143147b0c815 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -391,11 +391,16 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { if h.checkpointHash != (common.Hash{}) { // Request the peer's checkpoint header for chain height/weight validation resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh) + if err != nil { return err } // Start a timer to disconnect if the peer doesn't reply in time go func() { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -437,10 +442,15 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If we have any explicit peer required block hashes, request them for number, hash := range h.requiredBlocks { resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh) + if err != nil { return err } - go func(number uint64, hash common.Hash) { + go func(number uint64, hash common.Hash, req *eth.Request) { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -469,7 +479,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Warn("Required block challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) h.removePeer(peer.ID()) } - }(number, hash) + }(number, hash, req) } // Handle incoming messages until the connection is torn down return handler(peer) From 27600a5b8452c39ce9ef5c502c98a0c1ee35859b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 27 Oct 2022 15:25:01 +0200 Subject: [PATCH 24/32] eth/filters: change filter block to be by-ref (#26054) This PR changes the block field in the filter to be a pointer, to disambiguate between empty hash and no hash --- eth/filters/filter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 0a70c9ece1db..8d80647fc241 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -34,8 +34,8 @@ type Filter struct { addresses []common.Address topics [][]common.Hash - block common.Hash // Block hash if filtering a single block - begin, end int64 // Range interval if filtering multiple blocks + block *common.Hash // Block hash if filtering a single block + begin, end int64 // Range interval if filtering multiple blocks matcher *bloombits.Matcher } @@ -78,7 +78,7 @@ func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Add func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter filter := newFilter(sys, addresses, topics) - filter.block = block + filter.block = &block return filter } @@ -96,8 +96,8 @@ func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common. // first block that contains matches, updating the start of the filter accordingly. func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return - if f.block != (common.Hash{}) { - header, err := f.sys.backend.HeaderByHash(ctx, f.block) + if f.block != nil { + header, err := f.sys.backend.HeaderByHash(ctx, *f.block) if err != nil { return nil, err } From 211dbb719711420358a3cdf718ce476c212adcf5 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Wed, 2 Nov 2022 09:29:33 -0500 Subject: [PATCH 25/32] rpc: handle wrong HTTP batch response length (#26064) --- rpc/client.go | 1 + rpc/client_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++ rpc/http.go | 3 +++ 3 files changed, 52 insertions(+) diff --git a/rpc/client.go b/rpc/client.go index d3ce0297754c..fcd31319a3c5 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -31,6 +31,7 @@ import ( ) var ( + ErrBadResult = errors.New("bad result in JSON-RPC response") ErrClientQuit = errors.New("client is closed") ErrNoResult = errors.New("no result in JSON-RPC response") ErrSubscriptionQueueOverflow = errors.New("subscription queue overflow") diff --git a/rpc/client_test.go b/rpc/client_test.go index 04c847d0d626..5ae549d86533 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math/rand" "net" @@ -144,6 +145,53 @@ func TestClientBatchRequest(t *testing.T) { } } +func TestClientBatchRequest_len(t *testing.T) { + b, err := json.Marshal([]jsonrpcMessage{ + {Version: "2.0", ID: json.RawMessage("1"), Method: "foo", Result: json.RawMessage(`"0x1"`)}, + {Version: "2.0", ID: json.RawMessage("2"), Method: "bar", Result: json.RawMessage(`"0x2"`)}, + }) + if err != nil { + t.Fatal("failed to encode jsonrpc message:", err) + } + s := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + _, err := rw.Write(b) + if err != nil { + t.Error("failed to write response:", err) + } + })) + t.Cleanup(s.Close) + + client, err := Dial(s.URL) + if err != nil { + t.Fatal("failed to dial test server:", err) + } + defer client.Close() + + t.Run("too-few", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + {Method: "bar"}, + {Method: "baz"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) + + t.Run("too-many", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) +} + func TestClientNotify(t *testing.T) { server := newTestServer() defer server.Stop() diff --git a/rpc/http.go b/rpc/http.go index 858d80858652..b82ea2707a54 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -173,6 +173,9 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { return err } + if len(respmsgs) != len(msgs) { + return fmt.Errorf("batch has %d requests but response has %d: %w", len(msgs), len(respmsgs), ErrBadResult) + } for i := 0; i < len(respmsgs); i++ { op.resp <- &respmsgs[i] } From e5eb32acee19cc9fca6a03b10283b7484246b15a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 3 Nov 2022 11:59:33 +0100 Subject: [PATCH 26/32] params: release geth v1.10.26 stable --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 3b5978e849cd..d9a3958512c3 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 25 // Patch version component of the current release + VersionPatch = 26 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From 76f1bb8b834198892dee8fa62d5d90b25ddb8e78 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 9 Nov 2022 08:06:02 +0100 Subject: [PATCH 27/32] core/state: replace fastcache code cache with gc-friendly structure (#26092) This PR replaces fastcache with a pretty simple LRU which does not require explicit closing. (cherry picked from commit 5fded040372784985265f83f33f15cb6a51bebdb) --- common/lru/blob_lru.go | 88 ++++++++++++++++++++++++++ common/lru/blob_lru_test.go | 122 ++++++++++++++++++++++++++++++++++++ core/state/database.go | 14 ++--- 3 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 common/lru/blob_lru.go create mode 100644 common/lru/blob_lru_test.go diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go new file mode 100644 index 000000000000..3138f422ccfa --- /dev/null +++ b/common/lru/blob_lru.go @@ -0,0 +1,88 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "math" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/hashicorp/golang-lru/simplelru" +) + +// SizeConstrainedLRU is a wrapper around simplelru.LRU. The simplelru.LRU is capable +// of item-count constraints, but is not capable of enforcing a byte-size constraint, +// hence this wrapper. +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +type SizeConstrainedLRU struct { + size uint64 + maxSize uint64 + lru *simplelru.LRU + lock sync.RWMutex +} + +// NewSizeConstrainedLRU creates a new SizeConstrainedLRU. +func NewSizeConstrainedLRU(max uint64) *SizeConstrainedLRU { + lru, err := simplelru.NewLRU(math.MaxInt, nil) + if err != nil { + panic(err) + } + return &SizeConstrainedLRU{ + size: 0, + maxSize: max, + lru: lru, + } +} + +// Add adds a value to the cache. Returns true if an eviction occurred. +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +// OBS: The value is _not_ copied on Add, so the caller must not modify it afterwards. +func (c *SizeConstrainedLRU) Add(key common.Hash, value []byte) (evicted bool) { + c.lock.Lock() + defer c.lock.Unlock() + + // Unless it is already present, might need to evict something. + // OBS: If it is present, we still call Add internally to bump the recentness. + if !c.lru.Contains(key) { + targetSize := c.size + uint64(len(value)) + for targetSize > c.maxSize { + evicted = true + _, v, ok := c.lru.RemoveOldest() + if !ok { + // list is now empty. Break + break + } + targetSize -= uint64(len(v.([]byte))) + } + c.size = targetSize + } + c.lru.Add(key, value) + return evicted +} + +// Get looks up a key's value from the cache. +func (c *SizeConstrainedLRU) Get(key common.Hash) []byte { + c.lock.RLock() + defer c.lock.RUnlock() + + if v, ok := c.lru.Get(key); ok { + return v.([]byte) + } + return nil +} diff --git a/common/lru/blob_lru_test.go b/common/lru/blob_lru_test.go new file mode 100644 index 000000000000..4b5e69340222 --- /dev/null +++ b/common/lru/blob_lru_test.go @@ -0,0 +1,122 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "encoding/binary" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func mkHash(i int) common.Hash { + h := make([]byte, 32) + binary.LittleEndian.PutUint64(h, uint64(i)) + return common.BytesToHash(h) +} + +func TestBlobLru(t *testing.T) { + lru := NewSizeConstrainedLRU(100) + var want uint64 + // Add 11 items of 10 byte each. First item should be swapped out + for i := 0; i < 11; i++ { + k := mkHash(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + want += uint64(len(v)) + if want > 100 { + want = 100 + } + if have := lru.size; have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } + // Zero:th should be evicted + { + k := mkHash(0) + if val := lru.Get(k); val != nil { + t.Fatalf("should be evicted: %v", k) + } + } + // Elems 1-11 should be present + for i := 1; i < 11; i++ { + k := mkHash(i) + want := fmt.Sprintf("value-%04d", i) + have := lru.Get(k) + if have == nil { + t.Fatalf("missing key %v", k) + } + if string(have) != want { + t.Fatalf("wrong value, have %v want %v", have, want) + } + } +} + +// TestBlobLruOverflow tests what happens when inserting an element exceeding +// the max size +func TestBlobLruOverflow(t *testing.T) { + lru := NewSizeConstrainedLRU(100) + // Add 10 items of 10 byte each, filling the cache + for i := 0; i < 10; i++ { + k := mkHash(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + } + // Add one single large elem. We expect it to swap out all entries. + { + k := mkHash(1337) + v := make([]byte, 200) + lru.Add(k, v) + } + // Elems 0-9 should be missing + for i := 1; i < 10; i++ { + k := mkHash(i) + if val := lru.Get(k); val != nil { + t.Fatalf("should be evicted: %v", k) + } + } + // The size should be accurate + if have, want := lru.size, uint64(200); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + // Adding one small item should swap out the large one + { + i := 0 + k := mkHash(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } +} + +// TestBlobLruSameItem tests what happens when inserting the same k/v multiple times. +func TestBlobLruSameItem(t *testing.T) { + lru := NewSizeConstrainedLRU(100) + // Add one 10 byte-item 10 times + k := mkHash(0) + v := fmt.Sprintf("value-%04d", 0) + for i := 0; i < 10; i++ { + lru.Add(k, []byte(v)) + } + // The size should be accurate + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } +} diff --git a/core/state/database.go b/core/state/database.go index 96b6bcfe6551..28de45261db4 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" - "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" + lru2 "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -131,14 +131,14 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, - codeCache: fastcache.New(codeCacheSize), + codeCache: lru2.NewSizeConstrainedLRU(codeCacheSize), } } type cachingDB struct { db *trie.Database codeSizeCache *lru.Cache - codeCache *fastcache.Cache + codeCache *lru2.SizeConstrainedLRU } // OpenTrie opens the main account trie at a specific root hash. @@ -171,12 +171,12 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { // ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { + if code := db.codeCache.Get(codeHash); len(code) > 0 { return code, nil } code := rawdb.ReadCode(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Set(codeHash.Bytes(), code) + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } @@ -187,12 +187,12 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error // code can't be found in the cache, then check the existence with **new** // db scheme. func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { - if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { + if code := db.codeCache.Get(codeHash); len(code) > 0 { return code, nil } code := rawdb.ReadCodeWithPrefix(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Set(codeHash.Bytes(), code) + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } From 354b6a797acbab480844f2a6d948dd52a8854b51 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 11 Nov 2022 19:48:36 +0100 Subject: [PATCH 28/32] common/lru: fix race in lru (#26164) This fixes a problem in the SizeConstrainedLRU. The SCLRU uses an underlying simple lru which is not thread safe. During the Get operation, the recentness of the accessed item is updated, so it is not a pure read-operation. Therefore, the mutex we need is a full mutex, not RLock. This PR changes the mutex to be a regular Mutex, instead of RWMutex, so a reviewer can at a glance see that all affected locations are fixed. (cherry picked from commit 8334b5f51a16b72cf2b3ebdc9e131a442c5289d7) --- common/lru/blob_lru.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go index 3138f422ccfa..b24684256c88 100644 --- a/common/lru/blob_lru.go +++ b/common/lru/blob_lru.go @@ -33,7 +33,7 @@ type SizeConstrainedLRU struct { size uint64 maxSize uint64 lru *simplelru.LRU - lock sync.RWMutex + lock sync.Mutex } // NewSizeConstrainedLRU creates a new SizeConstrainedLRU. @@ -78,8 +78,8 @@ func (c *SizeConstrainedLRU) Add(key common.Hash, value []byte) (evicted bool) { // Get looks up a key's value from the cache. func (c *SizeConstrainedLRU) Get(key common.Hash) []byte { - c.lock.RLock() - defer c.lock.RUnlock() + c.lock.Lock() + defer c.lock.Unlock() if v, ok := c.lru.Get(key); ok { return v.([]byte) From 283fb4ec13f7a140311c1bf77c7b7406dd2436de Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 25 Nov 2022 16:10:31 +0800 Subject: [PATCH 29/32] core/rawdb: fix freezer validation (#26251) * core/rawdb: fix freezer validation * core/rawdb: address comment (cherry picked from commit add1bff13fff722acbf1bd06b57245361ff82359) --- core/rawdb/freezer.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 6dea98c3d3c4..03093e7b1dd7 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -318,30 +318,35 @@ func (f *Freezer) Sync() error { return nil } -// validate checks that every table has the same length. +// validate checks that every table has the same boundary. // Used instead of `repair` in readonly mode. func (f *Freezer) validate() error { if len(f.tables) == 0 { return nil } var ( - length uint64 - name string + head uint64 + tail uint64 + name string ) - // Hack to get length of any table + // Hack to get boundary of any table for kind, table := range f.tables { - length = atomic.LoadUint64(&table.items) + head = atomic.LoadUint64(&table.items) + tail = atomic.LoadUint64(&table.itemHidden) name = kind break } - // Now check every table against that length + // Now check every table against those boundaries. for kind, table := range f.tables { - items := atomic.LoadUint64(&table.items) - if length != items { - return fmt.Errorf("freezer tables %s and %s have differing lengths: %d != %d", kind, name, items, length) + if head != atomic.LoadUint64(&table.items) { + return fmt.Errorf("freezer tables %s and %s have differing head: %d != %d", kind, name, atomic.LoadUint64(&table.items), head) + } + if tail != atomic.LoadUint64(&table.itemHidden) { + return fmt.Errorf("freezer tables %s and %s have differing tail: %d != %d", kind, name, atomic.LoadUint64(&table.itemHidden), tail) } } - atomic.StoreUint64(&f.frozen, length) + atomic.StoreUint64(&f.frozen, head) + atomic.StoreUint64(&f.tail, tail) return nil } From 31ed80298dc683ae8e11eba0af9961165cc05890 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 24 Nov 2022 10:50:28 +0100 Subject: [PATCH 30/32] core/rawdb: improve freezerTable.Sync (#26245) While investigating #22374, I noticed that the Sync operation of the freezer does not take the table lock. It also doesn't call sync for all files if there is an error with one of them. I doubt this will fix anything, but didn't want to drop the fix on the floor either. (cherry picked from commit 193f350eb911c9d8a93577d619986e97f490b700) --- core/rawdb/freezer_table.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 3fe691cf6d2a..e9998fa463c6 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -875,13 +875,20 @@ func (t *freezerTable) advanceHead() error { // Sync pushes any pending data from memory out to disk. This is an expensive // operation, so use it with care. func (t *freezerTable) Sync() error { - if err := t.index.Sync(); err != nil { - return err - } - if err := t.meta.Sync(); err != nil { - return err + t.lock.Lock() + defer t.lock.Unlock() + + var err error + trackError := func(e error) { + if e != nil && err == nil { + err = e + } } - return t.head.Sync() + + trackError(t.index.Sync()) + trackError(t.meta.Sync()) + trackError(t.head.Sync()) + return err } func (t *freezerTable) dumpIndexStdout(start, stop int64) { From 3d579f58f8a3a27224f290616806b1d9d52133a2 Mon Sep 17 00:00:00 2001 From: Jin Date: Mon, 9 Jan 2023 10:07:03 +0800 Subject: [PATCH 31/32] expose GlobalBig --- expose.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/expose.go b/expose.go index f63ffafaae92..e369faaa7f31 100644 --- a/expose.go +++ b/expose.go @@ -2,6 +2,15 @@ package ethereum -import "github.com/ethereum/go-ethereum/internal/web3ext" +import ( + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/web3ext" + "github.com/urfave/cli/v2" + "math/big" +) var Modules = web3ext.Modules + +func GlobalBig(ctx *cli.Context, name string) *big.Int { + return flags.GlobalBig(ctx,name) +} \ No newline at end of file From a7e8e2c89ae034628516a5a1a93557924a5c9f1f Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 28 Mar 2023 21:58:59 +0800 Subject: [PATCH 32/32] MeerChainConfig --- params/config.go | 12 +++++ params/qng_config.go | 123 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 params/qng_config.go diff --git a/params/config.go b/params/config.go index adb5f44f0952..b9d116810eec 100644 --- a/params/config.go +++ b/params/config.go @@ -497,6 +497,18 @@ func (c *ChainConfig) Description() string { } else { banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" } + case IsQngNetwork(c.ChainID): + banner += "Consensus: MeerDAG (proof-of-work)\n" + return banner + case IsAmanaNetwork(c.ChainID): + banner += "Consensus: Amana (proof-of-authority)\n" + return banner + case IsFlanaNetwork(c.ChainID): + banner += "Consensus: Flana (rollup)\n" + return banner + case IsMizanaNetwork(c.ChainID): + banner += "Consensus: Mizana (ZK rollup)\n" + return banner default: banner += "Consensus: unknown\n" } diff --git a/params/qng_config.go b/params/qng_config.go new file mode 100644 index 000000000000..2de3b0a8bd31 --- /dev/null +++ b/params/qng_config.go @@ -0,0 +1,123 @@ +package params + +import "math/big" + +type MeerChainConfig struct { + ChainID *big.Int // chainId identifies the current chain and is used for replay protection +} + +var ( + QngMainnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(813), + } + QngTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8131), + } + QngMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8132), + } + QngPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8133), + } + + AmanaChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8134), + } + AmanaTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81341), + } + AmanaMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81342), + } + AmanaPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81343), + } + + FlanaChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8135), + } + FlanaTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81351), + } + FlanaMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81352), + } + FlanaPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81353), + } + + MizanaChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8136), + } + MizanaTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81361), + } + MizanaMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81362), + } + MizanaPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81363), + } +) + +func init() { + NetworkNames[QngMainnetChainConfig.ChainID.String()] = "qng" + NetworkNames[QngTestnetChainConfig.ChainID.String()] = "qng-test" + NetworkNames[QngMixnetChainConfig.ChainID.String()] = "qng-mix" + NetworkNames[QngPrivnetChainConfig.ChainID.String()] = "qng-priv" + + NetworkNames[AmanaChainConfig.ChainID.String()] = "amana" + NetworkNames[AmanaTestnetChainConfig.ChainID.String()] = "amana-test" + NetworkNames[AmanaMixnetChainConfig.ChainID.String()] = "amana-mix" + NetworkNames[AmanaPrivnetChainConfig.ChainID.String()] = "amana-priv" + + NetworkNames[FlanaChainConfig.ChainID.String()] = "flana" + NetworkNames[FlanaTestnetChainConfig.ChainID.String()] = "flana-test" + NetworkNames[FlanaMixnetChainConfig.ChainID.String()] = "flana-mix" + NetworkNames[FlanaPrivnetChainConfig.ChainID.String()] = "flana-priv" + + NetworkNames[MizanaChainConfig.ChainID.String()] = "mizana" + NetworkNames[MizanaTestnetChainConfig.ChainID.String()] = "mizana-test" + NetworkNames[MizanaMixnetChainConfig.ChainID.String()] = "mizana-mix" + NetworkNames[MizanaPrivnetChainConfig.ChainID.String()] = "mizana-priv" +} + +func IsQngNetwork(chainID *big.Int) bool { + if chainID == QngMainnetChainConfig.ChainID || + chainID == QngTestnetChainConfig.ChainID || + chainID == QngMixnetChainConfig.ChainID || + chainID == QngPrivnetChainConfig.ChainID { + return true + } + return false +} + +func IsAmanaNetwork(chainID *big.Int) bool { + if chainID == AmanaChainConfig.ChainID || + chainID == AmanaTestnetChainConfig.ChainID || + chainID == AmanaMixnetChainConfig.ChainID || + chainID == AmanaPrivnetChainConfig.ChainID { + return true + } + return false +} + +func IsFlanaNetwork(chainID *big.Int) bool { + if chainID == FlanaChainConfig.ChainID || + chainID == FlanaTestnetChainConfig.ChainID || + chainID == FlanaMixnetChainConfig.ChainID || + chainID == FlanaPrivnetChainConfig.ChainID { + return true + } + return false +} + +func IsMizanaNetwork(chainID *big.Int) bool { + if chainID == MizanaChainConfig.ChainID || + chainID == MizanaTestnetChainConfig.ChainID || + chainID == MizanaMixnetChainConfig.ChainID || + chainID == MizanaPrivnetChainConfig.ChainID { + return true + } + return false +}