From 5b74de52ef1986244bae0f982bb751c70293599a Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 16 Nov 2021 14:27:57 +0800 Subject: [PATCH 01/54] Support customization consensus engine --- eth/backend.go | 2 +- eth/ethconfig/config.go | 7 ++++++- les/client.go | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 793d3b81f1b1..65e4b3f01473 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -145,7 +145,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), + engine: config.ConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 89cdb75597e0..ad988ca0ca13 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -91,6 +91,7 @@ var Defaults = Config{ RPCGasCap: 50000000, GPO: FullNodeGPO, RPCTxFeeCap: 1, // 1 ether + ConsensusEngine:CreateDefaultConsensusEngine, } func init() { @@ -114,6 +115,8 @@ func init() { } } +type CreateConsensusEngine func (*node.Node, *params.ChainConfig, *ethash.Config, []string, bool, ethdb.Database) consensus.Engine + //go:generate gencodec -type Config -formats toml -out gen_config.go // Config contains configuration options for of the ETH and LES protocols. @@ -200,10 +203,12 @@ type Config struct { // Berlin block override (TODO: remove after the fork) OverrideLondon *big.Int `toml:",omitempty"` + + ConsensusEngine CreateConsensusEngine } // CreateConsensusEngine creates a consensus engine for the given chain configuration. -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { +func CreateDefaultConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up if chainConfig.Clique != nil { return clique.New(chainConfig.Clique, db) diff --git a/les/client.go b/les/client.go index 5d07c783e99d..df6ed9aa5904 100644 --- a/les/client.go +++ b/les/client.go @@ -109,7 +109,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { eventMux: stack.EventMux(), reqDist: newRequestDistributor(peers, &mclock.System{}), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), + engine: config.ConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), p2pServer: stack.Server(), From 37caa547780a07ed84d970033d4962950f86de16 Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Sun, 12 Dec 2021 12:31:14 +0800 Subject: [PATCH 02/54] go.mod : fix go-duktape warning --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5f4330af3ea9..58e17940c11f 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea github.com/deepmap/oapi-codegen v1.8.2 // indirect + github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 @@ -70,7 +71,6 @@ require ( golang.org/x/text v0.3.6 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce - gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools v2.2.0+incompatible // indirect From c2e721e02fac407def52e594a5fc1d32c41188bf Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Sun, 12 Dec 2021 17:25:20 +0800 Subject: [PATCH 03/54] eth/tracers: fix go-duktape warning --- eth/tracers/tracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index fdeffb8705da..10f9abdedd8a 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "gopkg.in/olebedev/go-duktape.v3" + "github.com/dindinw/go-duktape" ) // bigIntegerJS is the minified version of https://github.com/peterolson/BigInteger.js. From 13b32c05a220491515180b3b28661b87470375cd Mon Sep 17 00:00:00 2001 From: Alex Wu Date: Sun, 12 Dec 2021 17:31:30 +0800 Subject: [PATCH 04/54] go.sum done go.mod tidy --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 0890b8c5e04c..0485733b2f3b 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 h1:zqe534a6faNWAHtQQw9Dz0evsn/3P9GTphvxGukL8aQ= +github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288/go.mod h1:2tBqBHv6UMUEDS9cXrAOJv6A+okIYUNoc910JjZvf3Q= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= @@ -634,8 +636,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 h1:a6cXbcDDUkSBlpnkWV1bJ+vv3mOgQEltEJ2rPxroVu0= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= From 5a818d0bc663e8064cc06e8b76346620b68c484d Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 4 Jan 2022 20:54:46 +0800 Subject: [PATCH 05/54] Support RemoveTx for pool --- core/tx_pool.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/tx_pool.go b/core/tx_pool.go index 5c70cf170578..5b569d40e70c 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -986,6 +986,10 @@ func (pool *TxPool) Has(hash common.Hash) bool { return pool.all.Get(hash) != nil } +func (pool *TxPool) RemoveTx(hash common.Hash, outofbound bool) { + pool.removeTx(hash,outofbound) +} + // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { From ca2e57390fb19ae224d86415386902f4cf7550e8 Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 10 Feb 2022 12:15:05 +0800 Subject: [PATCH 06/54] Expose web3 interface --- expose.go | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 expose.go diff --git a/expose.go b/expose.go new file mode 100644 index 000000000000..9f8e2fe8294b --- /dev/null +++ b/expose.go @@ -0,0 +1,5 @@ +package ethereum + +import "github.com/ethereum/go-ethereum/internal/web3ext" + +var Modules = web3ext.Modules From 29cefce14256e92977bdb1074ec6fa37116edc2d Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 10 Feb 2022 12:18:12 +0800 Subject: [PATCH 07/54] optimize c --- expose.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/expose.go b/expose.go index 9f8e2fe8294b..f63ffafaae92 100644 --- a/expose.go +++ b/expose.go @@ -1,3 +1,5 @@ +// Copyright (c) 2017-2022 The qitmeer developers + package ethereum import "github.com/ethereum/go-ethereum/internal/web3ext" From f092f204e8b120dad3abe6e502599585762df539 Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 17 Feb 2022 10:37:49 +0800 Subject: [PATCH 08/54] Delete dependency about docker --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 58e17940c11f..f6f62ec68923 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 github.com/dlclark/regexp2 v1.2.0 // indirect - github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 github.com/edsrzf/mmap-go v1.0.0 github.com/fatih/color v1.7.0 diff --git a/go.sum b/go.sum index 0485733b2f3b..142fbfac73a1 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,6 @@ github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288 h1:zqe534a6faNW github.com/dindinw/go-duktape v0.0.0-20211014112850-ba2007c86288/go.mod h1:2tBqBHv6UMUEDS9cXrAOJv6A+okIYUNoc910JjZvf3Q= github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 h1:Y9vTBSsV4hSwPSj4bacAU/eSnV3dAxVpepaghAdhGoQ= github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= From 07a4b97a266ffd95564adf9127b915e7c722fa5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 8 Mar 2022 12:00:29 +0200 Subject: [PATCH 09/54] core/statedb: always clear out access list when setting a new one --- core/state/statedb.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index e3541339eaa5..cf46e1f174c6 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -885,7 +885,6 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { func (s *StateDB) Prepare(thash common.Hash, ti int) { s.thash = thash s.txIndex = ti - s.accessList = newAccessList() } func (s *StateDB) clearJournalAndRefund() { @@ -995,6 +994,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // // This method should only be called if Berlin/2929+2930 is applicable at the current number. func (s *StateDB) PrepareAccessList(sender common.Address, dst *common.Address, precompiles []common.Address, list types.AccessList) { + // Clear out any leftover from previous executions + s.accessList = newAccessList() + s.AddAddressToAccessList(sender) if dst != nil { s.AddAddressToAccessList(*dst) From ba548988e4e37965e7659e475d03ed2b2384207f Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 22 Mar 2022 10:42:46 +0800 Subject: [PATCH 10/54] BUG:tx pool multi thread lock conflict --- core/tx_pool.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/tx_pool.go b/core/tx_pool.go index 5b569d40e70c..29c880859a6b 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -987,6 +987,8 @@ func (pool *TxPool) Has(hash common.Hash) bool { } func (pool *TxPool) RemoveTx(hash common.Hash, outofbound bool) { + pool.mu.Lock() + defer pool.mu.Unlock() pool.removeTx(hash,outofbound) } From 3f851b29304a225547d9b96c0c4c5f26b005004a Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 22 Mar 2022 20:08:21 +0800 Subject: [PATCH 11/54] Support external proxy for miner --- eth/api_backend.go | 2 +- eth/backend.go | 11 ++++++++--- miner/interface.go | 27 +++++++++++++++++++++++++++ miner/miner.go | 1 + 4 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 miner/interface.go diff --git a/eth/api_backend.go b/eth/api_backend.go index 1af33414cda4..9aa2ab0b9308 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -339,7 +339,7 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *EthAPIBackend) Miner() *miner.Miner { +func (b *EthAPIBackend) Miner() miner.IMiner { return b.eth.Miner() } diff --git a/eth/backend.go b/eth/backend.go index 65e4b3f01473..5d127c0f8a01 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -85,7 +85,7 @@ type Ethereum struct { APIBackend *EthAPIBackend - miner *miner.Miner + miner miner.IMiner gasPrice *big.Int etherbase common.Address @@ -225,7 +225,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) + if config.Miner.External == nil { + eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) + }else{ + eth.miner = config.Miner.External + } + eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -497,7 +502,7 @@ func (s *Ethereum) StopMining() { } func (s *Ethereum) IsMining() bool { return s.miner.Mining() } -func (s *Ethereum) Miner() *miner.Miner { return s.miner } +func (s *Ethereum) Miner() miner.IMiner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } diff --git a/miner/interface.go b/miner/interface.go new file mode 100644 index 000000000000..a29fc9c3715e --- /dev/null +++ b/miner/interface.go @@ -0,0 +1,27 @@ +package miner + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "time" +) + +type IMiner interface { + Start(coinbase common.Address) + Stop() + Close() + Mining() bool + Hashrate() uint64 + SetExtra(extra []byte) error + SetRecommitInterval(interval time.Duration) + Pending() (*types.Block, *state.StateDB) + PendingBlock() *types.Block + PendingBlockAndReceipts() (*types.Block, types.Receipts) + SetEtherbase(addr common.Address) + SetGasCeil(ceil uint64) + EnablePreseal() + DisablePreseal() + SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription +} diff --git a/miner/miner.go b/miner/miner.go index a4a01b9f4ff7..1a6c36c1edb6 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -51,6 +51,7 @@ type Config struct { GasPrice *big.Int // Minimum gas price for mining a transaction Recommit time.Duration // The time interval for miner to re-create mining work. Noverify bool // Disable remote mining solution verification(only useful in ethash). + External IMiner // External miner } // Miner creates blocks and searches for proof-of-work values. From e3fc7f6b629974f16855e898fd948b7c817d9d87 Mon Sep 17 00:00:00 2001 From: Jin Date: Fri, 10 Jun 2022 19:30:30 +0800 Subject: [PATCH 12/54] optimize miner interface for new version --- cmd/utils/flags.go | 2 +- miner/interface.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4ce16ef90031..839fafa53d8a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2018,7 +2018,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai if ctx.GlobalBool(FakePoWFlag.Name) { ethashConf.PowMode = ethash.ModeFake } - engine = ethconfig.CreateConsensusEngine(stack, config, ðashConf, nil, false, chainDb) + engine = ethconfig.CreateDefaultConsensusEngine(stack, config, ðashConf, nil, false, chainDb) if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } diff --git a/miner/interface.go b/miner/interface.go index a29fc9c3715e..e67ed4f66145 100644 --- a/miner/interface.go +++ b/miner/interface.go @@ -24,4 +24,6 @@ type IMiner interface { EnablePreseal() DisablePreseal() SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription + GetSealingBlockAsync(parent common.Hash, timestamp uint64, coinbase common.Address, random common.Hash, noTxs bool) (chan *types.Block, error) + GetSealingBlockSync(parent common.Hash, timestamp uint64, coinbase common.Address, random common.Hash, noTxs bool) (*types.Block, error) } From f20bb40c07b166106e8c251dadd0f1386815d5ca Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 18 Aug 2022 16:25:58 +0800 Subject: [PATCH 13/54] Remove fork info and optimize ethcatalyst --- cmd/utils/flags.go | 12 ++++++++---- eth/backend.go | 17 ++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f9584340b29a..0d971e8de1e0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1984,8 +1984,10 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - if err := lescatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the Engine API service: %v", err) + if backend.BlockChain().Config().TerminalTotalDifficulty != nil { + if err := lescatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) + } } return backend.ApiBackend, nil } @@ -1999,8 +2001,10 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } - if err := ethcatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the Engine API service: %v", err) + if backend.BlockChain().Config().TerminalTotalDifficulty != nil { + if err := ethcatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) + } } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend diff --git a/eth/backend.go b/eth/backend.go index 73589211a75b..cdcb3b19b0f5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,7 +22,6 @@ import ( "fmt" "math/big" "runtime" - "strings" "sync" "sync/atomic" "time" @@ -141,13 +140,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } - log.Info("") - log.Info(strings.Repeat("-", 153)) - for _, line := range strings.Split(chainConfig.String(), "\n") { - log.Info(line) - } - log.Info(strings.Repeat("-", 153)) - log.Info("") + //log.Info("") + //log.Info(strings.Repeat("-", 153)) + //for _, line := range strings.Split(chainConfig.String(), "\n") { + // log.Info(line) + //} + //log.Info(strings.Repeat("-", 153)) + //log.Info("") if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) @@ -243,7 +242,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.Miner.External == nil { eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) - }else{ + } else { eth.miner = config.Miner.External } From d0dc349fd36bd79f94516c866251783641ed12f1 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Wed, 31 Aug 2022 16:14:53 +0200 Subject: [PATCH 14/54] graphql: return correct logs for tx (#25612) * graphql: fix tx logs * minor * Use optimized search for selecting tx logs --- graphql/graphql.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 97b460c205ce..66c25841db98 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "sort" "strconv" "github.com/ethereum/go-ethereum" @@ -478,13 +479,16 @@ func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { if err != nil { return nil, err } - ret := make([]*Log, 0, len(logs)) - for _, log := range logs { + var ret []*Log + // Select tx logs from all block logs + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) == t.index }) + for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ r: t.r, transaction: t, - log: log, + log: logs[ix], }) + ix++ } return &ret, nil } From 3b41be695e6e12829abf6e5edec0606ee37f14d9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 13 Sep 2022 21:49:52 +0200 Subject: [PATCH 15/54] graphql: fixes missing tx logs (#25745) * graphql: fix tx logs * graphql: refactor test service setup * graphql: add test for tx logs --- graphql/graphql.go | 2 +- graphql/graphql_test.go | 227 +++++++++++++++++++++------------------- graphql/service.go | 9 +- 3 files changed, 124 insertions(+), 114 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 66c25841db98..356ff669fb16 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -481,7 +481,7 @@ func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { } var ret []*Log // Select tx logs from all block logs - ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) == t.index }) + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) >= t.index }) for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ r: t.r, diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index d55f4e063486..491c73152113 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -17,6 +17,8 @@ package graphql import ( + "context" + "encoding/json" "fmt" "io" "math/big" @@ -51,15 +53,21 @@ func TestBuildSchema(t *testing.T) { } defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { + if _, err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint func TestGraphQLBlockSerialization(t *testing.T) { - stack := createNode(t, true, false) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + } + newGQLService(t, stack, genesis, 10, func(i int, gen *core.BlockGen) {}) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -161,8 +169,55 @@ func TestGraphQLBlockSerialization(t *testing.T) { } func TestGraphQLBlockSerializationEIP2718(t *testing.T) { - stack := createNode(t, true, true) + // Account for signing txes + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + dad = common.HexToAddress("0x0000000000000000000000000000000000000dad") + ) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + address: {Balance: funds}, + // The address 0xdad sloads 0x00 and 0x01 + dad: { + Code: []byte{byte(vm.PC), byte(vm.PC), byte(vm.SLOAD), byte(vm.SLOAD)}, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer := types.LatestSigner(genesis.Config) + newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(common.Address{1}) + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: uint64(0), + To: &dad, + Value: big.NewInt(100), + Gas: 50000, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(1), + To: &dad, + Gas: 30000, + GasPrice: big.NewInt(params.InitialBaseFee), + Value: big.NewInt(50), + AccessList: types.AccessList{{ + Address: dad, + StorageKeys: []common.Hash{{0}}, + }}, + }) + gen.AddTx(tx) + }) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -198,7 +253,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { - stack := createNode(t, false, false) + stack := createNode(t) defer stack.Close() if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -212,7 +267,59 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } -func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { +func TestGraphQLTransactionLogs(t *testing.T) { + var ( + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + dadStr = "0x0000000000000000000000000000000000000dad" + dad = common.HexToAddress(dadStr) + genesis = &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(params.Ether)}, + dad: { + // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) + Code: common.Hex2Bytes("60006000a060006000a060006000f3"), + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + signer = types.LatestSigner(genesis.Config) + stack = createNode(t) + ) + defer stack.Close() + + handler := newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 1, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 2, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + }) + // start node + if err := stack.Start(); err != nil { + t.Fatalf("could not start node: %v", err) + } + query := `{block { transactions { logs { account { address } } } } }` + res := handler.Schema.Exec(context.Background(), query, "", map[string]interface{}{}) + if res.Errors != nil { + t.Fatalf("graphql query failed: %v", res.Errors) + } + have, err := json.Marshal(res.Data) + if err != nil { + t.Fatalf("failed to encode graphql response: %s", err) + } + want := fmt.Sprintf(`{"block":{"transactions":[{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]}]}}`, dadStr, dadStr, dadStr, dadStr, dadStr, dadStr) + if string(have) != want { + t.Errorf("response unmatch. expected %s, got %s", want, have) + } +} + +func createNode(t *testing.T) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", HTTPPort: 0, @@ -222,83 +329,12 @@ func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { if err != nil { t.Fatalf("could not create node: %v", err) } - if !gqlEnabled { - return stack - } - if !txEnabled { - createGQLService(t, stack) - } else { - createGQLServiceWithTransactions(t, stack) - } return stack } -func createGQLService(t *testing.T, stack *node.Node) { - // create backend - ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - }, - Ethash: ethash.Config{ - PowMode: ethash.ModeFake, - }, - NetworkId: 1337, - TrieCleanCache: 5, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 5, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 5, - } - ethBackend, err := eth.New(stack, ethConf) - if err != nil { - t.Fatalf("could not create eth backend: %v", err) - } - // Create some blocks and import them - chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 10, func(i int, gen *core.BlockGen) {}) - _, err = ethBackend.BlockChain().InsertChain(chain) - if err != nil { - t.Fatalf("could not create import blocks: %v", err) - } - // create gql service - filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) - err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) - if err != nil { - t.Fatalf("could not create graphql service: %v", err) - } -} - -func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { - // create backend - key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address := crypto.PubkeyToAddress(key.PublicKey) - funds := big.NewInt(1000000000000000) - dad := common.HexToAddress("0x0000000000000000000000000000000000000dad") - +func newGQLService(t *testing.T, stack *node.Node, gspec *core.Genesis, genBlocks int, genfunc func(i int, gen *core.BlockGen)) *handler { ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ - address: {Balance: funds}, - // The address 0xdad sloads 0x00 and 0x01 - dad: { - Code: []byte{ - byte(vm.PC), - byte(vm.PC), - byte(vm.SLOAD), - byte(vm.SLOAD), - }, - Nonce: 0, - Balance: big.NewInt(0), - }, - }, - BaseFee: big.NewInt(params.InitialBaseFee), - }, + Genesis: gspec, Ethash: ethash.Config{ PowMode: ethash.ModeFake, }, @@ -310,49 +346,22 @@ func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { TrieTimeout: 60 * time.Minute, SnapshotCache: 5, } - ethBackend, err := eth.New(stack, ethConf) if err != nil { t.Fatalf("could not create eth backend: %v", err) } - signer := types.LatestSigner(ethConf.Genesis.Config) - - legacyTx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: uint64(0), - To: &dad, - Value: big.NewInt(100), - Gas: 50000, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - envelopTx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ - ChainID: ethConf.Genesis.Config.ChainID, - Nonce: uint64(1), - To: &dad, - Gas: 30000, - GasPrice: big.NewInt(params.InitialBaseFee), - Value: big.NewInt(50), - AccessList: types.AccessList{{ - Address: dad, - StorageKeys: []common.Hash{{0}}, - }}, - }) - // Create some blocks and import them chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 1, func(i int, b *core.BlockGen) { - b.SetCoinbase(common.Address{1}) - b.AddTx(legacyTx) - b.AddTx(envelopTx) - }) - + ethash.NewFaker(), ethBackend.ChainDb(), genBlocks, genfunc) _, err = ethBackend.BlockChain().InsertChain(chain) if err != nil { t.Fatalf("could not create import blocks: %v", err) } - // create gql service + // Set up handler filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) - err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) + handler, err := newHandler(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } + return handler } diff --git a/graphql/service.go b/graphql/service.go index 019026bc7ea7..6f6e58335991 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -57,17 +57,18 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // New constructs a new GraphQL service instance. func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { - return newHandler(stack, backend, filterSystem, cors, vhosts) + _, err := newHandler(stack, backend, filterSystem, cors, vhosts) + return err } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) (*handler, error) { q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { - return err + return nil, err } h := handler{Schema: s} handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) @@ -76,5 +77,5 @@ func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters. stack.RegisterHandler("GraphQL", "/graphql", handler) stack.RegisterHandler("GraphQL", "/graphql/", handler) - return nil + return &h, nil } From 972007a517c49ee9e2a359950d81c74467492ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 14 Sep 2022 11:07:10 +0200 Subject: [PATCH 16/54] Release Geth v1.10.24 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 5f24b41f2852..349bdf1c18d1 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 23 // Patch version component of the current release + VersionPatch = 24 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From 8f61fc8b73565ff15d27fd9731b945d9d1878ec3 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 15 Sep 2022 17:50:54 +0200 Subject: [PATCH 17/54] params: set TerminalTotalDifficultyPassed to true (#25769) * params: set TerminalTotalDifficultyPassed to true * Update params/config.go Co-authored-by: Martin Holst Swende --- params/config.go | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/params/config.go b/params/config.go index 4d6eec8939bc..80e671f9bf41 100644 --- a/params/config.go +++ b/params/config.go @@ -59,25 +59,26 @@ var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - GrayGlacierBlock: big.NewInt(15_050_000), - TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. From 69568c554880b3567bace64f8848ff1be27d084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 15 Sep 2022 17:55:58 +0200 Subject: [PATCH 18/54] params: release Geth v1.10.25 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 349bdf1c18d1..3b5978e849cd 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 24 // Patch version component of the current release + VersionPatch = 25 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From cf51cba21b91e770f2ea29bf734f2a235ec987d6 Mon Sep 17 00:00:00 2001 From: Jin Date: Wed, 5 Oct 2022 21:39:14 +0800 Subject: [PATCH 19/54] usbwallet support Ledger Nano S Plus --- accounts/usbwallet/hub.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/accounts/usbwallet/hub.go b/accounts/usbwallet/hub.go index 23be98a08483..0705bad1eb01 100644 --- a/accounts/usbwallet/hub.go +++ b/accounts/usbwallet/hub.go @@ -75,14 +75,17 @@ func NewLedgerHub() (*Hub, error) { 0x0000, /* Ledger Blue */ 0x0001, /* Ledger Nano S */ 0x0004, /* Ledger Nano X */ + 0x0005, /* Ledger Nano S Plus */ // Upcoming product IDs: https://www.ledger.com/2019/05/17/windows-10-update-sunsetting-u2f-tunnel-transport-for-ledger-devices/ 0x0015, /* HID + U2F + WebUSB Ledger Blue */ 0x1015, /* HID + U2F + WebUSB Ledger Nano S */ 0x4015, /* HID + U2F + WebUSB Ledger Nano X */ + 0x5015, /* HID + U2F + WebUSB Ledger Nano S Plus */ 0x0011, /* HID + WebUSB Ledger Blue */ 0x1011, /* HID + WebUSB Ledger Nano S */ 0x4011, /* HID + WebUSB Ledger Nano X */ + 0x5011, /* HID + WebUSB Ledger Nano S Plus */ }, 0xffa0, 0, newLedgerDriver) } From 85e469f787be53192fcbcee86ce67635020a4ad5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 31 Aug 2022 17:58:18 +0200 Subject: [PATCH 20/54] eth/protocols/snap: fix problems due to idle-but-busy peers (#25651) --- eth/protocols/snap/sync.go | 90 +++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index deaa4456e0fd..1455bacbcb88 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -2248,14 +2248,18 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.accountIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.accountIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.accountReqs[id] if !ok { @@ -2360,14 +2364,18 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeReqs[id] if !ok { @@ -2469,14 +2477,18 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.storageIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.storageIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.storageReqs[id] if !ok { @@ -2596,14 +2608,18 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.trienodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.trienodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.trienodeHealReqs[id] if !ok { @@ -2691,14 +2707,18 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeHealReqs[id] if !ok { From 937ea491f95e5676d2ce3e0364ba9f05452bc351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 9 Sep 2022 11:42:57 +0300 Subject: [PATCH 21/54] eth/protocols/snap: throttle trie heal requests when peers DoS us (#25666) * eth/protocols/snap: throttle trie heal requests when peers DoS us * eth/protocols/snap: lower heal throttle log to debug Co-authored-by: Martin Holst Swende * eth/protocols/snap: fix comment Co-authored-by: Martin Holst Swende --- eth/protocols/snap/sync.go | 107 ++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 1455bacbcb88..eb8260bf7c97 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -21,10 +21,12 @@ import ( "encoding/json" "errors" "fmt" + gomath "math" "math/big" "math/rand" "sort" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -78,6 +80,29 @@ const ( // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. maxTrieRequestCount = maxRequestSize / 512 + + // trienodeHealRateMeasurementImpact is the impact a single measurement has on + // the local node's trienode processing capacity. A value closer to 0 reacts + // slower to sudden changes, but it is also more stable against temporary hiccups. + trienodeHealRateMeasurementImpact = 0.005 + + // minTrienodeHealThrottle is the minimum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + minTrienodeHealThrottle = 1 + + // maxTrienodeHealThrottle is the maximum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + maxTrienodeHealThrottle = maxTrieRequestCount + + // trienodeHealThrottleIncrease is the multiplier for the throttle when the + // rate of arriving data is higher than the rate of processing it. + trienodeHealThrottleIncrease = 1.33 + + // trienodeHealThrottleDecrease is the divisor for the throttle when the + // rate of arriving data is lower than the rate of processing it. + trienodeHealThrottleDecrease = 1.25 ) var ( @@ -431,6 +456,11 @@ type Syncer struct { trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running + trienodeHealRate float64 // Average heal rate for processing trie node data + trienodeHealPend uint64 // Number of trie nodes currently pending for processing + trienodeHealThrottle float64 // Divisor for throttling the amount of trienode heal data requested + trienodeHealThrottled time.Time // Timestamp the last time the throttle was updated + trienodeHealSynced uint64 // Number of state trie nodes downloaded trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk trienodeHealDups uint64 // Number of state trie nodes already processed @@ -476,9 +506,10 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { trienodeHealIdlers: make(map[string]struct{}), bytecodeHealIdlers: make(map[string]struct{}), - trienodeHealReqs: make(map[uint64]*trienodeHealRequest), - bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), - stateWriter: db.NewBatch(), + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + trienodeHealThrottle: maxTrienodeHealThrottle, // Tune downward instead of insta-filling with junk + stateWriter: db.NewBatch(), extProgress: new(SyncProgress), } @@ -1321,6 +1352,10 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai if cap > maxTrieRequestCount { cap = maxTrieRequestCount } + cap = int(float64(cap) / s.trienodeHealThrottle) + if cap <= 0 { + cap = 1 + } var ( hashes = make([]common.Hash, 0, cap) paths = make([]string, 0, cap) @@ -2090,6 +2125,10 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // processTrienodeHealResponse integrates an already validated trienode response // into the healer tasks. func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { + var ( + start = time.Now() + fills int + ) for i, hash := range res.hashes { node := res.nodes[i] @@ -2098,6 +2137,8 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { res.task.trieTasks[res.paths[i]] = res.hashes[i] continue } + fills++ + // Push the trie node into the state syncer s.trienodeHealSynced++ s.trienodeHealBytes += common.StorageSize(len(node)) @@ -2121,6 +2162,50 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { log.Crit("Failed to persist healing data", "err", err) } log.Debug("Persisted set of healing data", "type", "trienodes", "bytes", common.StorageSize(batch.ValueSize())) + + // Calculate the processing rate of one filled trie node + rate := float64(fills) / (float64(time.Since(start)) / float64(time.Second)) + + // Update the currently measured trienode queueing and processing throughput. + // + // The processing rate needs to be updated uniformly independent if we've + // processed 1x100 trie nodes or 100x1 to keep the rate consistent even in + // the face of varying network packets. As such, we cannot just measure the + // time it took to process N trie nodes and update once, we need one update + // per trie node. + // + // Naively, that would be: + // + // for i:=0; i time.Second { + // Periodically adjust the trie node throttler + if float64(pending) > 2*s.trienodeHealRate { + s.trienodeHealThrottle *= trienodeHealThrottleIncrease + } else { + s.trienodeHealThrottle /= trienodeHealThrottleDecrease + } + if s.trienodeHealThrottle > maxTrienodeHealThrottle { + s.trienodeHealThrottle = maxTrienodeHealThrottle + } else if s.trienodeHealThrottle < minTrienodeHealThrottle { + s.trienodeHealThrottle = minTrienodeHealThrottle + } + s.trienodeHealThrottled = time.Now() + + log.Debug("Updated trie node heal throttler", "rate", s.trienodeHealRate, "pending", pending, "throttle", s.trienodeHealThrottle) + } } // processBytecodeHealResponse integrates an already validated bytecode response @@ -2655,10 +2740,12 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Cross reference the requested trienodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) - hash := make([]byte, 32) - - nodes := make([][]byte, len(req.hashes)) + var ( + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + nodes = make([][]byte, len(req.hashes)) + fills uint64 + ) for i, j := 0, 0; i < len(trienodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() @@ -2670,16 +2757,22 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error } if j < len(req.hashes) { nodes[j] = trienodes[i] + fills++ j++ continue } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + // Signal this request as failed, and ready for rescheduling s.scheduleRevertTrienodeHealRequest(req) return errors.New("unexpected healing trienode") } // Response validated, send it to the scheduler for filling + atomic.AddUint64(&s.trienodeHealPend, fills) + defer func() { + atomic.AddUint64(&s.trienodeHealPend, ^(fills - 1)) + }() response := &trienodeHealResponse{ paths: req.paths, task: req.task, From a32e69a28c2abef6a6b96e8d2466f1f27fd693ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 6 Sep 2022 12:57:03 +0300 Subject: [PATCH 22/54] trie: check childrens' existence concurrently for snap heal (#25694) --- trie/sync.go | 57 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/trie/sync.go b/trie/sync.go index 303fcbfa22e2..862ce7e16e6c 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -19,6 +19,7 @@ package trie import ( "errors" "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" @@ -381,11 +382,11 @@ func (s *Sync) scheduleCodeRequest(req *codeRequest) { // retrieval scheduling. func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // Gather all the children of the node, irrelevant whether known or not - type child struct { + type childNode struct { path []byte node node } - var children []child + var children []childNode switch node := (object).(type) { case *shortNode: @@ -393,14 +394,14 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if hasTerm(key) { key = key[:len(key)-1] } - children = []child{{ + children = []childNode{{ node: node.Val, path: append(append([]byte(nil), req.path...), key...), }} case *fullNode: for i := 0; i < 17; i++ { if node.Children[i] != nil { - children = append(children, child{ + children = append(children, childNode{ node: node.Children[i], path: append(append([]byte(nil), req.path...), byte(i)), }) @@ -410,7 +411,10 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { panic(fmt.Sprintf("unknown node: %+v", node)) } // Iterate over the children, and request all unknown ones - requests := make([]*nodeRequest, 0, len(children)) + var ( + missing = make(chan *nodeRequest, len(children)) + pending sync.WaitGroup + ) for _, child := range children { // Notify any external watcher of a new key/value node if req.callback != nil { @@ -433,19 +437,36 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { if s.membatch.hasNode(child.path) { continue } - // If database says duplicate, then at least the trie node is present - // and we hold the assumption that it's NOT legacy contract code. - chash := common.BytesToHash(node) - if rawdb.HasTrieNode(s.database, chash) { - continue - } - // Locally unknown node, schedule for retrieval - requests = append(requests, &nodeRequest{ - path: child.path, - hash: chash, - parent: req, - callback: req.callback, - }) + // Check the presence of children concurrently + pending.Add(1) + go func(child childNode) { + defer pending.Done() + + // If database says duplicate, then at least the trie node is present + // and we hold the assumption that it's NOT legacy contract code. + chash := common.BytesToHash(node) + if rawdb.HasTrieNode(s.database, chash) { + return + } + // Locally unknown node, schedule for retrieval + missing <- &nodeRequest{ + path: child.path, + hash: chash, + parent: req, + callback: req.callback, + } + }(child) + } + } + pending.Wait() + + requests := make([]*nodeRequest, 0, len(children)) + for done := false; !done; { + select { + case miss := <-missing: + requests = append(requests, miss) + default: + done = true } } return requests, nil From 99bbb337011ef614fc5d25c130aa1cef92915366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 20 Sep 2022 14:14:24 +0300 Subject: [PATCH 23/54] eth: fix a rare datarace on CHT challenge reply / shutdown (#25831) --- eth/handler.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 4224a9f33a84..143147b0c815 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -391,11 +391,16 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { if h.checkpointHash != (common.Hash{}) { // Request the peer's checkpoint header for chain height/weight validation resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh) + if err != nil { return err } // Start a timer to disconnect if the peer doesn't reply in time go func() { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -437,10 +442,15 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If we have any explicit peer required block hashes, request them for number, hash := range h.requiredBlocks { resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh) + if err != nil { return err } - go func(number uint64, hash common.Hash) { + go func(number uint64, hash common.Hash, req *eth.Request) { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -469,7 +479,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Warn("Required block challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) h.removePeer(peer.ID()) } - }(number, hash) + }(number, hash, req) } // Handle incoming messages until the connection is torn down return handler(peer) From 27600a5b8452c39ce9ef5c502c98a0c1ee35859b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 27 Oct 2022 15:25:01 +0200 Subject: [PATCH 24/54] eth/filters: change filter block to be by-ref (#26054) This PR changes the block field in the filter to be a pointer, to disambiguate between empty hash and no hash --- eth/filters/filter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 0a70c9ece1db..8d80647fc241 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -34,8 +34,8 @@ type Filter struct { addresses []common.Address topics [][]common.Hash - block common.Hash // Block hash if filtering a single block - begin, end int64 // Range interval if filtering multiple blocks + block *common.Hash // Block hash if filtering a single block + begin, end int64 // Range interval if filtering multiple blocks matcher *bloombits.Matcher } @@ -78,7 +78,7 @@ func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Add func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter filter := newFilter(sys, addresses, topics) - filter.block = block + filter.block = &block return filter } @@ -96,8 +96,8 @@ func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common. // first block that contains matches, updating the start of the filter accordingly. func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return - if f.block != (common.Hash{}) { - header, err := f.sys.backend.HeaderByHash(ctx, f.block) + if f.block != nil { + header, err := f.sys.backend.HeaderByHash(ctx, *f.block) if err != nil { return nil, err } From 211dbb719711420358a3cdf718ce476c212adcf5 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Wed, 2 Nov 2022 09:29:33 -0500 Subject: [PATCH 25/54] rpc: handle wrong HTTP batch response length (#26064) --- rpc/client.go | 1 + rpc/client_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++ rpc/http.go | 3 +++ 3 files changed, 52 insertions(+) diff --git a/rpc/client.go b/rpc/client.go index d3ce0297754c..fcd31319a3c5 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -31,6 +31,7 @@ import ( ) var ( + ErrBadResult = errors.New("bad result in JSON-RPC response") ErrClientQuit = errors.New("client is closed") ErrNoResult = errors.New("no result in JSON-RPC response") ErrSubscriptionQueueOverflow = errors.New("subscription queue overflow") diff --git a/rpc/client_test.go b/rpc/client_test.go index 04c847d0d626..5ae549d86533 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math/rand" "net" @@ -144,6 +145,53 @@ func TestClientBatchRequest(t *testing.T) { } } +func TestClientBatchRequest_len(t *testing.T) { + b, err := json.Marshal([]jsonrpcMessage{ + {Version: "2.0", ID: json.RawMessage("1"), Method: "foo", Result: json.RawMessage(`"0x1"`)}, + {Version: "2.0", ID: json.RawMessage("2"), Method: "bar", Result: json.RawMessage(`"0x2"`)}, + }) + if err != nil { + t.Fatal("failed to encode jsonrpc message:", err) + } + s := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + _, err := rw.Write(b) + if err != nil { + t.Error("failed to write response:", err) + } + })) + t.Cleanup(s.Close) + + client, err := Dial(s.URL) + if err != nil { + t.Fatal("failed to dial test server:", err) + } + defer client.Close() + + t.Run("too-few", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + {Method: "bar"}, + {Method: "baz"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) + + t.Run("too-many", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) +} + func TestClientNotify(t *testing.T) { server := newTestServer() defer server.Stop() diff --git a/rpc/http.go b/rpc/http.go index 858d80858652..b82ea2707a54 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -173,6 +173,9 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { return err } + if len(respmsgs) != len(msgs) { + return fmt.Errorf("batch has %d requests but response has %d: %w", len(msgs), len(respmsgs), ErrBadResult) + } for i := 0; i < len(respmsgs); i++ { op.resp <- &respmsgs[i] } From e5eb32acee19cc9fca6a03b10283b7484246b15a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 3 Nov 2022 11:59:33 +0100 Subject: [PATCH 26/54] params: release geth v1.10.26 stable --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 3b5978e849cd..d9a3958512c3 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 25 // Patch version component of the current release + VersionPatch = 26 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) From 76f1bb8b834198892dee8fa62d5d90b25ddb8e78 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 9 Nov 2022 08:06:02 +0100 Subject: [PATCH 27/54] core/state: replace fastcache code cache with gc-friendly structure (#26092) This PR replaces fastcache with a pretty simple LRU which does not require explicit closing. (cherry picked from commit 5fded040372784985265f83f33f15cb6a51bebdb) --- common/lru/blob_lru.go | 88 ++++++++++++++++++++++++++ common/lru/blob_lru_test.go | 122 ++++++++++++++++++++++++++++++++++++ core/state/database.go | 14 ++--- 3 files changed, 217 insertions(+), 7 deletions(-) create mode 100644 common/lru/blob_lru.go create mode 100644 common/lru/blob_lru_test.go diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go new file mode 100644 index 000000000000..3138f422ccfa --- /dev/null +++ b/common/lru/blob_lru.go @@ -0,0 +1,88 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "math" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/hashicorp/golang-lru/simplelru" +) + +// SizeConstrainedLRU is a wrapper around simplelru.LRU. The simplelru.LRU is capable +// of item-count constraints, but is not capable of enforcing a byte-size constraint, +// hence this wrapper. +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +type SizeConstrainedLRU struct { + size uint64 + maxSize uint64 + lru *simplelru.LRU + lock sync.RWMutex +} + +// NewSizeConstrainedLRU creates a new SizeConstrainedLRU. +func NewSizeConstrainedLRU(max uint64) *SizeConstrainedLRU { + lru, err := simplelru.NewLRU(math.MaxInt, nil) + if err != nil { + panic(err) + } + return &SizeConstrainedLRU{ + size: 0, + maxSize: max, + lru: lru, + } +} + +// Add adds a value to the cache. Returns true if an eviction occurred. +// OBS: This cache assumes that items are content-addressed: keys are unique per content. +// In other words: two Add(..) with the same key K, will always have the same value V. +// OBS: The value is _not_ copied on Add, so the caller must not modify it afterwards. +func (c *SizeConstrainedLRU) Add(key common.Hash, value []byte) (evicted bool) { + c.lock.Lock() + defer c.lock.Unlock() + + // Unless it is already present, might need to evict something. + // OBS: If it is present, we still call Add internally to bump the recentness. + if !c.lru.Contains(key) { + targetSize := c.size + uint64(len(value)) + for targetSize > c.maxSize { + evicted = true + _, v, ok := c.lru.RemoveOldest() + if !ok { + // list is now empty. Break + break + } + targetSize -= uint64(len(v.([]byte))) + } + c.size = targetSize + } + c.lru.Add(key, value) + return evicted +} + +// Get looks up a key's value from the cache. +func (c *SizeConstrainedLRU) Get(key common.Hash) []byte { + c.lock.RLock() + defer c.lock.RUnlock() + + if v, ok := c.lru.Get(key); ok { + return v.([]byte) + } + return nil +} diff --git a/common/lru/blob_lru_test.go b/common/lru/blob_lru_test.go new file mode 100644 index 000000000000..4b5e69340222 --- /dev/null +++ b/common/lru/blob_lru_test.go @@ -0,0 +1,122 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package lru + +import ( + "encoding/binary" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func mkHash(i int) common.Hash { + h := make([]byte, 32) + binary.LittleEndian.PutUint64(h, uint64(i)) + return common.BytesToHash(h) +} + +func TestBlobLru(t *testing.T) { + lru := NewSizeConstrainedLRU(100) + var want uint64 + // Add 11 items of 10 byte each. First item should be swapped out + for i := 0; i < 11; i++ { + k := mkHash(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + want += uint64(len(v)) + if want > 100 { + want = 100 + } + if have := lru.size; have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } + // Zero:th should be evicted + { + k := mkHash(0) + if val := lru.Get(k); val != nil { + t.Fatalf("should be evicted: %v", k) + } + } + // Elems 1-11 should be present + for i := 1; i < 11; i++ { + k := mkHash(i) + want := fmt.Sprintf("value-%04d", i) + have := lru.Get(k) + if have == nil { + t.Fatalf("missing key %v", k) + } + if string(have) != want { + t.Fatalf("wrong value, have %v want %v", have, want) + } + } +} + +// TestBlobLruOverflow tests what happens when inserting an element exceeding +// the max size +func TestBlobLruOverflow(t *testing.T) { + lru := NewSizeConstrainedLRU(100) + // Add 10 items of 10 byte each, filling the cache + for i := 0; i < 10; i++ { + k := mkHash(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + } + // Add one single large elem. We expect it to swap out all entries. + { + k := mkHash(1337) + v := make([]byte, 200) + lru.Add(k, v) + } + // Elems 0-9 should be missing + for i := 1; i < 10; i++ { + k := mkHash(i) + if val := lru.Get(k); val != nil { + t.Fatalf("should be evicted: %v", k) + } + } + // The size should be accurate + if have, want := lru.size, uint64(200); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + // Adding one small item should swap out the large one + { + i := 0 + k := mkHash(i) + v := fmt.Sprintf("value-%04d", i) + lru.Add(k, []byte(v)) + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } + } +} + +// TestBlobLruSameItem tests what happens when inserting the same k/v multiple times. +func TestBlobLruSameItem(t *testing.T) { + lru := NewSizeConstrainedLRU(100) + // Add one 10 byte-item 10 times + k := mkHash(0) + v := fmt.Sprintf("value-%04d", 0) + for i := 0; i < 10; i++ { + lru.Add(k, []byte(v)) + } + // The size should be accurate + if have, want := lru.size, uint64(10); have != want { + t.Fatalf("size wrong, have %d want %d", have, want) + } +} diff --git a/core/state/database.go b/core/state/database.go index 96b6bcfe6551..28de45261db4 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" - "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" + lru2 "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -131,14 +131,14 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, - codeCache: fastcache.New(codeCacheSize), + codeCache: lru2.NewSizeConstrainedLRU(codeCacheSize), } } type cachingDB struct { db *trie.Database codeSizeCache *lru.Cache - codeCache *fastcache.Cache + codeCache *lru2.SizeConstrainedLRU } // OpenTrie opens the main account trie at a specific root hash. @@ -171,12 +171,12 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { // ContractCode retrieves a particular contract's code. func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { - if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { + if code := db.codeCache.Get(codeHash); len(code) > 0 { return code, nil } code := rawdb.ReadCode(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Set(codeHash.Bytes(), code) + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } @@ -187,12 +187,12 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error // code can't be found in the cache, then check the existence with **new** // db scheme. func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { - if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { + if code := db.codeCache.Get(codeHash); len(code) > 0 { return code, nil } code := rawdb.ReadCodeWithPrefix(db.db.DiskDB(), codeHash) if len(code) > 0 { - db.codeCache.Set(codeHash.Bytes(), code) + db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) return code, nil } From 354b6a797acbab480844f2a6d948dd52a8854b51 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 11 Nov 2022 19:48:36 +0100 Subject: [PATCH 28/54] common/lru: fix race in lru (#26164) This fixes a problem in the SizeConstrainedLRU. The SCLRU uses an underlying simple lru which is not thread safe. During the Get operation, the recentness of the accessed item is updated, so it is not a pure read-operation. Therefore, the mutex we need is a full mutex, not RLock. This PR changes the mutex to be a regular Mutex, instead of RWMutex, so a reviewer can at a glance see that all affected locations are fixed. (cherry picked from commit 8334b5f51a16b72cf2b3ebdc9e131a442c5289d7) --- common/lru/blob_lru.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/lru/blob_lru.go b/common/lru/blob_lru.go index 3138f422ccfa..b24684256c88 100644 --- a/common/lru/blob_lru.go +++ b/common/lru/blob_lru.go @@ -33,7 +33,7 @@ type SizeConstrainedLRU struct { size uint64 maxSize uint64 lru *simplelru.LRU - lock sync.RWMutex + lock sync.Mutex } // NewSizeConstrainedLRU creates a new SizeConstrainedLRU. @@ -78,8 +78,8 @@ func (c *SizeConstrainedLRU) Add(key common.Hash, value []byte) (evicted bool) { // Get looks up a key's value from the cache. func (c *SizeConstrainedLRU) Get(key common.Hash) []byte { - c.lock.RLock() - defer c.lock.RUnlock() + c.lock.Lock() + defer c.lock.Unlock() if v, ok := c.lru.Get(key); ok { return v.([]byte) From 283fb4ec13f7a140311c1bf77c7b7406dd2436de Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 25 Nov 2022 16:10:31 +0800 Subject: [PATCH 29/54] core/rawdb: fix freezer validation (#26251) * core/rawdb: fix freezer validation * core/rawdb: address comment (cherry picked from commit add1bff13fff722acbf1bd06b57245361ff82359) --- core/rawdb/freezer.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 6dea98c3d3c4..03093e7b1dd7 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -318,30 +318,35 @@ func (f *Freezer) Sync() error { return nil } -// validate checks that every table has the same length. +// validate checks that every table has the same boundary. // Used instead of `repair` in readonly mode. func (f *Freezer) validate() error { if len(f.tables) == 0 { return nil } var ( - length uint64 - name string + head uint64 + tail uint64 + name string ) - // Hack to get length of any table + // Hack to get boundary of any table for kind, table := range f.tables { - length = atomic.LoadUint64(&table.items) + head = atomic.LoadUint64(&table.items) + tail = atomic.LoadUint64(&table.itemHidden) name = kind break } - // Now check every table against that length + // Now check every table against those boundaries. for kind, table := range f.tables { - items := atomic.LoadUint64(&table.items) - if length != items { - return fmt.Errorf("freezer tables %s and %s have differing lengths: %d != %d", kind, name, items, length) + if head != atomic.LoadUint64(&table.items) { + return fmt.Errorf("freezer tables %s and %s have differing head: %d != %d", kind, name, atomic.LoadUint64(&table.items), head) + } + if tail != atomic.LoadUint64(&table.itemHidden) { + return fmt.Errorf("freezer tables %s and %s have differing tail: %d != %d", kind, name, atomic.LoadUint64(&table.itemHidden), tail) } } - atomic.StoreUint64(&f.frozen, length) + atomic.StoreUint64(&f.frozen, head) + atomic.StoreUint64(&f.tail, tail) return nil } From 31ed80298dc683ae8e11eba0af9961165cc05890 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 24 Nov 2022 10:50:28 +0100 Subject: [PATCH 30/54] core/rawdb: improve freezerTable.Sync (#26245) While investigating #22374, I noticed that the Sync operation of the freezer does not take the table lock. It also doesn't call sync for all files if there is an error with one of them. I doubt this will fix anything, but didn't want to drop the fix on the floor either. (cherry picked from commit 193f350eb911c9d8a93577d619986e97f490b700) --- core/rawdb/freezer_table.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 3fe691cf6d2a..e9998fa463c6 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -875,13 +875,20 @@ func (t *freezerTable) advanceHead() error { // Sync pushes any pending data from memory out to disk. This is an expensive // operation, so use it with care. func (t *freezerTable) Sync() error { - if err := t.index.Sync(); err != nil { - return err - } - if err := t.meta.Sync(); err != nil { - return err + t.lock.Lock() + defer t.lock.Unlock() + + var err error + trackError := func(e error) { + if e != nil && err == nil { + err = e + } } - return t.head.Sync() + + trackError(t.index.Sync()) + trackError(t.meta.Sync()) + trackError(t.head.Sync()) + return err } func (t *freezerTable) dumpIndexStdout(start, stop int64) { From 3d579f58f8a3a27224f290616806b1d9d52133a2 Mon Sep 17 00:00:00 2001 From: Jin Date: Mon, 9 Jan 2023 10:07:03 +0800 Subject: [PATCH 31/54] expose GlobalBig --- expose.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/expose.go b/expose.go index f63ffafaae92..e369faaa7f31 100644 --- a/expose.go +++ b/expose.go @@ -2,6 +2,15 @@ package ethereum -import "github.com/ethereum/go-ethereum/internal/web3ext" +import ( + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/web3ext" + "github.com/urfave/cli/v2" + "math/big" +) var Modules = web3ext.Modules + +func GlobalBig(ctx *cli.Context, name string) *big.Int { + return flags.GlobalBig(ctx,name) +} \ No newline at end of file From a7e8e2c89ae034628516a5a1a93557924a5c9f1f Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 28 Mar 2023 21:58:59 +0800 Subject: [PATCH 32/54] MeerChainConfig --- params/config.go | 12 +++++ params/qng_config.go | 123 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 params/qng_config.go diff --git a/params/config.go b/params/config.go index adb5f44f0952..b9d116810eec 100644 --- a/params/config.go +++ b/params/config.go @@ -497,6 +497,18 @@ func (c *ChainConfig) Description() string { } else { banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" } + case IsQngNetwork(c.ChainID): + banner += "Consensus: MeerDAG (proof-of-work)\n" + return banner + case IsAmanaNetwork(c.ChainID): + banner += "Consensus: Amana (proof-of-authority)\n" + return banner + case IsFlanaNetwork(c.ChainID): + banner += "Consensus: Flana (rollup)\n" + return banner + case IsMizanaNetwork(c.ChainID): + banner += "Consensus: Mizana (ZK rollup)\n" + return banner default: banner += "Consensus: unknown\n" } diff --git a/params/qng_config.go b/params/qng_config.go new file mode 100644 index 000000000000..2de3b0a8bd31 --- /dev/null +++ b/params/qng_config.go @@ -0,0 +1,123 @@ +package params + +import "math/big" + +type MeerChainConfig struct { + ChainID *big.Int // chainId identifies the current chain and is used for replay protection +} + +var ( + QngMainnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(813), + } + QngTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8131), + } + QngMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8132), + } + QngPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8133), + } + + AmanaChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8134), + } + AmanaTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81341), + } + AmanaMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81342), + } + AmanaPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81343), + } + + FlanaChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8135), + } + FlanaTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81351), + } + FlanaMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81352), + } + FlanaPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81353), + } + + MizanaChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(8136), + } + MizanaTestnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81361), + } + MizanaMixnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81362), + } + MizanaPrivnetChainConfig = &MeerChainConfig{ + ChainID: big.NewInt(81363), + } +) + +func init() { + NetworkNames[QngMainnetChainConfig.ChainID.String()] = "qng" + NetworkNames[QngTestnetChainConfig.ChainID.String()] = "qng-test" + NetworkNames[QngMixnetChainConfig.ChainID.String()] = "qng-mix" + NetworkNames[QngPrivnetChainConfig.ChainID.String()] = "qng-priv" + + NetworkNames[AmanaChainConfig.ChainID.String()] = "amana" + NetworkNames[AmanaTestnetChainConfig.ChainID.String()] = "amana-test" + NetworkNames[AmanaMixnetChainConfig.ChainID.String()] = "amana-mix" + NetworkNames[AmanaPrivnetChainConfig.ChainID.String()] = "amana-priv" + + NetworkNames[FlanaChainConfig.ChainID.String()] = "flana" + NetworkNames[FlanaTestnetChainConfig.ChainID.String()] = "flana-test" + NetworkNames[FlanaMixnetChainConfig.ChainID.String()] = "flana-mix" + NetworkNames[FlanaPrivnetChainConfig.ChainID.String()] = "flana-priv" + + NetworkNames[MizanaChainConfig.ChainID.String()] = "mizana" + NetworkNames[MizanaTestnetChainConfig.ChainID.String()] = "mizana-test" + NetworkNames[MizanaMixnetChainConfig.ChainID.String()] = "mizana-mix" + NetworkNames[MizanaPrivnetChainConfig.ChainID.String()] = "mizana-priv" +} + +func IsQngNetwork(chainID *big.Int) bool { + if chainID == QngMainnetChainConfig.ChainID || + chainID == QngTestnetChainConfig.ChainID || + chainID == QngMixnetChainConfig.ChainID || + chainID == QngPrivnetChainConfig.ChainID { + return true + } + return false +} + +func IsAmanaNetwork(chainID *big.Int) bool { + if chainID == AmanaChainConfig.ChainID || + chainID == AmanaTestnetChainConfig.ChainID || + chainID == AmanaMixnetChainConfig.ChainID || + chainID == AmanaPrivnetChainConfig.ChainID { + return true + } + return false +} + +func IsFlanaNetwork(chainID *big.Int) bool { + if chainID == FlanaChainConfig.ChainID || + chainID == FlanaTestnetChainConfig.ChainID || + chainID == FlanaMixnetChainConfig.ChainID || + chainID == FlanaPrivnetChainConfig.ChainID { + return true + } + return false +} + +func IsMizanaNetwork(chainID *big.Int) bool { + if chainID == MizanaChainConfig.ChainID || + chainID == MizanaTestnetChainConfig.ChainID || + chainID == MizanaMixnetChainConfig.ChainID || + chainID == MizanaPrivnetChainConfig.ChainID { + return true + } + return false +} From 4616f375215f2cf6b478ce97ae2ee2a424ecfeaa Mon Sep 17 00:00:00 2001 From: Jin Date: Sat, 1 Apr 2023 10:27:57 +0800 Subject: [PATCH 33/54] BUG:ChainConfig Description error --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 47566276fc12..9abb009c3d2b 100644 --- a/params/config.go +++ b/params/config.go @@ -482,6 +482,18 @@ func (c *ChainConfig) Description() string { } banner += fmt.Sprintf("Chain ID: %v (%s)\n", c.ChainID, network) switch { + case IsQngNetwork(c.ChainID): + banner += "Consensus: MeerDAG (proof-of-work)\n" + return banner + case IsAmanaNetwork(c.ChainID): + banner += "Consensus: Amana (proof-of-authority)\n" + return banner + case IsFlanaNetwork(c.ChainID): + banner += "Consensus: Flana (rollup)\n" + return banner + case IsMizanaNetwork(c.ChainID): + banner += "Consensus: Mizana (ZK rollup)\n" + return banner case c.Ethash != nil: if c.TerminalTotalDifficulty == nil { banner += "Consensus: Ethash (proof-of-work)\n" @@ -498,18 +510,6 @@ func (c *ChainConfig) Description() string { } else { banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" } - case IsQngNetwork(c.ChainID): - banner += "Consensus: MeerDAG (proof-of-work)\n" - return banner - case IsAmanaNetwork(c.ChainID): - banner += "Consensus: Amana (proof-of-authority)\n" - return banner - case IsFlanaNetwork(c.ChainID): - banner += "Consensus: Flana (rollup)\n" - return banner - case IsMizanaNetwork(c.ChainID): - banner += "Consensus: Mizana (ZK rollup)\n" - return banner default: banner += "Consensus: unknown\n" } From c4e76b09488cca7f5bd00c2a2dc46e7cc0f2be1f Mon Sep 17 00:00:00 2001 From: Jin Date: Sat, 1 Apr 2023 11:06:03 +0800 Subject: [PATCH 34/54] Support QngEIPsBanner --- params/config.go | 8 +++--- params/qng_config.go | 66 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/params/config.go b/params/config.go index 9abb009c3d2b..baf81534978c 100644 --- a/params/config.go +++ b/params/config.go @@ -484,16 +484,16 @@ func (c *ChainConfig) Description() string { switch { case IsQngNetwork(c.ChainID): banner += "Consensus: MeerDAG (proof-of-work)\n" - return banner + return QngEIPsBanner(banner, c) case IsAmanaNetwork(c.ChainID): banner += "Consensus: Amana (proof-of-authority)\n" - return banner + return QngEIPsBanner(banner, c) case IsFlanaNetwork(c.ChainID): banner += "Consensus: Flana (rollup)\n" - return banner + return QngEIPsBanner(banner, c) case IsMizanaNetwork(c.ChainID): banner += "Consensus: Mizana (ZK rollup)\n" - return banner + return QngEIPsBanner(banner, c) case c.Ethash != nil: if c.TerminalTotalDifficulty == nil { banner += "Consensus: Ethash (proof-of-work)\n" diff --git a/params/qng_config.go b/params/qng_config.go index 2de3b0a8bd31..9c576b4b1e40 100644 --- a/params/qng_config.go +++ b/params/qng_config.go @@ -1,6 +1,9 @@ package params -import "math/big" +import ( + "fmt" + "math/big" +) type MeerChainConfig struct { ChainID *big.Int // chainId identifies the current chain and is used for replay protection @@ -121,3 +124,64 @@ func IsMizanaNetwork(chainID *big.Int) bool { } return false } + +func QngEIPsBanner(banner string, c *ChainConfig) string { + banner += "\n" + + // Create a list of forks with a short description of them. Forks that only + // makes sense for mainnet should be optional at printing to avoid bloating + // the output for testnets and private networks. + banner += "Pre-Merge hard forks (block based):\n" + banner += fmt.Sprintf(" - Homestead: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/homestead.md)\n", c.HomesteadBlock) + if c.DAOForkBlock != nil { + banner += fmt.Sprintf(" - DAO Fork: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/dao-fork.md)\n", c.DAOForkBlock) + } + banner += fmt.Sprintf(" - Tangerine Whistle (EIP 150): #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/tangerine-whistle.md)\n", c.EIP150Block) + banner += fmt.Sprintf(" - Spurious Dragon/1 (EIP 155): #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Spurious Dragon/2 (EIP 158): #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Byzantium: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/byzantium.md)\n", c.ByzantiumBlock) + banner += fmt.Sprintf(" - Constantinople: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/constantinople.md)\n", c.ConstantinopleBlock) + banner += fmt.Sprintf(" - Petersburg: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/petersburg.md)\n", c.PetersburgBlock) + banner += fmt.Sprintf(" - Istanbul: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/istanbul.md)\n", c.IstanbulBlock) + if c.MuirGlacierBlock != nil { + banner += fmt.Sprintf(" - Muir Glacier: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/muir-glacier.md)\n", c.MuirGlacierBlock) + } + banner += fmt.Sprintf(" - Berlin: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/berlin.md)\n", c.BerlinBlock) + banner += fmt.Sprintf(" - London: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/london.md)\n", c.LondonBlock) + if c.ArrowGlacierBlock != nil { + banner += fmt.Sprintf(" - Arrow Glacier: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/arrow-glacier.md)\n", c.ArrowGlacierBlock) + } + if c.GrayGlacierBlock != nil { + banner += fmt.Sprintf(" - Gray Glacier: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/gray-glacier.md)\n", c.GrayGlacierBlock) + } + banner += "\n" + + // Add a special section for the merge as it's non-obvious + if c.TerminalTotalDifficulty == nil { + banner += "The Merge is not yet available for this network!\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + } else { + banner += "Merge configured:\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) + banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) + if c.MergeNetsplitBlock != nil { + banner += fmt.Sprintf(" - Merge netsplit block: #%-8v\n", c.MergeNetsplitBlock) + } + } + banner += "\n" + + // Create a list of forks post-merge + banner += "Post-Merge hard forks (timestamp based):\n" + if c.ShanghaiTime != nil { + banner += fmt.Sprintf(" - Shanghai: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", *c.ShanghaiTime) + } + if c.CancunTime != nil { + banner += fmt.Sprintf(" - Cancun: @%-10v\n", *c.CancunTime) + } + if c.PragueTime != nil { + banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) + } + + return banner +} From ac3804ca1c34e805a2ae63f2d0b33000ff088030 Mon Sep 17 00:00:00 2001 From: Jin Date: Sat, 1 Apr 2023 12:00:41 +0800 Subject: [PATCH 35/54] Optimize QngEIPsBanner --- params/qng_config.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/params/qng_config.go b/params/qng_config.go index 9c576b4b1e40..885a8a694d34 100644 --- a/params/qng_config.go +++ b/params/qng_config.go @@ -132,37 +132,37 @@ func QngEIPsBanner(banner string, c *ChainConfig) string { // makes sense for mainnet should be optional at printing to avoid bloating // the output for testnets and private networks. banner += "Pre-Merge hard forks (block based):\n" - banner += fmt.Sprintf(" - Homestead: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/homestead.md)\n", c.HomesteadBlock) + banner += fmt.Sprintf(" - Homestead: #%-8v (EIP-606 Requires:EIP-2, EIP-7, EIP-8)\n", c.HomesteadBlock) if c.DAOForkBlock != nil { - banner += fmt.Sprintf(" - DAO Fork: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/dao-fork.md)\n", c.DAOForkBlock) - } - banner += fmt.Sprintf(" - Tangerine Whistle (EIP 150): #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/tangerine-whistle.md)\n", c.EIP150Block) - banner += fmt.Sprintf(" - Spurious Dragon/1 (EIP 155): #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) - banner += fmt.Sprintf(" - Spurious Dragon/2 (EIP 158): #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) - banner += fmt.Sprintf(" - Byzantium: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/byzantium.md)\n", c.ByzantiumBlock) - banner += fmt.Sprintf(" - Constantinople: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/constantinople.md)\n", c.ConstantinopleBlock) - banner += fmt.Sprintf(" - Petersburg: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/petersburg.md)\n", c.PetersburgBlock) - banner += fmt.Sprintf(" - Istanbul: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/istanbul.md)\n", c.IstanbulBlock) + banner += fmt.Sprintf(" - DAO Fork: #%-8v (EIP-779 Requires:EIP-606)\n", c.DAOForkBlock) + } + banner += fmt.Sprintf(" - Tangerine Whistle (EIP 150): #%-8v (EIP-608 Requires:EIP-150, EIP-779)\n", c.EIP150Block) + banner += fmt.Sprintf(" - Spurious Dragon/1 (EIP 155): #%-8v (EIP-607 Requires:EIP-155, EIP-160, EIP-161, EIP-170, EIP-608)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Spurious Dragon/2 (EIP 158): #%-8v (EIP-607 Requires:EIP-155, EIP-160, EIP-161, EIP-170, EIP-608)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Byzantium: #%-8v (EIP-609 Requires:EIP-100, EIP-140, EIP-196, EIP-197, EIP-198, EIP-211, EIP-214, EIP-607, EIP-649, EIP-658)\n", c.ByzantiumBlock) + banner += fmt.Sprintf(" - Constantinople: #%-8v (EIP-1013 Requires:EIP-145, EIP-609, EIP-1014, EIP-1052, EIP-1234, EIP-1283)\n", c.ConstantinopleBlock) + banner += fmt.Sprintf(" - Petersburg: #%-8v (EIP-1716 Requires:EIP-1013, EIP-1283)\n", c.PetersburgBlock) + banner += fmt.Sprintf(" - Istanbul: #%-8v (EIP-1679 Requires:EIP-152, EIP-1108, EIP-1344, EIP-1716, EIP-1884, EIP-2028, EIP-2200)\n", c.IstanbulBlock) if c.MuirGlacierBlock != nil { - banner += fmt.Sprintf(" - Muir Glacier: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/muir-glacier.md)\n", c.MuirGlacierBlock) + banner += fmt.Sprintf(" - Muir Glacier: #%-8v (EIP-2387 Requires:EIP-1679, EIP-2384)\n", c.MuirGlacierBlock) } - banner += fmt.Sprintf(" - Berlin: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/berlin.md)\n", c.BerlinBlock) - banner += fmt.Sprintf(" - London: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/london.md)\n", c.LondonBlock) + banner += fmt.Sprintf(" - Berlin: #%-8v (EIP-2565 EIP-2929 EIP-2718 EIP-2930 Requires:EIP-198)\n", c.BerlinBlock) + banner += fmt.Sprintf(" - London: #%-8v (EIP-1559 EIP-3198 EIP-3529 EIP-3541 EIP-3554 Requires:EIP-2718, EIP-2930, EIP-2200, EIP-2929, EIP-2930)\n", c.LondonBlock) if c.ArrowGlacierBlock != nil { - banner += fmt.Sprintf(" - Arrow Glacier: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/arrow-glacier.md)\n", c.ArrowGlacierBlock) + banner += fmt.Sprintf(" - Arrow Glacier: #%-8v (EIP-4345)\n", c.ArrowGlacierBlock) } if c.GrayGlacierBlock != nil { - banner += fmt.Sprintf(" - Gray Glacier: #%-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/gray-glacier.md)\n", c.GrayGlacierBlock) + banner += fmt.Sprintf(" - Gray Glacier: #%-8v (EIP-5133)\n", c.GrayGlacierBlock) } banner += "\n" // Add a special section for the merge as it's non-obvious if c.TerminalTotalDifficulty == nil { banner += "The Merge is not yet available for this network!\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + banner += " - Hard-fork specification: EIP-3675 EIP-4399 Requires:EIP-2124\n" } else { banner += "Merge configured:\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + banner += " - Hard-fork specification: EIP-3675 EIP-4399 Requires:EIP-2124\n" banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) if c.MergeNetsplitBlock != nil { @@ -174,7 +174,7 @@ func QngEIPsBanner(banner string, c *ChainConfig) string { // Create a list of forks post-merge banner += "Post-Merge hard forks (timestamp based):\n" if c.ShanghaiTime != nil { - banner += fmt.Sprintf(" - Shanghai: @%-10v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", *c.ShanghaiTime) + banner += fmt.Sprintf(" - Shanghai: @%-10v (EIP-3651 EIP-3855 EIP-3860 EIP-4895 EIP-6049 Requires:EIP-2929, EIP-170)\n", *c.ShanghaiTime) } if c.CancunTime != nil { banner += fmt.Sprintf(" - Cancun: @%-10v\n", *c.CancunTime) From 928e85f9e4485eef19673ca85080935eb527863c Mon Sep 17 00:00:00 2001 From: Jin Date: Sat, 1 Apr 2023 20:39:26 +0800 Subject: [PATCH 36/54] BUG:chain id compare --- params/qng_config.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/params/qng_config.go b/params/qng_config.go index 885a8a694d34..99ded40db7eb 100644 --- a/params/qng_config.go +++ b/params/qng_config.go @@ -86,40 +86,40 @@ func init() { } func IsQngNetwork(chainID *big.Int) bool { - if chainID == QngMainnetChainConfig.ChainID || - chainID == QngTestnetChainConfig.ChainID || - chainID == QngMixnetChainConfig.ChainID || - chainID == QngPrivnetChainConfig.ChainID { + if chainID.Cmp(QngMainnetChainConfig.ChainID) == 0 || + chainID.Cmp(QngTestnetChainConfig.ChainID) == 0 || + chainID.Cmp(QngMixnetChainConfig.ChainID) == 0 || + chainID.Cmp(QngPrivnetChainConfig.ChainID) == 0 { return true } return false } func IsAmanaNetwork(chainID *big.Int) bool { - if chainID == AmanaChainConfig.ChainID || - chainID == AmanaTestnetChainConfig.ChainID || - chainID == AmanaMixnetChainConfig.ChainID || - chainID == AmanaPrivnetChainConfig.ChainID { + if chainID.Cmp(AmanaChainConfig.ChainID) == 0 || + chainID.Cmp(AmanaTestnetChainConfig.ChainID) == 0 || + chainID.Cmp(AmanaMixnetChainConfig.ChainID) == 0 || + chainID.Cmp(AmanaPrivnetChainConfig.ChainID) == 0 { return true } return false } func IsFlanaNetwork(chainID *big.Int) bool { - if chainID == FlanaChainConfig.ChainID || - chainID == FlanaTestnetChainConfig.ChainID || - chainID == FlanaMixnetChainConfig.ChainID || - chainID == FlanaPrivnetChainConfig.ChainID { + if chainID.Cmp(FlanaChainConfig.ChainID) == 0 || + chainID.Cmp(FlanaTestnetChainConfig.ChainID) == 0 || + chainID.Cmp(FlanaMixnetChainConfig.ChainID) == 0 || + chainID.Cmp(FlanaPrivnetChainConfig.ChainID) == 0 { return true } return false } func IsMizanaNetwork(chainID *big.Int) bool { - if chainID == MizanaChainConfig.ChainID || - chainID == MizanaTestnetChainConfig.ChainID || - chainID == MizanaMixnetChainConfig.ChainID || - chainID == MizanaPrivnetChainConfig.ChainID { + if chainID.Cmp(MizanaChainConfig.ChainID) == 0 || + chainID.Cmp(MizanaTestnetChainConfig.ChainID) == 0 || + chainID.Cmp(MizanaMixnetChainConfig.ChainID) == 0 || + chainID.Cmp(MizanaPrivnetChainConfig.ChainID) == 0 { return true } return false From f8dcbdab14cd1f55134067caa81348b552c949ec Mon Sep 17 00:00:00 2001 From: jin <35813306+lochjin@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:33:10 +0800 Subject: [PATCH 37/54] internal/web3ext: fix parameter count of miner_start (#27400) --- internal/web3ext/web3ext.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index a2fc3b5a9040..ea4abef1f6f8 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -641,8 +641,6 @@ web3._extend({ new web3._extend.Method({ name: 'start', call: 'miner_start', - params: 1, - inputFormatter: [null] }), new web3._extend.Method({ name: 'stop', From fefd13d66d189e2bfe34f26f6faf1666b2e4bb3e Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 1 Jun 2023 22:30:01 +0800 Subject: [PATCH 38/54] txpool for qng --- core/txpool/txpool_qng.go | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 core/txpool/txpool_qng.go diff --git a/core/txpool/txpool_qng.go b/core/txpool/txpool_qng.go new file mode 100644 index 000000000000..945daa287f5e --- /dev/null +++ b/core/txpool/txpool_qng.go @@ -0,0 +1,5 @@ +package txpool + +func (pool *TxPool) All() *lookup { + return pool.all +} From 2d5644b59d4db5e1b2c365f4232dd1e257efd90d Mon Sep 17 00:00:00 2001 From: Jin Date: Fri, 16 Jun 2023 07:30:15 +0800 Subject: [PATCH 39/54] txpool:reheap the priced list if london fork not enabled --- core/txpool/txpool.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 8bd13a5fc87c..bb897fb20c85 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -1277,9 +1277,13 @@ func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirt // because of another transaction (e.g. higher gas price). if reset != nil { pool.demoteUnexecutables() - if reset.newHead != nil && pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { - pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) - pool.priced.SetBaseFee(pendingBaseFee) + if reset.newHead != nil { + if pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { + pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) + pool.priced.SetBaseFee(pendingBaseFee) + } else { + pool.priced.Reheap() + } } // Update all accounts to the latest known pending nonce nonces := make(map[common.Address]uint64, len(pool.pending)) From 8a933a5cd9a2fc83174112b00cb264040c1f993c Mon Sep 17 00:00:00 2001 From: Jin Date: Thu, 6 Jul 2023 10:41:49 +0800 Subject: [PATCH 40/54] core/txpool: handle missing head in reset --- core/txpool/txpool.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index bb897fb20c85..211ea9d19d76 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -1354,6 +1354,14 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) // We still need to update the current state s.th. the lost transactions can be readded by the user } else { + if add == nil { + // if the new head is nil, it means that something happened between + // the firing of newhead-event and _now_: most likely a + // reorg caused by sync-reversion or explicit sethead back to an + // earlier block. + log.Warn("New head missing in txpool reset", "number", newHead.Number, "hash", newHead.Hash()) + return + } for rem.NumberU64() > add.NumberU64() { discarded = append(discarded, rem.Transactions()...) if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { From a0491a04b5beb295bc90d4408881ac4aaa00712d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 06:50:09 -0400 Subject: [PATCH 41/54] deps: update supranational/blst to 0.3.11 (#27890) build(deps): bump github.com/supranational/blst Bumps [github.com/supranational/blst](https://github.com/supranational/blst) from 0.3.11-0.20230406105308-e9dfc5ee724b to 0.3.11. - [Release notes](https://github.com/supranational/blst/releases) - [Commits](https://github.com/supranational/blst/commits/v0.3.11) --- updated-dependencies: - dependency-name: github.com/supranational/blst dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9c7121c0355c..7694f6eeaf47 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 github.com/stretchr/testify v1.8.1 - github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b + github.com/supranational/blst v0.3.11 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.24.1 diff --git a/go.sum b/go.sum index 591764e65830..a54b22e3dc0d 100644 --- a/go.sum +++ b/go.sum @@ -426,8 +426,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b h1:u49mjRnygnB34h8OKbnNJFVUtWSKIKb1KukdV8bILUM= -github.com/supranational/blst v0.3.11-0.20230406105308-e9dfc5ee724b/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= From 55863ce0d811d490888b297d36dae2673ce9409b Mon Sep 17 00:00:00 2001 From: imulmat4 <117636097+imulmat4@users.noreply.github.com> Date: Fri, 11 Aug 2023 02:12:26 -0500 Subject: [PATCH 42/54] core/txpool/blobpool: fix metrics name for prometheus export (#27901) --- core/txpool/blobpool/metrics.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/txpool/blobpool/metrics.go b/core/txpool/blobpool/metrics.go index 280913b3a916..070cc5ca4712 100644 --- a/core/txpool/blobpool/metrics.go +++ b/core/txpool/blobpool/metrics.go @@ -35,15 +35,15 @@ var ( // The below metrics track the per-shelf metrics for the primary blob store // and the temporary limbo store. - shelfDatausedGaugeName = "blobpool/shelf-%d/dataused" - shelfDatagapsGaugeName = "blobpool/shelf-%d/datagaps" - shelfSlotusedGaugeName = "blobpool/shelf-%d/slotused" - shelfSlotgapsGaugeName = "blobpool/shelf-%d/slotgaps" + shelfDatausedGaugeName = "blobpool/shelf_%d/dataused" + shelfDatagapsGaugeName = "blobpool/shelf_%d/datagaps" + shelfSlotusedGaugeName = "blobpool/shelf_%d/slotused" + shelfSlotgapsGaugeName = "blobpool/shelf_%d/slotgaps" - limboShelfDatausedGaugeName = "blobpool/limbo/shelf-%d/dataused" - limboShelfDatagapsGaugeName = "blobpool/limbo/shelf-%d/datagaps" - limboShelfSlotusedGaugeName = "blobpool/limbo/shelf-%d/slotused" - limboShelfSlotgapsGaugeName = "blobpool/limbo/shelf-%d/slotgaps" + limboShelfDatausedGaugeName = "blobpool/limbo/shelf_%d/dataused" + limboShelfDatagapsGaugeName = "blobpool/limbo/shelf_%d/datagaps" + limboShelfSlotusedGaugeName = "blobpool/limbo/shelf_%d/slotused" + limboShelfSlotgapsGaugeName = "blobpool/limbo/shelf_%d/slotgaps" // The oversized metrics aggregate the shelf stats above the max blob count // limits to track transactions that are just huge, but don't contain blobs. From 770db145ab9b37187b03456246496766ea769f54 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 11 Aug 2023 21:15:49 +0800 Subject: [PATCH 43/54] ethdb/leveldb: support more than 7 levels in metrics (#27904) --- ethdb/leveldb/leveldb.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 190327accb75..c0e0eb250a75 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -75,7 +75,7 @@ type Database struct { seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt manualMemAllocGauge metrics.Gauge // Gauge to track the amount of memory that has been manually allocated (not a part of runtime/GC) - levelsGauge [7]metrics.Gauge // Gauge for tracking the number of tables in levels + levelsGauge []metrics.Gauge // Gauge for tracking the number of tables in levels quitLock sync.Mutex // Mutex protecting the quit channel access quitChan chan chan error // Quit channel to stop the metrics collection before closing the database @@ -146,13 +146,8 @@ func NewCustom(file string, namespace string, customize func(options *opt.Option ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) ldb.manualMemAllocGauge = metrics.NewRegisteredGauge(namespace+"memory/manualalloc", nil) - // leveldb has only up to 7 levels - for i := range ldb.levelsGauge { - ldb.levelsGauge[i] = metrics.NewRegisteredGauge(namespace+fmt.Sprintf("tables/level%v", i), nil) - } - // Start up the metrics gathering and return - go ldb.meter(metricsGatheringInterval) + go ldb.meter(metricsGatheringInterval, namespace) return ldb, nil } @@ -271,7 +266,7 @@ func (db *Database) Path() string { // meter periodically retrieves internal leveldb counters and reports them to // the metrics subsystem. -func (db *Database) meter(refresh time.Duration) { +func (db *Database) meter(refresh time.Duration, namespace string) { // Create the counters to store current and previous compaction values compactions := make([][]int64, 2) for i := 0; i < 2; i++ { @@ -360,8 +355,11 @@ func (db *Database) meter(refresh time.Duration) { db.nonlevel0CompGauge.Update(int64(stats.NonLevel0Comp)) db.seekCompGauge.Update(int64(stats.SeekComp)) - // update tables amount for i, tables := range stats.LevelTablesCounts { + // Append metrics for additional layers + if i >= len(db.levelsGauge) { + db.levelsGauge = append(db.levelsGauge, metrics.NewRegisteredGauge(namespace+fmt.Sprintf("tables/level%v", i), nil)) + } db.levelsGauge[i].Update(int64(tables)) } From 811a674059af075203b4897fe995f1664e954048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 12 Aug 2023 01:04:12 +0300 Subject: [PATCH 44/54] all: update golang/x/ext and fix slice sorting fallout (#27909) The Go authors updated golang/x/ext to change the function signature of the slices sort method. It's an entire shitshow now because x/ext is not tagged, so everyone's codebase just picked a new version that some other dep depends on, causing our code to fail building. This PR updates the dep on our code too and does all the refactorings to follow upstream... --- accounts/keystore/account_cache.go | 4 +-- cmd/devp2p/dns_route53.go | 12 +++++-- cmd/devp2p/nodeset.go | 14 +++++--- common/types.go | 12 +++---- consensus/clique/snapshot.go | 2 +- consensus/clique/snapshot_test.go | 2 +- core/blockchain.go | 4 +-- core/rawdb/accessors_chain.go | 4 +-- core/rawdb/chain_iterator_test.go | 8 ++--- core/state/snapshot/difflayer.go | 4 +-- core/state/snapshot/iterator_fast.go | 18 +++++++---- eth/api_debug_test.go | 2 +- eth/gasprice/feehistory.go | 4 +-- eth/gasprice/gasprice.go | 6 ++-- eth/protocols/snap/sync_test.go | 16 +++++----- eth/tracers/api_test.go | 2 +- ethdb/dbtest/testsuite.go | 2 +- go.mod | 4 +-- go.sum | 8 ++--- internal/ethapi/api_test.go | 2 +- les/servingqueue.go | 10 ++++-- les/utils/limiter.go | 10 ++++-- metrics/writer.go | 8 ++--- metrics/writer_test.go | 2 +- p2p/discover/table_util_test.go | 8 ++--- p2p/discover/v4_lookup_test.go | 4 +-- p2p/discover/v5_udp_test.go | 4 +-- p2p/dnsdisc/tree.go | 4 +-- p2p/peer.go | 2 +- p2p/protocol.go | 15 ++++++--- p2p/server.go | 2 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 4 +-- tests/fuzzers/stacktrie/trie_fuzzer.go | 4 +-- trie/iterator_test.go | 4 +-- trie/proof_test.go | 32 +++++++++---------- trie/triedb/pathdb/history.go | 6 ++-- trie/triedb/pathdb/testutils.go | 2 +- trie/trienode/node.go | 6 ++-- 38 files changed, 144 insertions(+), 113 deletions(-) diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index a88b0fa5f369..4ed1439514e1 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -40,8 +40,8 @@ import ( const minReloadInterval = 2 * time.Second // byURL defines the sorting order for accounts. -func byURL(a, b accounts.Account) bool { - return a.URL.Cmp(b.URL) < 0 +func byURL(a, b accounts.Account) int { + return a.URL.Cmp(b.URL) } // AmbiguousAddrError is returned when attempting to unlock diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index eae6f70f5827..21a32f9414e4 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -288,11 +288,17 @@ func makeDeletionChanges(records map[string]recordSet, keep map[string]string) [ // sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order. func sortChanges(changes []types.Change) { score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3} - slices.SortFunc(changes, func(a, b types.Change) bool { + slices.SortFunc(changes, func(a, b types.Change) int { if a.Action == b.Action { - return *a.ResourceRecordSet.Name < *b.ResourceRecordSet.Name + return strings.Compare(*a.ResourceRecordSet.Name, *b.ResourceRecordSet.Name) } - return score[string(a.Action)] < score[string(b.Action)] + if score[string(a.Action)] < score[string(b.Action)] { + return -1 + } + if score[string(a.Action)] > score[string(b.Action)] { + return 1 + } + return 0 }) } diff --git a/cmd/devp2p/nodeset.go b/cmd/devp2p/nodeset.go index c040c1a908ba..7360dc5bcfd9 100644 --- a/cmd/devp2p/nodeset.go +++ b/cmd/devp2p/nodeset.go @@ -77,8 +77,8 @@ func (ns nodeSet) nodes() []*enode.Node { result = append(result, n.N) } // Sort by ID. - slices.SortFunc(result, func(a, b *enode.Node) bool { - return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) < 0 + slices.SortFunc(result, func(a, b *enode.Node) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) return result } @@ -103,8 +103,14 @@ func (ns nodeSet) topN(n int) nodeSet { for _, v := range ns { byscore = append(byscore, v) } - slices.SortFunc(byscore, func(a, b nodeJSON) bool { - return a.Score >= b.Score + slices.SortFunc(byscore, func(a, b nodeJSON) int { + if a.Score > b.Score { + return -1 + } + if a.Score < b.Score { + return 1 + } + return 0 }) result := make(nodeSet, n) for _, v := range byscore[:n] { diff --git a/common/types.go b/common/types.go index 93fb0904545e..bf74e43716a1 100644 --- a/common/types.go +++ b/common/types.go @@ -65,9 +65,9 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } -// Less compares two hashes. -func (h Hash) Less(other Hash) bool { - return bytes.Compare(h[:], other[:]) < 0 +// Cmp compares two hashes. +func (h Hash) Cmp(other Hash) int { + return bytes.Compare(h[:], other[:]) } // Bytes gets the byte representation of the underlying hash. @@ -231,9 +231,9 @@ func IsHexAddress(s string) bool { return len(s) == 2*AddressLength && isHex(s) } -// Less compares two addresses. -func (a Address) Less(other Address) bool { - return bytes.Compare(a[:], other[:]) < 0 +// Cmp compares two addresses. +func (a Address) Cmp(other Address) int { + return bytes.Compare(a[:], other[:]) } // Bytes gets the string representation of the underlying address. diff --git a/consensus/clique/snapshot.go b/consensus/clique/snapshot.go index 0f1b9f4b3a16..a97115121b82 100644 --- a/consensus/clique/snapshot.go +++ b/consensus/clique/snapshot.go @@ -308,7 +308,7 @@ func (s *Snapshot) signers() []common.Address { for sig := range s.Signers { sigs = append(sigs, sig) } - slices.SortFunc(sigs, common.Address.Less) + slices.SortFunc(sigs, common.Address.Cmp) return sigs } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index f30900f274a6..26cebe008a88 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -53,7 +53,7 @@ func (ap *testerAccountPool) checkpoint(header *types.Header, signers []string) for i, signer := range signers { auths[i] = ap.address(signer) } - slices.SortFunc(auths, common.Address.Less) + slices.SortFunc(auths, common.Address.Cmp) for i, auth := range auths { copy(header.Extra[extraVanity+i*common.AddressLength:], auth.Bytes()) } diff --git a/core/blockchain.go b/core/blockchain.go index 3952c31b688f..ba6eb574667c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -996,8 +996,8 @@ func (bc *BlockChain) procFutureBlocks() { } } if len(blocks) > 0 { - slices.SortFunc(blocks, func(a, b *types.Block) bool { - return a.NumberU64() < b.NumberU64() + slices.SortFunc(blocks, func(a, b *types.Block) int { + return a.Number().Cmp(b.Number()) }) // Insert one by one as chain insertion needs contiguous ancestry between blocks for i := range blocks { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 9eeaaee941ff..c7abd44f365c 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -902,9 +902,9 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { Header: block.Header(), Body: block.Body(), }) - slices.SortFunc(badBlocks, func(a, b *badBlock) bool { + slices.SortFunc(badBlocks, func(a, b *badBlock) int { // Note: sorting in descending number order. - return a.Header.Number.Uint64() >= b.Header.Number.Uint64() + return -a.Header.Number.Cmp(b.Header.Number) }) if len(badBlocks) > badBlockToKeep { badBlocks = badBlocks[:badBlockToKeep] diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 9e18c8605cf1..9580cd92a873 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -19,12 +19,12 @@ package rawdb import ( "math/big" "reflect" + "sort" "sync" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "golang.org/x/exp/slices" ) func TestChainIterator(t *testing.T) { @@ -92,11 +92,9 @@ func TestChainIterator(t *testing.T) { } } if !c.reverse { - slices.Sort(numbers) + sort.Ints(numbers) } else { - slices.SortFunc(numbers, func(a, b int) bool { - return a > b // Sort descending - }) + sort.Sort(sort.Reverse(sort.IntSlice(numbers))) } if !reflect.DeepEqual(numbers, c.expect) { t.Fatalf("Case %d failed, visit element mismatch, want %v, got %v", i, c.expect, numbers) diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index b10b43b1ae58..b6aca599c509 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -525,7 +525,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.accountList = append(dl.accountList, hash) } } - slices.SortFunc(dl.accountList, common.Hash.Less) + slices.SortFunc(dl.accountList, common.Hash.Cmp) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList } @@ -563,7 +563,7 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) for k := range storageMap { storageList = append(storageList, k) } - slices.SortFunc(storageList, common.Hash.Less) + slices.SortFunc(storageList, common.Hash.Cmp) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) return storageList, destructed diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 339f930ffd81..0502d9cf8596 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -33,19 +33,25 @@ type weightedIterator struct { priority int } -func (it *weightedIterator) Less(other *weightedIterator) bool { +func (it *weightedIterator) Cmp(other *weightedIterator) int { // Order the iterators primarily by the account hashes hashI := it.it.Hash() hashJ := other.it.Hash() switch bytes.Compare(hashI[:], hashJ[:]) { case -1: - return true + return -1 case 1: - return false + return 1 } // Same account/storage-slot in multiple layers, split by priority - return it.priority < other.priority + if it.priority < other.priority { + return -1 + } + if it.priority > other.priority { + return 1 + } + return 0 } // fastIterator is a more optimized multi-layer iterator which maintains a @@ -155,9 +161,7 @@ func (fi *fastIterator) init() { } } // Re-sort the entire list - slices.SortFunc(fi.iterators, func(a, b *weightedIterator) bool { - return a.Less(b) - }) + slices.SortFunc(fi.iterators, func(a, b *weightedIterator) int { return a.Cmp(b) }) fi.initiated = false } diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 4b08bd4fc57d..6a1b537c25ce 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -105,7 +105,7 @@ func TestAccountRange(t *testing.T) { } // Test to see if it's possible to recover from the middle of the previous // set and get an even split between the first and second sets. - slices.SortFunc(hList, common.Hash.Less) + slices.SortFunc(hList, common.Hash.Cmp) middleH := hList[AccountRangeMaxResults/2] middleResult := accountRangeTest(t, &trie, sdb, middleH, AccountRangeMaxResults, AccountRangeMaxResults) missing, infirst, insecond := 0, 0, 0 diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index b9cee45ae77c..226991b24b2c 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -111,8 +111,8 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { reward, _ := tx.EffectiveGasTip(bf.block.BaseFee()) sorter[i] = txGasAndReward{gasUsed: bf.receipts[i].GasUsed, reward: reward} } - slices.SortStableFunc(sorter, func(a, b txGasAndReward) bool { - return a.reward.Cmp(b.reward) < 0 + slices.SortStableFunc(sorter, func(a, b txGasAndReward) int { + return a.reward.Cmp(b.reward) }) var txIndex int diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 9eccd7242862..b71964981145 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -208,7 +208,7 @@ func (oracle *Oracle) SuggestTipCap(ctx context.Context) (*big.Int, error) { } price := lastPrice if len(results) > 0 { - slices.SortFunc(results, func(a, b *big.Int) bool { return a.Cmp(b) < 0 }) + slices.SortFunc(results, func(a, b *big.Int) int { return a.Cmp(b) }) price = results[(len(results)-1)*oracle.percentile/100] } if price.Cmp(oracle.maxPrice) > 0 { @@ -247,12 +247,12 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit sortedTxs := make([]*types.Transaction, len(txs)) copy(sortedTxs, txs) baseFee := block.BaseFee() - slices.SortFunc(sortedTxs, func(a, b *types.Transaction) bool { + slices.SortFunc(sortedTxs, func(a, b *types.Transaction) int { // It's okay to discard the error because a tx would never be // accepted into a block with an invalid effective tip. tip1, _ := a.EffectiveGasTip(baseFee) tip2, _ := b.EffectiveGasTip(baseFee) - return tip1.Cmp(tip2) < 0 + return tip1.Cmp(tip2) }) var prices []*big.Int diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 79b1cad25b0b..0aa6fd8730dd 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1321,8 +1321,8 @@ type kv struct { k, v []byte } -func (k *kv) less(other *kv) bool { - return bytes.Compare(k.k, other.k) < 0 +func (k *kv) cmp(other *kv) int { + return bytes.Compare(k.k, other.k) } func key32(i uint64) []byte { @@ -1382,7 +1382,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { accTrie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit the state changes into db and re-create the trie // for accessing later. @@ -1444,7 +1444,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { accTrie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit the state changes into db and re-create the trie // for accessing later. @@ -1491,7 +1491,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit account trie root, set, _ := accTrie.Commit(true) @@ -1556,7 +1556,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Commit account trie root, set, _ := accTrie.Commit(true) @@ -1600,7 +1600,7 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas trie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) root, nodes, _ := trie.Commit(false) return root, nodes, entries } @@ -1651,7 +1651,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo trie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) root, nodes, _ := trie.Commit(false) return root, nodes, entries } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 11d7ff4b694e..0f78af9a01e5 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -790,7 +790,7 @@ func newAccounts(n int) (accounts []Account) { addr := crypto.PubkeyToAddress(key.PublicKey) accounts = append(accounts, Account{key: key, addr: addr}) } - slices.SortFunc(accounts, func(a, b Account) bool { return a.addr.Less(b.addr) }) + slices.SortFunc(accounts, func(a, b Account) int { return a.addr.Cmp(b.addr) }) return accounts } diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index ac10ddde93b2..0d3d5f5aa694 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -527,7 +527,7 @@ func makeDataset(size, ksize, vsize int, order bool) ([][]byte, [][]byte) { vals = append(vals, randBytes(vsize)) } if order { - slices.SortFunc(keys, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 }) + slices.SortFunc(keys, func(a, b []byte) int { return bytes.Compare(a, b) }) } return keys, vals } diff --git a/go.mod b/go.mod index 7694f6eeaf47..e4273ef5b034 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/urfave/cli/v2 v2.24.1 go.uber.org/automaxprocs v1.5.2 golang.org/x/crypto v0.9.0 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc + golang.org/x/exp v0.0.0-20230810033253-352e893a4cad golang.org/x/sync v0.3.0 golang.org/x/sys v0.9.0 golang.org/x/text v0.9.0 @@ -125,7 +125,7 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/mod v0.10.0 // indirect + golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index a54b22e3dc0d..50271e857fdf 100644 --- a/go.sum +++ b/go.sum @@ -479,8 +479,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= +golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -490,8 +490,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 671b9b2e4972..b0877dc37849 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -817,7 +817,7 @@ func newAccounts(n int) (accounts []Account) { addr := crypto.PubkeyToAddress(key.PublicKey) accounts = append(accounts, Account{key: key, addr: addr}) } - slices.SortFunc(accounts, func(a, b Account) bool { return a.addr.Less(b.addr) }) + slices.SortFunc(accounts, func(a, b Account) int { return a.addr.Cmp(b.addr) }) return accounts } diff --git a/les/servingqueue.go b/les/servingqueue.go index c25925403ad7..68cad9cb5a17 100644 --- a/les/servingqueue.go +++ b/les/servingqueue.go @@ -215,8 +215,14 @@ func (sq *servingQueue) freezePeers() { tasks.list = append(tasks.list, task) tasks.sumTime += task.expTime } - slices.SortFunc(peerList, func(a, b *peerTasks) bool { - return a.priority < b.priority + slices.SortFunc(peerList, func(a, b *peerTasks) int { + if a.priority < b.priority { + return -1 + } + if a.priority > b.priority { + return 1 + } + return 0 }) drop := true for _, tasks := range peerList { diff --git a/les/utils/limiter.go b/les/utils/limiter.go index 155bd801500a..70b7ff64f721 100644 --- a/les/utils/limiter.go +++ b/les/utils/limiter.go @@ -369,8 +369,14 @@ func (l *Limiter) dropRequests() { priority: w / float64(nq.sumCost), }) } - slices.SortFunc(list, func(a, b dropListItem) bool { - return a.priority < b.priority + slices.SortFunc(list, func(a, b dropListItem) int { + if a.priority < b.priority { + return -1 + } + if a.priority < b.priority { + return 1 + } + return 0 }) for _, item := range list { for _, request := range item.nq.queue { diff --git a/metrics/writer.go b/metrics/writer.go index 3d60485b6b5b..82434e9d1d62 100644 --- a/metrics/writer.go +++ b/metrics/writer.go @@ -3,6 +3,7 @@ package metrics import ( "fmt" "io" + "strings" "time" "golang.org/x/exp/slices" @@ -23,8 +24,7 @@ func WriteOnce(r Registry, w io.Writer) { r.Each(func(name string, i interface{}) { namedMetrics = append(namedMetrics, namedMetric{name, i}) }) - - slices.SortFunc(namedMetrics, namedMetric.less) + slices.SortFunc(namedMetrics, namedMetric.cmp) for _, namedMetric := range namedMetrics { switch metric := namedMetric.m.(type) { case Counter: @@ -92,6 +92,6 @@ type namedMetric struct { m interface{} } -func (m namedMetric) less(other namedMetric) bool { - return m.name < other.name +func (m namedMetric) cmp(other namedMetric) int { + return strings.Compare(m.name, other.name) } diff --git a/metrics/writer_test.go b/metrics/writer_test.go index a4c92addc9b9..8376bf8975c8 100644 --- a/metrics/writer_test.go +++ b/metrics/writer_test.go @@ -14,7 +14,7 @@ func TestMetricsSorting(t *testing.T) { {name: "ggg"}, } - slices.SortFunc(namedMetrics, namedMetric.less) + slices.SortFunc(namedMetrics, namedMetric.cmp) for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { if namedMetrics[i].name != name { t.Fail() diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 52544a2c9348..8f3813bdcf0f 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -217,14 +217,14 @@ func nodeEqual(n1 *enode.Node, n2 *enode.Node) bool { } func sortByID(nodes []*enode.Node) { - slices.SortFunc(nodes, func(a, b *enode.Node) bool { - return string(a.ID().Bytes()) < string(b.ID().Bytes()) + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) } func sortedByDistanceTo(distbase enode.ID, slice []*node) bool { - return slices.IsSortedFunc(slice, func(a, b *node) bool { - return enode.DistCmp(distbase, a.ID(), b.ID()) < 0 + return slices.IsSortedFunc(slice, func(a, b *node) int { + return enode.DistCmp(distbase, a.ID(), b.ID()) }) } diff --git a/p2p/discover/v4_lookup_test.go b/p2p/discover/v4_lookup_test.go index 83c462628867..1f9ad69d0a23 100644 --- a/p2p/discover/v4_lookup_test.go +++ b/p2p/discover/v4_lookup_test.go @@ -302,8 +302,8 @@ func (tn *preminedTestnet) closest(n int) (nodes []*enode.Node) { nodes = append(nodes, tn.node(d, i)) } } - slices.SortFunc(nodes, func(a, b *enode.Node) bool { - return enode.DistCmp(tn.target.id(), a.ID(), b.ID()) < 0 + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return enode.DistCmp(tn.target.id(), a.ID(), b.ID()) }) return nodes[:n] } diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 887d46c4975a..880b71a991ce 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -61,8 +61,8 @@ func TestUDPv5_lookupE2E(t *testing.T) { for i := range nodes { expectedResult[i] = nodes[i].Self() } - slices.SortFunc(expectedResult, func(a, b *enode.Node) bool { - return enode.DistCmp(target.ID(), a.ID(), b.ID()) < 0 + slices.SortFunc(expectedResult, func(a, b *enode.Node) int { + return enode.DistCmp(target.ID(), a.ID(), b.ID()) }) // Do the lookup. diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index f91aff458fb0..06b7681f18ae 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -214,8 +214,8 @@ func (t *Tree) build(entries []entry) entry { } func sortByID(nodes []*enode.Node) []*enode.Node { - slices.SortFunc(nodes, func(a, b *enode.Node) bool { - return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) < 0 + slices.SortFunc(nodes, func(a, b *enode.Node) int { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) }) return nodes } diff --git a/p2p/peer.go b/p2p/peer.go index e800be80fc6e..65a7903f5863 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -386,7 +386,7 @@ func countMatchingProtocols(protocols []Protocol, caps []Cap) int { // matchProtocols creates structures for matching named subprotocols. func matchProtocols(protocols []Protocol, caps []Cap, rw MsgReadWriter) map[string]*protoRW { - slices.SortFunc(caps, Cap.Less) + slices.SortFunc(caps, Cap.Cmp) offset := baseProtocolLength result := make(map[string]*protoRW) diff --git a/p2p/protocol.go b/p2p/protocol.go index 4b3cca6d4a09..9bb6785a2250 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -18,6 +18,7 @@ package p2p import ( "fmt" + "strings" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" @@ -77,10 +78,16 @@ func (cap Cap) String() string { return fmt.Sprintf("%s/%d", cap.Name, cap.Version) } -// Less defines the canonical sorting order of capabilities. -func (cap Cap) Less(other Cap) bool { +// Cmp defines the canonical sorting order of capabilities. +func (cap Cap) Cmp(other Cap) int { if cap.Name == other.Name { - return cap.Version < other.Version + if cap.Version < other.Version { + return -1 + } + if cap.Version > other.Version { + return 1 + } + return 0 } - return cap.Name < other.Name + return strings.Compare(cap.Name, other.Name) } diff --git a/p2p/server.go b/p2p/server.go index d4e2be67830e..8f42765a8c26 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -510,7 +510,7 @@ func (srv *Server) setupLocalNode() error { for _, p := range srv.Protocols { srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) } - slices.SortFunc(srv.ourHandshake.Caps, Cap.Less) + slices.SortFunc(srv.ourHandshake.Caps, Cap.Cmp) // Create the local node. db, err := enode.OpenDB(srv.NodeDatabase) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index d5aee206da66..aa81e5c9d9b1 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -98,8 +98,8 @@ func (f *fuzzer) fuzz() int { if len(entries) <= 1 { return 0 } - slices.SortFunc(entries, func(a, b *kv) bool { - return bytes.Compare(a.k, b.k) < 0 + slices.SortFunc(entries, func(a, b *kv) int { + return bytes.Compare(a.k, b.k) }) var ok = 0 diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 1eb639f6441a..391bdf300b72 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -182,8 +182,8 @@ func (f *fuzzer) fuzz() int { dbA.Commit(rootA, false) // Stacktrie requires sorted insertion - slices.SortFunc(vals, func(a, b kv) bool { - return bytes.Compare(a.k, b.k) < 0 + slices.SortFunc(vals, func(a, b kv) int { + return bytes.Compare(a.k, b.k) }) for _, kv := range vals { if f.debugging { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index bf20c00c6436..e711ffb81583 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -84,8 +84,8 @@ type kv struct { t bool } -func (k *kv) less(other *kv) bool { - return bytes.Compare(k.k, other.k) < 0 +func (k *kv) cmp(other *kv) int { + return bytes.Compare(k.k, other.k) } func TestIteratorLargeData(t *testing.T) { diff --git a/trie/proof_test.go b/trie/proof_test.go index d8effd0d04b2..e8ea116c8803 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -173,7 +173,7 @@ func TestRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 @@ -206,7 +206,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 @@ -278,7 +278,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Case 1 start, end := 100, 200 @@ -335,7 +335,7 @@ func TestOneElementRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // One element with existent edge proof, both edge proofs // point to the SAME key. @@ -422,7 +422,7 @@ func TestAllElementsProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var k [][]byte var v [][]byte @@ -474,7 +474,7 @@ func TestSingleSideRangeProof(t *testing.T) { trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { @@ -509,7 +509,7 @@ func TestReverseSingleSideRangeProof(t *testing.T) { trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { @@ -543,7 +543,7 @@ func TestBadRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) @@ -646,7 +646,7 @@ func TestSameSideProofs(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) pos := 1000 first := decreaseKey(common.CopyBytes(entries[pos].k)) @@ -690,7 +690,7 @@ func TestHasRightElement(t *testing.T) { trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []struct { start int @@ -762,7 +762,7 @@ func TestEmptyRangeProof(t *testing.T) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var cases = []struct { pos int @@ -797,7 +797,7 @@ func TestBloatedProof(t *testing.T) { for _, kv := range kvs { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var keys [][]byte var vals [][]byte @@ -831,7 +831,7 @@ func TestEmptyValueRangeProof(t *testing.T) { for _, kv := range values { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Create a new entry with a slightly modified key mid := len(entries) / 2 @@ -875,7 +875,7 @@ func TestAllElementsEmptyValueRangeProof(t *testing.T) { for _, kv := range values { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) // Create a new entry with a slightly modified key mid := len(entries) / 2 @@ -981,7 +981,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) start := 2 end := start + size @@ -1018,7 +1018,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { for _, kv := range vals { entries = append(entries, kv) } - slices.SortFunc(entries, (*kv).less) + slices.SortFunc(entries, (*kv).cmp) var keys [][]byte var values [][]byte diff --git a/trie/triedb/pathdb/history.go b/trie/triedb/pathdb/history.go index c12fc5518472..ce82532507c7 100644 --- a/trie/triedb/pathdb/history.go +++ b/trie/triedb/pathdb/history.go @@ -262,20 +262,20 @@ func newHistory(root common.Hash, parent common.Hash, block uint64, states *trie for addr := range states.Accounts { accountList = append(accountList, addr) } - slices.SortFunc(accountList, func(a, b common.Address) bool { return a.Less(b) }) + slices.SortFunc(accountList, common.Address.Cmp) for addr, slots := range states.Storages { slist := make([]common.Hash, 0, len(slots)) for slotHash := range slots { slist = append(slist, slotHash) } - slices.SortFunc(slist, func(a, b common.Hash) bool { return a.Less(b) }) + slices.SortFunc(slist, common.Hash.Cmp) storageList[addr] = slist } for addr := range states.Incomplete { incomplete = append(incomplete, addr) } - slices.SortFunc(incomplete, func(a, b common.Address) bool { return a.Less(b) }) + slices.SortFunc(incomplete, common.Address.Cmp) return &history{ meta: &meta{ diff --git a/trie/triedb/pathdb/testutils.go b/trie/triedb/pathdb/testutils.go index cb3a240cc4f3..4406dbc52124 100644 --- a/trie/triedb/pathdb/testutils.go +++ b/trie/triedb/pathdb/testutils.go @@ -117,7 +117,7 @@ func hash(states map[common.Hash][]byte) (common.Hash, []byte) { for hash := range states { hs = append(hs, hash) } - slices.SortFunc(hs, func(a, b common.Hash) bool { return a.Less(b) }) + slices.SortFunc(hs, common.Hash.Cmp) var input []byte for _, hash := range hs { diff --git a/trie/trienode/node.go b/trie/trienode/node.go index 8998bcba060a..98d5588b6d3a 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -18,10 +18,10 @@ package trienode import ( "fmt" + "sort" "strings" "github.com/ethereum/go-ethereum/common" - "golang.org/x/exp/slices" ) // Node is a wrapper which contains the encoded blob of the trie node and its @@ -83,9 +83,7 @@ func (set *NodeSet) ForEachWithOrder(callback func(path string, n *Node)) { paths = append(paths, path) } // Bottom-up, the longest path first - slices.SortFunc(paths, func(a, b string) bool { - return a > b // Sort in reverse order - }) + sort.Sort(sort.Reverse(sort.StringSlice(paths))) for _, path := range paths { callback(path, set.Nodes[path]) } From f298ec4fe14bfaa99df76d64c76f7cd988519393 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 12 Aug 2023 00:21:46 +0200 Subject: [PATCH 45/54] go.mod, build: upgrade c-kzg-4844 (#27907) This upgrades to the latest release of ckzg, and also attempts to fix some blst-related build errors that occur on launchpad.net. --- build/ci.go | 19 ++++++++++++++----- build/deb/ethereum/deb.rules | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/build/env.go | 14 ++++++++------ internal/build/gotool.go | 4 ++-- 6 files changed, 28 insertions(+), 17 deletions(-) diff --git a/build/ci.go b/build/ci.go index 78b8bc69726b..242f4d841e5f 100644 --- a/build/ci.go +++ b/build/ci.go @@ -200,6 +200,7 @@ func doInstall(cmdline []string) { staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) + env := build.Env() // Configure the toolchain. tc := build.GoToolchain{GOARCH: *arch, CC: *cc} @@ -207,12 +208,16 @@ func doInstall(cmdline []string) { csdb := build.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb, dlgoVersion) } - // Disable CLI markdown doc generation in release builds and enable linking - // the CKZG library since we can make it portable here. - buildTags := []string{"urfave_cli_no_docs", "ckzg"} + + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + + // Enable linking the CKZG library since we can make it work with additional flags. + if env.UbuntuVersion != "trusty" { + buildTags = append(buildTags, "ckzg") + } // Configure the build. - env := build.Env() gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, @@ -298,10 +303,14 @@ func doTest(cmdline []string) { csdb := build.MustLoadChecksums("build/checksums.txt") tc.Root = build.DownloadGo(csdb, dlgoVersion) } - gotest := tc.Go("test", "-tags=ckzg") + gotest := tc.Go("test") + // CI needs a bit more time for the statetests (default 10m). gotest.Args = append(gotest.Args, "-timeout=20m") + // Enable CKZG backend in CI. + gotest.Args = append(gotest.Args, "-tags=ckzg") + // Test a single package at a time. CI builders are slow // and some tests run into timeouts under load. gotest.Args = append(gotest.Args, "-p", "1") diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 475e628b2f5e..daca793e555f 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -28,7 +28,7 @@ override_dh_auto_build: mv .mod $(GOPATH)/pkg/mod # A fresh Go was built, all dependency downloads faked, hope build works now - ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} -ubuntu {{.Distro}} override_dh_auto_test: diff --git a/go.mod b/go.mod index e4273ef5b034..d62f4ebfd5b0 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/deckarep/golang-set/v2 v2.1.0 github.com/docker/docker v1.6.2 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844 v0.3.0 + github.com/ethereum/c-kzg-4844 v0.3.1 github.com/fatih/color v1.7.0 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 diff --git a/go.sum b/go.sum index 50271e857fdf..8f91afee82bf 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844 v0.3.0 h1:3Y3hD6l5i0dEYsBL50C+Om644kve3pNqoAcvE26o9zI= -github.com/ethereum/c-kzg-4844 v0.3.0/go.mod h1:WI2Nd82DMZAAZI1wV2neKGost9EKjvbpQR9OqE5Qqa8= +github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= +github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= diff --git a/internal/build/env.go b/internal/build/env.go index d70c0d50a499..9cfcbb69b343 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -28,12 +28,13 @@ import ( var ( // These flags override values in build env. - GitCommitFlag = flag.String("git-commit", "", `Overrides git commit hash embedded into executables`) - GitBranchFlag = flag.String("git-branch", "", `Overrides git branch being built`) - GitTagFlag = flag.String("git-tag", "", `Overrides git tag being built`) - BuildnumFlag = flag.String("buildnum", "", `Overrides CI build number`) - PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`) - CronJobFlag = flag.Bool("cron-job", false, `Overrides cron job status of the build`) + GitCommitFlag = flag.String("git-commit", "", `Overrides git commit hash embedded into executables`) + GitBranchFlag = flag.String("git-branch", "", `Overrides git branch being built`) + GitTagFlag = flag.String("git-tag", "", `Overrides git tag being built`) + BuildnumFlag = flag.String("buildnum", "", `Overrides CI build number`) + PullRequestFlag = flag.Bool("pull-request", false, `Overrides pull request status of the build`) + CronJobFlag = flag.Bool("cron-job", false, `Overrides cron job status of the build`) + UbuntuVersionFlag = flag.String("ubuntu", "", `Sets the ubuntu version being built for`) ) // Environment contains metadata provided by the build environment. @@ -43,6 +44,7 @@ type Environment struct { Repo string // name of GitHub repo Commit, Date, Branch, Tag string // Git info Buildnum string + UbuntuVersion string // Ubuntu version being built for IsPullRequest bool IsCronJob bool } diff --git a/internal/build/gotool.go b/internal/build/gotool.go index b5452f80cc3c..296ba8c36ecb 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -54,8 +54,8 @@ func (g *GoToolchain) Go(command string, args ...string) *exec.Cmd { tool.Env = append(tool.Env, "CC="+os.Getenv("CC")) } // CKZG by default is not portable, append the necessary build flags to make - // it not rely on modern CPU instructions and enable linking against - tool.Env = append(tool.Env, "CGO_CFLAGS=-D__BLST_PORTABLE__") + // it not rely on modern CPU instructions and enable linking against. + tool.Env = append(tool.Env, "CGO_CFLAGS=-O2 -g -D__BLST_PORTABLE__") return tool } From a867039c53790bca68c44911c8a0d86dfdc503b5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 12 Aug 2023 10:32:50 +0200 Subject: [PATCH 46/54] internal/build: apply -ubuntu to env (#27910) --- internal/build/env.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/build/env.go b/internal/build/env.go index 9cfcbb69b343..0854fba2498d 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -170,5 +170,8 @@ func applyEnvFlags(env Environment) Environment { if *CronJobFlag { env.IsCronJob = true } + if *UbuntuVersionFlag != "" { + env.UbuntuVersion = *UbuntuVersionFlag + } return env } From bed84606583893fdb698cc1b5058cc47b4dbd837 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 12 Aug 2023 11:05:07 +0200 Subject: [PATCH 47/54] params: release go-ethereum v1.12.2 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 4d70038e4c8f..7417dc0be2f6 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 12 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 12 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 4ea6c11774783f92dcd8a068ef982a3b861db57f Mon Sep 17 00:00:00 2001 From: Jin Date: Wed, 13 Sep 2023 17:40:06 +0800 Subject: [PATCH 48/54] Optimize miner for qng --- miner/{interface.go => miner_qng.go} | 12 ++++++++++++ params/{qng_config.go => config_qng.go} | 0 2 files changed, 12 insertions(+) rename miner/{interface.go => miner_qng.go} (64%) rename params/{qng_config.go => config_qng.go} (100%) diff --git a/miner/interface.go b/miner/miner_qng.go similarity index 64% rename from miner/interface.go rename to miner/miner_qng.go index 2e1dc069952f..6706f068f6ef 100644 --- a/miner/interface.go +++ b/miner/miner_qng.go @@ -3,8 +3,10 @@ package miner import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "math/big" "time" ) @@ -24,3 +26,13 @@ type IMiner interface { SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription BuildPayload(args *BuildPayloadArgs) (*Payload, error) } + +type TransactionsByPriceAndNonce interface { + Peek() *txpool.LazyTransaction + Shift() + Pop() +} + +func NewTransactionsByPriceAndNonce(signer types.Signer, txs map[common.Address][]*txpool.LazyTransaction, baseFee *big.Int) TransactionsByPriceAndNonce { + return newTransactionsByPriceAndNonce(signer,txs,baseFee) +} \ No newline at end of file diff --git a/params/qng_config.go b/params/config_qng.go similarity index 100% rename from params/qng_config.go rename to params/config_qng.go From 916d6a441a866cb618ae826c220866de118899f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 14 Nov 2023 15:02:24 +0300 Subject: [PATCH 49/54] params: release Geth v1.15.5 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index bcab461a4358..5fb9631f1b3b 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 13 // Minor version component of the current release - VersionPatch = 5 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 13 // Minor version component of the current release + VersionPatch = 5 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From d4c2f911050d8e397f7a76792194d1ed0230e442 Mon Sep 17 00:00:00 2001 From: Jin Date: Mon, 18 Mar 2024 15:44:27 +0800 Subject: [PATCH 50/54] API for qng --- miner/miner_qng.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/miner/miner_qng.go b/miner/miner_qng.go index dc5cf4ed3482..351acfeb6be8 100644 --- a/miner/miner_qng.go +++ b/miner/miner_qng.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" + "github.com/holiman/uint256" "math/big" "time" ) @@ -23,12 +24,13 @@ type IMiner interface { PendingBlockAndReceipts() (*types.Block, types.Receipts) SetEtherbase(addr common.Address) SetGasCeil(ceil uint64) + SetGasTip(tip *big.Int) error SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription BuildPayload(args *BuildPayloadArgs) (*Payload, error) } type TransactionsByPriceAndNonce interface { - Peek() *txpool.LazyTransaction + Peek() (*txpool.LazyTransaction, *uint256.Int) Shift() Pop() } From d1d4e60dca98cbeff2c46e77e6696bdcdd345d9a Mon Sep 17 00:00:00 2001 From: Jin Date: Fri, 10 May 2024 15:13:28 +0800 Subject: [PATCH 51/54] go-secp256k1 --- crypto/signature_cgo.go | 2 +- go.mod | 3 ++- go.sum | 2 ++ tests/fuzzers/secp256k1/secp_test.go | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crypto/signature_cgo.go b/crypto/signature_cgo.go index 87289253c0ff..f359fc99ce52 100644 --- a/crypto/signature_cgo.go +++ b/crypto/signature_cgo.go @@ -24,8 +24,8 @@ import ( "errors" "fmt" + secp256k1 "github.com/Qitmeer/go-secp256k1" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/crypto/secp256k1" ) // Ecrecover returns the uncompressed public key that created the given signature. diff --git a/go.mod b/go.mod index 3cd0d82bdf7c..0431fa034735 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module github.com/ethereum/go-ethereum -go 1.21 +go 1.22.3 require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/Microsoft/go-winio v0.6.1 + github.com/Qitmeer/go-secp256k1 v0.5.1 github.com/VictoriaMetrics/fastcache v1.12.1 github.com/aws/aws-sdk-go-v2 v1.21.2 github.com/aws/aws-sdk-go-v2/config v1.18.45 diff --git a/go.sum b/go.sum index a7b4eb1c138a..8302100b40e8 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Qitmeer/go-secp256k1 v0.5.1 h1:t3u4VMSz7LRn9Ki1lecG7tyTNwbLGILt1CwdOAA5N7M= +github.com/Qitmeer/go-secp256k1 v0.5.1/go.mod h1:AkrSQDAMwdV4+pXUQnEAPvPbUiIN4DAHMAm0ueneJB8= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= diff --git a/tests/fuzzers/secp256k1/secp_test.go b/tests/fuzzers/secp256k1/secp_test.go index ca3039764b42..374a992a34dc 100644 --- a/tests/fuzzers/secp256k1/secp_test.go +++ b/tests/fuzzers/secp256k1/secp_test.go @@ -20,8 +20,8 @@ import ( "fmt" "testing" + secp256k1 "github.com/Qitmeer/go-secp256k1" "github.com/btcsuite/btcd/btcec/v2" - "github.com/ethereum/go-ethereum/crypto/secp256k1" ) func TestFuzzer(t *testing.T) { From b08c01334ed300da38314098b3a9fc56a1ea808d Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 14 May 2024 09:28:34 +0800 Subject: [PATCH 52/54] BUG:Unnecessary prompts about zero state root hash --- core/blockchain.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 56e00e85b615..4adbd159d483 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -711,11 +711,14 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ // noState represents if the target state requested for search // is unavailable and impossible to be recovered. - noState = !bc.HasState(root) && !bc.stateRecoverable(root) + noState = root == common.Hash{} start = time.Now() // Timestamp the rewinding is restarted logged = time.Now() // Timestamp last progress log was printed ) + if !noState { + noState = !bc.HasState(root) && !bc.stateRecoverable(root) + } // Rewind the head block tag until an available state is found. for { logger := log.Trace From 43b4a2264bc66d3e5a6880b29810e2f1af0d4c39 Mon Sep 17 00:00:00 2001 From: Jin Date: Sat, 18 May 2024 20:28:00 +0800 Subject: [PATCH 53/54] BUG:default consensus engine is nil --- eth/backend.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/backend.go b/eth/backend.go index 42106ba9deb7..53a4de64685e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -146,6 +146,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + if config.ConsensusEngine == nil { + config.ConsensusEngine = ethconfig.CreateDefaultConsensusEngine + } engine, err := config.ConsensusEngine(chainConfig, chainDb) if err != nil { return nil, err From 3e4823df44148850c574a3aca5024ea8540e8c81 Mon Sep 17 00:00:00 2001 From: Jin Date: Tue, 21 May 2024 22:16:00 +0800 Subject: [PATCH 54/54] Engine API for qng --- beacon/engine/types.go | 4 +- beacon/engine/types_qng.go | 69 +++++++++ eth/catalyst/api_qng.go | 277 +++++++++++++++++++++++++++++++++++++ 3 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 beacon/engine/types_qng.go create mode 100644 eth/catalyst/api_qng.go diff --git a/beacon/engine/types.go b/beacon/engine/types.go index a73691ca0576..c1f2ea48b8cc 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -75,6 +75,7 @@ type ExecutableData struct { Withdrawals []*types.Withdrawal `json:"withdrawals"` BlobGasUsed *uint64 `json:"blobGasUsed"` ExcessBlobGas *uint64 `json:"excessBlobGas"` + Difficulty *big.Int `json:"difficulty"` } // JSON type overrides for executableData. @@ -237,7 +238,7 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash, TxHash: types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)), ReceiptHash: params.ReceiptsRoot, Bloom: types.BytesToBloom(params.LogsBloom), - Difficulty: common.Big0, + Difficulty: params.Difficulty, Number: new(big.Int).SetUint64(params.Number), GasLimit: params.GasLimit, GasUsed: params.GasUsed, @@ -278,6 +279,7 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types. Withdrawals: block.Withdrawals(), BlobGasUsed: block.BlobGasUsed(), ExcessBlobGas: block.ExcessBlobGas(), + Difficulty: block.Difficulty(), } bundle := BlobsBundleV1{ Commitments: make([]hexutil.Bytes, 0), diff --git a/beacon/engine/types_qng.go b/beacon/engine/types_qng.go new file mode 100644 index 000000000000..cfea3e7b02d4 --- /dev/null +++ b/beacon/engine/types_qng.go @@ -0,0 +1,69 @@ +package engine + +import ( + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "math/big" +) + +func ExecutableDataToBlockQng(params ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (*types.Block, error) { + txs, err := decodeTransactions(params.Transactions) + if err != nil { + return nil, err + } + if len(params.LogsBloom) != 256 { + return nil, fmt.Errorf("invalid logsBloom length: %v", len(params.LogsBloom)) + } + // Check that baseFeePerGas is not negative or too big + if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) { + return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas) + } + var blobHashes []common.Hash + for _, tx := range txs { + blobHashes = append(blobHashes, tx.BlobHashes()...) + } + if len(blobHashes) != len(versionedHashes) { + return nil, fmt.Errorf("invalid number of versionedHashes: %v blobHashes: %v", versionedHashes, blobHashes) + } + for i := 0; i < len(blobHashes); i++ { + if blobHashes[i] != versionedHashes[i] { + return nil, fmt.Errorf("invalid versionedHash at %v: %v blobHashes: %v", i, versionedHashes, blobHashes) + } + } + // Only set withdrawalsRoot if it is non-nil. This allows CLs to use + // ExecutableData before withdrawals are enabled by marshaling + // Withdrawals as the json null value. + var withdrawalsRoot *common.Hash + if params.Withdrawals != nil { + h := types.DeriveSha(types.Withdrawals(params.Withdrawals), trie.NewStackTrie(nil)) + withdrawalsRoot = &h + } + header := &types.Header{ + ParentHash: params.ParentHash, + UncleHash: types.EmptyUncleHash, + Coinbase: params.FeeRecipient, + Root: params.StateRoot, + TxHash: types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)), + ReceiptHash: params.ReceiptsRoot, + Bloom: types.BytesToBloom(params.LogsBloom), + Difficulty: params.Difficulty, + Number: new(big.Int).SetUint64(params.Number), + GasLimit: params.GasLimit, + GasUsed: params.GasUsed, + Time: params.Timestamp, + BaseFee: params.BaseFeePerGas, + Extra: params.ExtraData, + MixDigest: params.Random, + WithdrawalsHash: withdrawalsRoot, + ExcessBlobGas: params.ExcessBlobGas, + BlobGasUsed: params.BlobGasUsed, + ParentBeaconRoot: beaconRoot, + } + block := types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: params.Withdrawals}) + if block.Hash() != params.BlockHash { + return nil, fmt.Errorf("blockhash mismatch, want %x, got %x", params.BlockHash, block.Hash()) + } + return block, nil +} diff --git a/eth/catalyst/api_qng.go b/eth/catalyst/api_qng.go new file mode 100644 index 000000000000..2fabafa99a82 --- /dev/null +++ b/eth/catalyst/api_qng.go @@ -0,0 +1,277 @@ +package catalyst + +import ( + "errors" + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" + "strconv" + "time" +) + +func (api *ConsensusAPI) ForkchoiceUpdatedQng(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + api.forkchoiceLock.Lock() + defer api.forkchoiceLock.Unlock() + + payloadVersion := engine.PayloadV1 + + log.Trace("Engine API request received", "method", "ForkchoiceUpdated", "head", update.HeadBlockHash, "finalized", update.FinalizedBlockHash, "safe", update.SafeBlockHash) + if update.HeadBlockHash == (common.Hash{}) { + log.Warn("Forkchoice requested update to zero hash") + return engine.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? + } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastForkchoiceLock.Lock() + api.lastForkchoiceUpdate = time.Now() + api.lastForkchoiceLock.Unlock() + + // Check whether we have the block yet in our database or not. If not, we'll + // need to either trigger a sync, or to reject this forkchoice update for a + // reason. + block := api.eth.BlockChain().GetBlockByHash(update.HeadBlockHash) + if block == nil { + // If this block was previously invalidated, keep rejecting it here too + if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { + return engine.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil + } + // If the head hash is unknown (was not given to us in a newPayload request), + // we cannot resolve the header, so not much to do. This could be extended in + // the future to resolve from the `eth` network, but it's an unexpected case + // that should be fixed, not papered over. + header := api.remoteBlocks.get(update.HeadBlockHash) + if header == nil { + log.Warn("Forkchoice requested unknown head", "hash", update.HeadBlockHash) + return engine.STATUS_SYNCING, nil + } + // If the finalized hash is known, we can direct the downloader to move + // potentially more data to the freezer from the get go. + finalized := api.remoteBlocks.get(update.FinalizedBlockHash) + + // Header advertised via a past newPayload request. Start syncing to it. + context := []interface{}{"number", header.Number, "hash", header.Hash()} + if update.FinalizedBlockHash != (common.Hash{}) { + if finalized == nil { + context = append(context, []interface{}{"finalized", "unknown"}...) + } else { + context = append(context, []interface{}{"finalized", finalized.Number}...) + } + } + log.Info("Forkchoice requested sync to new head", context...) + if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), header, finalized); err != nil { + return engine.STATUS_SYNCING, err + } + return engine.STATUS_SYNCING, nil + } + valid := func(id *engine.PayloadID) engine.ForkChoiceResponse { + return engine.ForkChoiceResponse{ + PayloadStatus: engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &update.HeadBlockHash}, + PayloadID: id, + } + } + if rawdb.ReadCanonicalHash(api.eth.ChainDb(), block.NumberU64()) != update.HeadBlockHash { + // Block is not canonical, set head. + if latestValid, err := api.eth.BlockChain().SetCanonical(block); err != nil { + return engine.ForkChoiceResponse{PayloadStatus: engine.PayloadStatusV1{Status: engine.INVALID, LatestValidHash: &latestValid}}, err + } + } else if api.eth.BlockChain().CurrentBlock().Hash() == update.HeadBlockHash { + // If the specified head matches with our local head, do nothing and keep + // generating the payload. It's a special corner case that a few slots are + // missing and we are requested to generate the payload in slot. + } else { + // If the head block is already in our canonical chain, the beacon client is + // probably resyncing. Ignore the update. + log.Info("Ignoring beacon update to old head", "number", block.NumberU64(), "hash", update.HeadBlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0)), "have", api.eth.BlockChain().CurrentBlock().Number) + return valid(nil), nil + } + api.eth.SetSynced() + + // If the beacon client also advertised a finalized block, mark the local + // chain final and completely in PoS mode. + if update.FinalizedBlockHash != (common.Hash{}) { + // If the finalized block is not in our canonical tree, something is wrong + finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash) + if finalBlock == nil { + log.Warn("Final block not available in database", "hash", update.FinalizedBlockHash) + return engine.STATUS_INVALID, engine.InvalidForkChoiceState.With(errors.New("final block not available in database")) + } else if rawdb.ReadCanonicalHash(api.eth.ChainDb(), finalBlock.NumberU64()) != update.FinalizedBlockHash { + log.Warn("Final block not in canonical chain", "number", block.NumberU64(), "hash", update.HeadBlockHash) + return engine.STATUS_INVALID, engine.InvalidForkChoiceState.With(errors.New("final block not in canonical chain")) + } + // Set the finalized block + api.eth.BlockChain().SetFinalized(finalBlock.Header()) + } + // Check if the safe block hash is in our canonical tree, if not something is wrong + if update.SafeBlockHash != (common.Hash{}) { + safeBlock := api.eth.BlockChain().GetBlockByHash(update.SafeBlockHash) + if safeBlock == nil { + log.Warn("Safe block not available in database") + return engine.STATUS_INVALID, engine.InvalidForkChoiceState.With(errors.New("safe block not available in database")) + } + if rawdb.ReadCanonicalHash(api.eth.ChainDb(), safeBlock.NumberU64()) != update.SafeBlockHash { + log.Warn("Safe block not in canonical chain") + return engine.STATUS_INVALID, engine.InvalidForkChoiceState.With(errors.New("safe block not in canonical chain")) + } + // Set the safe block + api.eth.BlockChain().SetSafe(safeBlock.Header()) + } + // If payload generation was requested, create a new block to be potentially + // sealed by the beacon client. The payload will be requested later, and we + // will replace it arbitrarily many times in between. + if payloadAttributes != nil { + args := &miner.BuildPayloadArgs{ + Parent: update.HeadBlockHash, + Timestamp: payloadAttributes.Timestamp, + FeeRecipient: payloadAttributes.SuggestedFeeRecipient, + Random: payloadAttributes.Random, + Withdrawals: payloadAttributes.Withdrawals, + BeaconRoot: payloadAttributes.BeaconRoot, + Version: payloadVersion, + } + id := args.Id() + // If we already are busy generating this work, then we do not need + // to start a second process. + if api.localBlocks.has(id) { + return valid(&id), nil + } + payload, err := api.eth.Miner().BuildPayload(args) + if err != nil { + log.Error("Failed to build payload", "err", err) + return valid(nil), engine.InvalidPayloadAttributes.With(err) + } + api.localBlocks.put(id, payload) + return valid(&id), nil + } + return valid(nil), nil +} + +func (api *ConsensusAPI) GetPayloadQng(payloadID engine.PayloadID, full bool) (*engine.ExecutionPayloadEnvelope, error) { + log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) + data := api.localBlocks.get(payloadID, full) + if data == nil { + return nil, engine.UnknownPayload + } + return data, nil +} + +func (api *ConsensusAPI) NewPayloadQng(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { + // The locking here is, strictly, not required. Without these locks, this can happen: + // + // 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to + // api.eth.BlockChain().InsertBlockWithoutSetHead, where it is blocked on + // e.g database compaction. + // 2. The call times out on the CL layer, which issues another NewPayload (execdata-N) call. + // Similarly, this also get stuck on the same place. Importantly, since the + // first call has not gone through, the early checks for "do we already have this block" + // will all return false. + // 3. When the db compaction ends, then N calls inserting the same payload are processed + // sequentially. + // Hence, we use a lock here, to be sure that the previous call has finished before we + // check whether we already have the block locally. + api.newPayloadLock.Lock() + defer api.newPayloadLock.Unlock() + + log.Trace("Engine API request received", "method", "NewPayload", "number", params.Number, "hash", params.BlockHash) + block, err := engine.ExecutableDataToBlockQng(params, versionedHashes, beaconRoot) + if err != nil { + bgu := "nil" + if params.BlobGasUsed != nil { + bgu = strconv.Itoa(int(*params.BlobGasUsed)) + } + ebg := "nil" + if params.BlobGasUsed != nil { + ebg = strconv.Itoa(int(*params.ExcessBlobGas)) + } + log.Warn("Invalid NewPayload params", + "params.Number", params.Number, + "params.ParentHash", params.ParentHash, + "params.BlockHash", params.BlockHash, + "params.StateRoot", params.StateRoot, + "params.FeeRecipient", params.FeeRecipient, + "params.LogsBloom", common.PrettyBytes(params.LogsBloom), + "params.Random", params.Random, + "params.GasLimit", params.GasLimit, + "params.GasUsed", params.GasUsed, + "params.Timestamp", params.Timestamp, + "params.ExtraData", common.PrettyBytes(params.ExtraData), + "params.BaseFeePerGas", params.BaseFeePerGas, + "params.BlobGasUsed", bgu, + "params.ExcessBlobGas", ebg, + "len(params.Transactions)", len(params.Transactions), + "len(params.Withdrawals)", len(params.Withdrawals), + "beaconRoot", beaconRoot, + "error", err) + return api.invalid(err, nil), nil + } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastNewPayloadLock.Lock() + api.lastNewPayloadUpdate = time.Now() + api.lastNewPayloadLock.Unlock() + + // If we already have the block locally, ignore the entire execution and just + // return a fake success. + if block := api.eth.BlockChain().GetBlockByHash(params.BlockHash); block != nil { + log.Warn("Ignoring already known beacon payload", "number", params.Number, "hash", params.BlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) + hash := block.Hash() + return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil + } + // If this block was rejected previously, keep rejecting it + if res := api.checkInvalidAncestor(block.Hash(), block.Hash()); res != nil { + return *res, nil + } + // If the parent is missing, we - in theory - could trigger a sync, but that + // would also entail a reorg. That is problematic if multiple sibling blocks + // are being fed to us, and even more so, if some semi-distant uncle shortens + // our live chain. As such, payload execution will not permit reorgs and thus + // will not trigger a sync cycle. That is fine though, if we get a fork choice + // update after legit payload executions. + parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return api.delayPayloadImport(block), nil + } + if block.Time() <= parent.Time() { + log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) + return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil + } + // Another corner case: if the node is in snap sync mode, but the CL client + // tries to make it import a block. That should be denied as pushing something + // into the database directly will conflict with the assumptions of snap sync + // that it has an empty db that it can fill itself. + if api.eth.SyncMode() != downloader.FullSync { + return api.delayPayloadImport(block), nil + } + if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { + api.remoteBlocks.put(block.Hash(), block.Header()) + log.Warn("State not available, ignoring new payload") + return engine.PayloadStatusV1{Status: engine.ACCEPTED}, nil + } + log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number()) + if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil { + log.Warn("NewPayloadV1: inserting block failed", "error", err) + + api.invalidLock.Lock() + api.invalidBlocksHits[block.Hash()] = 1 + api.invalidTipsets[block.Hash()] = block.Header() + api.invalidLock.Unlock() + + return api.invalid(err, parent.Header()), nil + } + hash := block.Hash() + return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil +} + +func NewConsensusAPIQng(eth *eth.Ethereum) *ConsensusAPI { + api := &ConsensusAPI{ + eth: eth, + remoteBlocks: newHeaderQueue(), + localBlocks: newPayloadQueue(), + invalidBlocksHits: make(map[common.Hash]int), + invalidTipsets: make(map[common.Hash]*types.Header), + } + eth.Downloader().SetBadBlockCallback(api.setInvalidAncestor) + return api +}