Problem: block-stm tx executor bad worst case performance

CHANGELOG.md

@@ -87,6 +87,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (deps) [#505](https://github.com/crypto-org-chain/ethermint/pull/505) Update cometbft to v0.38.10.
 * (ante) [#504](https://github.com/crypto-org-chain/ethermint/pull/504) Optimize AnteHandle method to skip checks if disabledMsgs is empty.
 * [#517](https://github.com/crypto-org-chain/ethermint/pull/517) Add check for integer overflow to ensure safe conversion.
+* [#522](https://github.com/crypto-org-chain/ethermint/pull/522) block-stm executor support optional pre-estimations.
 
 ## v0.21.x-cronos
 

app/app.go

@@ -337,18 +337,6 @@
  okeys: okeys,
  }
 
- executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor))
- switch executor {
- case srvconfig.BlockExecutorBlockSTM:
- sdk.SetAddrCacheEnabled(false)
- workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers))
- app.SetTxExecutor(STMTxExecutor(app.GetStoreKeys(), workers))
- case "", srvconfig.BlockExecutorSequential:
- app.SetTxExecutor(DefaultTxExecutor)
- default:
- panic(fmt.Errorf("unknown EVM block executor: %s", executor))
- }
-
  // init params keeper and subspaces
  app.ParamsKeeper = initParamsKeeper(appCodec, cdc, keys[paramstypes.StoreKey], tkeys[paramstypes.TStoreKey])
 
@@ -803,6 +791,19 @@
  app.ScopedIBCKeeper = scopedIBCKeeper
  app.ScopedTransferKeeper = scopedTransferKeeper
 
+ executor := cast.ToString(appOpts.Get(srvflags.EVMBlockExecutor))
+ switch executor {
+ case srvconfig.BlockExecutorBlockSTM:
+ sdk.SetAddrCacheEnabled(false)
+ workers := cast.ToInt(appOpts.Get(srvflags.EVMBlockSTMWorkers))
+ preEstimate := cast.ToBool(appOpts.Get(srvflags.EVMBlockSTMPreEstimate))
+ app.SetTxExecutor(STMTxExecutor(app.GetStoreKeys(), workers, preEstimate, app.EvmKeeper, txConfig.TxDecoder()))
+ case "", srvconfig.BlockExecutorSequential:
+ app.SetTxExecutor(DefaultTxExecutor)
+ default:
+ panic(fmt.Errorf("unknown EVM block executor: %s", executor))
+ }
+
  return app
 }
 

app/executor.go

@@ -5,39 +5,64 @@
  "io"
  "sync/atomic"
 
+ "cosmossdk.io/collections"
+ "cosmossdk.io/log"
  "cosmossdk.io/store/cachemulti"
  storetypes "cosmossdk.io/store/types"
  abci "github.com/cometbft/cometbft/abci/types"
+ cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
+ authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
+ banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
  evmtypes "github.com/evmos/ethermint/x/evm/types"
 
  "github.com/cosmos/cosmos-sdk/baseapp"
+ sdk "github.com/cosmos/cosmos-sdk/types"
 
  blockstm "github.com/crypto-org-chain/go-block-stm"
 )
 
 func DefaultTxExecutor(_ context.Context,
- blockSize int,
+ txs [][]byte,
  ms storetypes.MultiStore,
- deliverTxWithMultiStore func(int, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
+ deliverTxWithMultiStore func(int, sdk.Tx, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
 ) ([]*abci.ExecTxResult, error) {
+ blockSize := len(txs)
  results := make([]*abci.ExecTxResult, blockSize)
  for i := 0; i < blockSize; i++ {
- results[i] = deliverTxWithMultiStore(i, ms, nil)
+ results[i] = deliverTxWithMultiStore(i, nil, ms, nil)
  }
  return evmtypes.PatchTxResponses(results), nil
 }
 
-func STMTxExecutor(stores []storetypes.StoreKey, workers int) baseapp.TxExecutor {
+type evmKeeper interface {
+ GetParams(ctx sdk.Context) evmtypes.Params
+}
+
+func STMTxExecutor(
+ stores []storetypes.StoreKey,
+ workers int,
+ estimate bool,
+ evmKeeper evmKeeper,
+ txDecoder sdk.TxDecoder,
+) baseapp.TxExecutor {
+ var authStore, bankStore int
  index := make(map[storetypes.StoreKey]int, len(stores))
  for i, k := range stores {
+ switch k.Name() {
+ case authtypes.StoreKey:
+ authStore = i
+ case banktypes.StoreKey:
+ bankStore = i
+ }
  index[k] = i
  }
  return func(
  ctx context.Context,
- blockSize int,
+ txs [][]byte,
  ms storetypes.MultiStore,
- deliverTxWithMultiStore func(int, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
+ deliverTxWithMultiStore func(int, sdk.Tx, storetypes.MultiStore, map[string]any) *abci.ExecTxResult,
  ) ([]*abci.ExecTxResult, error) {
+ blockSize := len(txs)
  if blockSize == 0 {
  return nil, nil
  }
@@ -47,12 +72,27 @@
  m := make(map[string]any)
  incarnationCache[i].Store(&m)
  }
- if err := blockstm.ExecuteBlock(
+
+ var estimates map[int]blockstm.MultiLocations
+ memTxs := make([]sdk.Tx, len(txs))
+ if estimate {
+ for i, rawTx := range txs {
+ if memTx, err := txDecoder(rawTx); err == nil {
+ memTxs[i] = memTx
+ }
+ }
+ // pre-estimation
+ evmDenom := evmKeeper.GetParams(sdk.NewContext(ms, cmtproto.Header{}, false, log.NewNopLogger())).EvmDenom
+ estimates = preEstimates(memTxs, authStore, bankStore, evmDenom)
+ }
+
+ if err := blockstm.ExecuteBlockWithEstimates(
  ctx,
  blockSize,
  index,
  stmMultiStoreWrapper{ms},
  workers,
+ estimates,
  func(txn blockstm.TxnIndex, ms blockstm.MultiStore) {
  var cache map[string]any
 
@@ -63,8 +103,7 @@
  cache = *v
  }
 
- result := deliverTxWithMultiStore(int(txn), msWrapper{ms}, cache)
- results[txn] = result
+ results[txn] = deliverTxWithMultiStore(int(txn), memTxs[txn], msWrapper{ms}, cache)
 
  if v != nil {
  incarnationCache[txn].Store(v)
@@ -146,3 +185,43 @@
 func (ms stmMultiStoreWrapper) GetObjKVStore(key storetypes.StoreKey) storetypes.ObjKVStore {
  return ms.MultiStore.GetObjKVStore(key)
 }
+
+// preEstimates returns a static estimation of the written keys for each transaction.
+// NOTE: make sure it sync with the latest sdk logic when sdk upgrade.
+func preEstimates(txs []sdk.Tx, authStore, bankStore int, evmDenom string) map[int]blockstm.MultiLocations {
+ estimates := make(map[int]blockstm.MultiLocations, len(txs))
+ for i, tx := range txs {
+ feeTx, ok := tx.(sdk.FeeTx)
+ if !ok {
+ continue
+ }
+ feePayer := sdk.AccAddress(feeTx.FeePayer())
+
+ // account key
+ accKey, err := collections.EncodeKeyWithPrefix(
+ authtypes.AddressStoreKeyPrefix,
+ sdk.AccAddressKey,
+ feePayer,
+ )
+ if err != nil {
+ continue
+ }
+
+ // balance key
+ balanceKey, err := collections.EncodeKeyWithPrefix(
+ banktypes.BalancesPrefix,
+ collections.PairKeyCodec(sdk.AccAddressKey, collections.StringKey),
+ collections.Join(feePayer, evmDenom),
+ )
+ if err != nil {
+ continue
+ }
+
+ estimates[i] = blockstm.MultiLocations{
+ authStore: {accKey},
+ bankStore: {balanceKey},
+ }
+ }
+
+ return estimates
+}

go.mod

@@ -7,6 +7,7 @@ toolchain go1.21.5
 require (
  cosmossdk.io/api v0.7.5
  cosmossdk.io/client/v2 v2.0.0-beta.1
+ cosmossdk.io/collections v0.4.0
  cosmossdk.io/core v0.11.0
  cosmossdk.io/errors v1.0.1
  cosmossdk.io/log v1.3.1
@@ -28,7 +29,7 @@ require (
  github.com/cosmos/ibc-go/modules/capability v1.0.0
  github.com/cosmos/ibc-go/v8 v8.1.0
  github.com/cosmos/rosetta v0.50.3-1
- github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883
+ github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae
  github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
  github.com/ethereum/go-ethereum v1.10.26
  github.com/gogo/protobuf v1.3.2
@@ -65,7 +66,6 @@ require (
  cloud.google.com/go/compute/metadata v0.2.3 // indirect
  cloud.google.com/go/iam v1.1.6 // indirect
  cloud.google.com/go/storage v1.36.0 // indirect
- cosmossdk.io/collections v0.4.0 // indirect
  cosmossdk.io/depinject v1.0.0-alpha.4 // indirect
  filippo.io/edwards25519 v1.1.0 // indirect
  github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
@@ -243,10 +243,10 @@ require (
 
 replace (
  // release/v0.50.x
- cosmossdk.io/client/v2 => github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240722033504-50f1fa0c49d1
- cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1
- cosmossdk.io/x/tx => github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1
- github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240722033504-50f1fa0c49d1
+ cosmossdk.io/client/v2 => github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2
+ cosmossdk.io/store => github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2
+ cosmossdk.io/x/tx => github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2
+ github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2
 )
 
 replace (

go.sum

@@ -411,16 +411,16 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c h1:MOgfS4+FBB8cMkDE2j2VBVsbY+HCkPIu0YsJ/9bbGeQ=
 github.com/crypto-org-chain/btree v0.0.0-20240406140148-2687063b042c/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
-github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240722033504-50f1fa0c49d1 h1:xahHemSiT79xgh8Ig8zOTeHSLHt9FfPzViK7rATWhUM=
-github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240722033504-50f1fa0c49d1/go.mod h1:Rb43DdB0i/rKcCN69Tg2X3+zA4WhJ7MC8K3a6Ezh38E=
-github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240722033504-50f1fa0c49d1 h1:zqTYZqMKnv15UkKwCEbGqshZ6tWczctBYP53FxmEiF8=
-github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240722033504-50f1fa0c49d1/go.mod h1:W5sR4asmVDUhJpEmuXTUBkk/yEefKlXTjVWcNciVSR0=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1 h1:ZlezTiQu9pYpVO+6sB9+W3fvthIpV1GgSI8kPjw+v5s=
-github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg=
-github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1 h1:r0ALP31Wnw19FqEmqzsK2SFNqdMetHshnM/X/FeJRIo=
-github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240722033504-50f1fa0c49d1/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY=
-github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883 h1:Oj7VvlK8iXRaugnpGA8CBXGrgkyigji+Ae5weSRD85I=
-github.com/crypto-org-chain/go-block-stm v0.0.0-20240806075927-09a64748f883/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
+github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2 h1:4SoAvnxDaiIWcgm6XOmPDIdCf4/WNhNYLXGbij1eaA0=
+github.com/crypto-org-chain/cosmos-sdk v0.50.6-0.20240911084450-6870ba130be2/go.mod h1:Rb43DdB0i/rKcCN69Tg2X3+zA4WhJ7MC8K3a6Ezh38E=
+github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2 h1:5oGZtOUcauk9dtv+8BCfj2PEQyXEEEV+K3sP4OSvBmg=
+github.com/crypto-org-chain/cosmos-sdk/client/v2 v2.0.0-20240911084450-6870ba130be2/go.mod h1:W5sR4asmVDUhJpEmuXTUBkk/yEefKlXTjVWcNciVSR0=
+github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2 h1:CGh5I0L6IYhe0AJevb4vf5TE3ru+qAgMs437BlWCwo8=
+github.com/crypto-org-chain/cosmos-sdk/store v0.0.0-20240911084450-6870ba130be2/go.mod h1:gjE3DZe4t/+VeIk6CmrouyqiuDbZ7QOVDDq3nLqBTpg=
+github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2 h1:mxlOSCru7YgmX055rrlkCSUu0D8lAqJ8Dnhp0yXCBuM=
+github.com/crypto-org-chain/cosmos-sdk/x/tx v0.0.0-20240911084450-6870ba130be2/go.mod h1:RTiTs4hkXG6IvYGknvB8p79YgjYJdcbzLUOGJChsPnY=
+github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae h1:gakWYsVubWX8P9NpxaPnvg0UJYfIZigfko5WN57t7OA=
+github.com/crypto-org-chain/go-block-stm v0.0.0-20240911081142-92839e79a3ae/go.mod h1:iwQTX9xMX8NV9k3o2BiWXA0SswpsZrDk5q3gA7nWYiE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7 h1:V43F3JFcqG4MUThf9W/DytnPblpR6CcaLBw2Wx6zTgE=
 github.com/crypto-org-chain/go-ethereum v1.10.20-0.20240425065928-ebb09502e7a7/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y=
 github.com/danieljoos/wincred v1.2.0 h1:ozqKHaLK0W/ii4KVbbvluM91W2H3Sh0BncbUNPS7jLE=

gomod2nix.toml

@@ -20,7 +20,7 @@ schema = 3
  version = "v0.7.5"
  hash = "sha256-Nuw697sJr56kU3EU7DV1eYNkyI76psznIVqYAV6RfbQ="
  [mod."cosmossdk.io/client/v2"]
- version = "v2.0.0-20240722033504-50f1fa0c49d1"
+ version = "v2.0.0-20240911084450-6870ba130be2"
  hash = "sha256-60hmufv3Ml4Pv3zNwgn8eeqlEINOR6n9MKr2QHddoxo="
  replaced = "github.com/crypto-org-chain/cosmos-sdk/client/v2"
  [mod."cosmossdk.io/collections"]
@@ -42,7 +42,7 @@ schema = 3
  version = "v1.3.0"
  hash = "sha256-EEFK43Cr0g0ndhQhkIKher0FqV3mvkmE9z0sP7uVSHg="
  [mod."cosmossdk.io/store"]
- version = "v0.0.0-20240722033504-50f1fa0c49d1"
+ version = "v0.0.0-20240911084450-6870ba130be2"
  hash = "sha256-Dm3sSZNJBcnBF33PULoTpK4rkNQbsZl0DfTqH1GPCQM="
  replaced = "github.com/crypto-org-chain/cosmos-sdk/store"
  [mod."cosmossdk.io/tools/confix"]
@@ -55,7 +55,7 @@ schema = 3
  version = "v0.1.0"
  hash = "sha256-/gWvrqvy6bW90+NU66T+9QysYgvG1VbwfYJZ8tkqpeA="
  [mod."cosmossdk.io/x/tx"]
- version = "v0.0.0-20240722033504-50f1fa0c49d1"
+ version = "v0.0.0-20240911084450-6870ba130be2"
  hash = "sha256-xT5IdapEx1h46ofBpxcBQfzGF2EntmC8xZl7aym/6xE="
  replaced = "github.com/crypto-org-chain/cosmos-sdk/x/tx"
  [mod."cosmossdk.io/x/upgrade"]
@@ -163,8 +163,8 @@ schema = 3
  version = "v1.0.0-beta.5"
  hash = "sha256-Fy/PbsOsd6iq0Njy3DVWK6HqWsogI+MkE8QslHGWyVg="
  [mod."github.com/cosmos/cosmos-sdk"]
- version = "v0.50.6-0.20240722033504-50f1fa0c49d1"
- hash = "sha256-OjJgi6tq5c4czeQLZVeJRBo4s4kC37h4IFT/GGvF7G0="
+ version = "v0.50.6-0.20240911084450-6870ba130be2"
+ hash = "sha256-kl2sLe8vITIguRGtdeviDeP3R5JIbRg+eDsheGd4PqM="
  replaced = "github.com/crypto-org-chain/cosmos-sdk"
  [mod."github.com/cosmos/go-bip39"]
  version = "v1.0.0"
@@ -203,8 +203,8 @@ schema = 3
  version = "v0.0.24"
  hash = "sha256-4vUukHONOjNn0qfQr4esK6TWfPWsIp+rbdz65og84lw="
  [mod."github.com/crypto-org-chain/go-block-stm"]
- version = "v0.0.0-20240806075927-09a64748f883"
- hash = "sha256-DuqcnTqpUVRxV+I+CaBuDg/1CLmMeXdw4VEFeuCsUbU="
+ version = "v0.0.0-20240911081142-92839e79a3ae"
+ hash = "sha256-8MhSeC5BB5BwOet3k3Rfua0TBeBATtLSwaW6s5WpYCM="
  [mod."github.com/danieljoos/wincred"]
  version = "v1.2.0"
  hash = "sha256-LHcvTJCc8++bFndbd8ZgMSTe4L5h2C4rN+cSWHCz54Y="

server/config/config.go

@@ -134,6 +134,8 @@
  BlockExecutor string `mapstructure:"block-executor"`
  // BlockSTMWorkers is the number of workers for block-stm execution, `0` means using all available CPUs.
  BlockSTMWorkers int `mapstructure:"block-stm-workers"`
+ // BlockSTMPreEstimate is the flag to enable pre-estimation for block-stm execution.
+ BlockSTMPreEstimate bool `mapstructure:"block-stm-pre-estimate"`
 }
 
 // JSONRPCConfig defines configuration for the EVM RPC server.
@@ -407,10 +409,11 @@
  return Config{
  Config: cfg,
  EVM: EVMConfig{
- Tracer: v.GetString("evm.tracer"),
- MaxTxGasWanted: v.GetUint64("evm.max-tx-gas-wanted"),
- BlockExecutor: v.GetString("evm.block-executor"),
- BlockSTMWorkers: v.GetInt("evm.block-stm-workers"),
+ Tracer: v.GetString("evm.tracer"),
+ MaxTxGasWanted: v.GetUint64("evm.max-tx-gas-wanted"),
+ BlockExecutor: v.GetString("evm.block-executor"),
+ BlockSTMWorkers: v.GetInt("evm.block-stm-workers"),
+ BlockSTMPreEstimate: v.GetBool("evm.block-stm-pre-estimate"),
  },
  JSONRPC: JSONRPCConfig{
  Enable: v.GetBool("json-rpc.enable"),

server/config/toml.go

@@ -35,6 +35,8 @@ max-tx-gas-wanted = {{ .EVM.MaxTxGasWanted }}
 block-executor = "{{ .EVM.BlockExecutor }}"
 # BlockSTMWorkers is the number of workers for block-stm execution, 0 means using all available CPUs.
 block-stm-workers = {{ .EVM.BlockSTMWorkers }}
+# BlockSTMPreEstimate is the flag to enable pre-estimation for block-stm execution.
+block-stm-pre-estimate = {{ .EVM.BlockSTMPreEstimate }}
 
 ###############################################################################
 ### JSON RPC Configuration ###

server/flags/flags.go

@@ -76,10 +76,11 @@ const (
 
 // EVM flags
 const (
- EVMTracer = "evm.tracer"
- EVMMaxTxGasWanted = "evm.max-tx-gas-wanted"
- EVMBlockExecutor = "evm.block-executor"
- EVMBlockSTMWorkers = "evm.block-stm-workers"
+ EVMTracer = "evm.tracer"
+ EVMMaxTxGasWanted = "evm.max-tx-gas-wanted"
+ EVMBlockExecutor = "evm.block-executor"
+ EVMBlockSTMWorkers = "evm.block-stm-workers"
+ EVMBlockSTMPreEstimate = "evm.block-stm-pre-estimate"
 )
 
 // TLS flags