Store eth tx index separately

Closes: evmos#1075
Solution:
- run a optional indexer service
- adapt the json-rpc to the more efficient query

changelog

changelog

fix lint

fix backward compatibility

fix lint

timeout

better strconv

fix linter

fix package name

add cli command to index old tx

fix for loop

indexer cmd don't have access to local rpc

workaround exceed block gas limit situation

add unit tests for indexer

refactor

polish the indexer module

Update server/config/toml.go

Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>

improve comments

share code between GetTxByEthHash and GetTxByIndex

fix unit test

Update server/indexer.go

Co-authored-by: Freddy Caceres <facs95@gmail.com>

CHANGELOG.md

@@ -86,6 +86,10 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (rpc) [\#1143](https://github.com/evmos/ethermint/pull/1143) Restrict unprotected txs on the node JSON-RPC configuration.
 * (all) [\#1137](https://github.com/evmos/ethermint/pull/1137) Rename go module to `evmos/ethermint`
 
+### API Breaking
+
+- (json-rpc) [tharsis#1121](https://github.com/tharsis/ethermint/pull/1121) Store eth tx index separately
+
 ### Improvements
 
 * (deps) [\#1147](https://github.com/evmos/ethermint/pull/1147) Bump Go version to `1.18`.

docs/api/proto-docs.md

@@ -79,6 +79,9 @@
 - [ethermint/types/v1/account.proto](#ethermint/types/v1/account.proto)
  - [EthAccount](#ethermint.types.v1.EthAccount)
 
+- [ethermint/types/v1/indexer.proto](#ethermint/types/v1/indexer.proto)
+ - [TxResult](#ethermint.types.v1.TxResult)
+
 - [ethermint/types/v1/web3.proto](#ethermint/types/v1/web3.proto)
  - [ExtensionOptionsWeb3Tx](#ethermint.types.v1.ExtensionOptionsWeb3Tx)
 
@@ -1134,6 +1137,43 @@ authtypes.BaseAccount type. It is compatible with the auth AccountKeeper.
 
 
 
+ <!-- end messages -->
+
+ <!-- end enums -->
+
+ <!-- end HasExtensions -->
+
+ <!-- end services -->
+
+
+
+<a name="ethermint/types/v1/indexer.proto"></a>
+<p align="right"><a href="#top">Top</a></p>
+
+## ethermint/types/v1/indexer.proto
+
+
+
+<a name="ethermint.types.v1.TxResult"></a>
+
+### TxResult
+TxResult is the value stored in eth tx indexer
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| `height` | [int64](#int64) | | the block height |
+| `tx_index` | [uint32](#uint32) | | cosmos tx index |
+| `msg_index` | [uint32](#uint32) | | the msg index in a batch tx |
+| `eth_tx_index` | [int32](#int32) | | eth tx index |
+| `failed` | [bool](#bool) | | if the eth tx is failed |
+| `gas_used` | [uint64](#uint64) | | gas used by tx, if exceeds block gas limit, it's set to gas limit which is what's actually deducted by ante handler. |
+| `cumulative_gas_used` | [uint64](#uint64) | | the cumulative gas used within current batch tx |
+
+
+
+
+
  <!-- end messages -->
 
  <!-- end enums -->

proto/ethermint/types/v1/indexer.proto

@@ -0,0 +1,28 @@
+syntax = "proto3";
+package ethermint.types.v1;
+
+import "gogoproto/gogo.proto";
+
+option go_package = "github.com/evmos/ethermint/types";
+
+// TxResult is the value stored in eth tx indexer
+message TxResult {
+ option (gogoproto.goproto_getters) = false;
+
+ // the block height
+ int64 height = 1;
+ // cosmos tx index
+ uint32 tx_index = 2;
+ // the msg index in a batch tx
+ uint32 msg_index = 3;
+
+ // eth tx index
+ int32 eth_tx_index = 4;
+ // if the eth tx is failed
+ bool failed = 5;
+ // gas used by tx, if exceeds block gas limit,
+ // it's set to gas limit which is what's actually deducted by ante handler.
+ uint64 gas_used = 6;
+ // the cumulative gas used within current batch tx
+ uint64 cumulative_gas_used = 7;
+}

rpc/apis.go

@@ -19,6 +19,7 @@ import (
  "github.com/evmos/ethermint/rpc/namespaces/ethereum/personal"
  "github.com/evmos/ethermint/rpc/namespaces/ethereum/txpool"
  "github.com/evmos/ethermint/rpc/namespaces/ethereum/web3"
+ ethermint "github.com/evmos/ethermint/types"
 
  rpcclient "github.com/tendermint/tendermint/rpc/jsonrpc/client"
 )
@@ -48,15 +49,16 @@ type APICreator = func(
  clientCtx client.Context,
  tendermintWebsocketClient *rpcclient.WSClient,
  allowUnprotectedTxs bool,
+ indexer ethermint.EVMTxIndexer,
 ) []rpc.API
 
 // apiCreators defines the JSON-RPC API namespaces.
 var apiCreators map[string]APICreator
 
 func init() {
  apiCreators = map[string]APICreator{
- EthNamespace: func(ctx *server.Context, clientCtx client.Context, tmWSClient *rpcclient.WSClient, allowUnprotectedTxs bool) []rpc.API {
- evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs)
+ EthNamespace: func(ctx *server.Context, clientCtx client.Context, tmWSClient *rpcclient.WSClient, allowUnprotectedTxs bool, indexer ethermint.EVMTxIndexer) []rpc.API {
+ evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
  return []rpc.API{
  {
  Namespace: EthNamespace,
@@ -72,7 +74,7 @@ func init() {
  },
  }
  },
- Web3Namespace: func(*server.Context, client.Context, *rpcclient.WSClient, bool) []rpc.API {
+ Web3Namespace: func(*server.Context, client.Context, *rpcclient.WSClient, bool, ethermint.EVMTxIndexer) []rpc.API {
  return []rpc.API{
  {
  Namespace: Web3Namespace,
@@ -82,7 +84,7 @@ func init() {
  },
  }
  },
- NetNamespace: func(_ *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, _ bool) []rpc.API {
+ NetNamespace: func(_ *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, _ bool, _ ethermint.EVMTxIndexer) []rpc.API {
  return []rpc.API{
  {
  Namespace: NetNamespace,
@@ -92,8 +94,8 @@ func init() {
  },
  }
  },
- PersonalNamespace: func(ctx *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, allowUnprotectedTxs bool) []rpc.API {
- evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs)
+ PersonalNamespace: func(ctx *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, allowUnprotectedTxs bool, indexer ethermint.EVMTxIndexer) []rpc.API {
+ evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
  return []rpc.API{
  {
  Namespace: PersonalNamespace,
@@ -103,7 +105,7 @@ func init() {
  },
  }
  },
- TxPoolNamespace: func(ctx *server.Context, _ client.Context, _ *rpcclient.WSClient, _ bool) []rpc.API {
+ TxPoolNamespace: func(ctx *server.Context, _ client.Context, _ *rpcclient.WSClient, _ bool, _ ethermint.EVMTxIndexer) []rpc.API {
  return []rpc.API{
  {
  Namespace: TxPoolNamespace,
@@ -113,8 +115,8 @@ func init() {
  },
  }
  },
- DebugNamespace: func(ctx *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, allowUnprotectedTxs bool) []rpc.API {
- evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs)
+ DebugNamespace: func(ctx *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, allowUnprotectedTxs bool, indexer ethermint.EVMTxIndexer) []rpc.API {
+ evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
  return []rpc.API{
  {
  Namespace: DebugNamespace,
@@ -124,8 +126,8 @@ func init() {
  },
  }
  },
- MinerNamespace: func(ctx *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, allowUnprotectedTxs bool) []rpc.API {
- evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs)
+ MinerNamespace: func(ctx *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, allowUnprotectedTxs bool, indexer ethermint.EVMTxIndexer) []rpc.API {
+ evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
  return []rpc.API{
  {
  Namespace: MinerNamespace,
@@ -139,12 +141,12 @@ func init() {
 }
 
 // GetRPCAPIs returns the list of all APIs
-func GetRPCAPIs(ctx *server.Context, clientCtx client.Context, tmWSClient *rpcclient.WSClient, allowUnprotectedTxs bool, selectedAPIs []string) []rpc.API {
+func GetRPCAPIs(ctx *server.Context, clientCtx client.Context, tmWSClient *rpcclient.WSClient, allowUnprotectedTxs bool, indexer ethermint.EVMTxIndexer, selectedAPIs []string) []rpc.API {
  var apis []rpc.API
 
  for _, ns := range selectedAPIs {
  if creator, ok := apiCreators[ns]; ok {
- apis = append(apis, creator(ctx, clientCtx, tmWSClient, allowUnprotectedTxs)...)
+ apis = append(apis, creator(ctx, clientCtx, tmWSClient, allowUnprotectedTxs, indexer)...)
  } else {
  ctx.Logger.Error("invalid namespace value", "namespace", ns)
  }

rpc/backend/backend.go

@@ -103,8 +103,8 @@ type EVMBackend interface {
 
  // Tx Info
  GetTransactionByHash(txHash common.Hash) (*rpctypes.RPCTransaction, error)
- GetTxByEthHash(txHash common.Hash) (*tmrpctypes.ResultTx, error)
- GetTxByTxIndex(height int64, txIndex uint) (*tmrpctypes.ResultTx, error)
+ GetTxByEthHash(txHash common.Hash) (*ethermint.TxResult, error)
+ GetTxByTxIndex(height int64, txIndex uint) (*ethermint.TxResult, error)
  GetTransactionByBlockAndIndex(block *tmrpctypes.ResultBlock, idx hexutil.Uint) (*rpctypes.RPCTransaction, error)
  GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error)
  GetTransactionByBlockHashAndIndex(hash common.Hash, idx hexutil.Uint) (*rpctypes.RPCTransaction, error)
@@ -140,10 +140,11 @@ type Backend struct {
  chainID *big.Int
  cfg config.Config
  allowUnprotectedTxs bool
+ indexer ethermint.EVMTxIndexer
 }
 
 // NewBackend creates a new Backend instance for cosmos and ethereum namespaces
-func NewBackend(ctx *server.Context, logger log.Logger, clientCtx client.Context, allowUnprotectedTxs bool) *Backend {
+func NewBackend(ctx *server.Context, logger log.Logger, clientCtx client.Context, allowUnprotectedTxs bool, indexer ethermint.EVMTxIndexer) *Backend {
  chainID, err := ethermint.ParseChainID(clientCtx.ChainID)
  if err != nil {
  panic(err)
@@ -176,5 +177,6 @@ func NewBackend(ctx *server.Context, logger log.Logger, clientCtx client.Context
  chainID: chainID,
  cfg: appConf,
  allowUnprotectedTxs: allowUnprotectedTxs,
+ indexer: indexer,
  }
 }

rpc/backend/backend_suite_test.go

@@ -56,7 +56,7 @@ func (suite *BackendTestSuite) SetupTest() {
 
  allowUnprotectedTxs := false
 
- suite.backend = NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs)
+ suite.backend = NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, nil)
  suite.backend.queryClient.QueryClient = mocks.NewQueryClient(suite.T())
  suite.backend.clientCtx.Client = mocks.NewClient(suite.T())
  suite.backend.ctx = rpctypes.ContextWithHeight(1)

rpc/backend/tracing.go

@@ -32,23 +32,14 @@ func (b *Backend) TraceTransaction(hash common.Hash, config *evmtypes.TraceConfi
  return nil, err
  }
 
- parsedTxs, err := rpctypes.ParseTxResult(&transaction.TxResult)
- if err != nil {
- return nil, fmt.Errorf("failed to parse tx events: %s", hash.Hex())
- }
- parsedTx := parsedTxs.GetTxByHash(hash)
- if parsedTx == nil {
- return nil, fmt.Errorf("ethereum tx not found in msgs: %s", hash.Hex())
- }
-
  // check tx index is not out of bound
- if uint32(len(blk.Block.Txs)) < transaction.Index {
- b.logger.Debug("tx index out of bounds", "index", transaction.Index, "hash", hash.String(), "height", blk.Block.Height)
+ if uint32(len(blk.Block.Txs)) < transaction.TxIndex {
+ b.logger.Debug("tx index out of bounds", "index", transaction.TxIndex, "hash", hash.String(), "height", blk.Block.Height)
  return nil, fmt.Errorf("transaction not included in block %v", blk.Block.Height)
  }
 
  var predecessors []*evmtypes.MsgEthereumTx
- for _, txBz := range blk.Block.Txs[:transaction.Index] {
+ for _, txBz := range blk.Block.Txs[:transaction.TxIndex] {
  tx, err := b.clientCtx.TxConfig.TxDecoder()(txBz)
  if err != nil {
  b.logger.Debug("failed to decode transaction in block", "height", blk.Block.Height, "error", err.Error())
@@ -64,22 +55,22 @@ func (b *Backend) TraceTransaction(hash common.Hash, config *evmtypes.TraceConfi
  }
  }
 
- tx, err := b.clientCtx.TxConfig.TxDecoder()(transaction.Tx)
+ tx, err := b.clientCtx.TxConfig.TxDecoder()(blk.Block.Txs[transaction.TxIndex])
  if err != nil {
  b.logger.Debug("tx not found", "hash", hash)
  return nil, err
  }
 
  // add predecessor messages in current cosmos tx
- for i := 0; i < parsedTx.MsgIndex; i++ {
+ for i := 0; i < int(transaction.MsgIndex); i++ {
  ethMsg, ok := tx.GetMsgs()[i].(*evmtypes.MsgEthereumTx)
  if !ok {
  continue
  }
  predecessors = append(predecessors, ethMsg)
  }
 
- ethMessage, ok := tx.GetMsgs()[parsedTx.MsgIndex].(*evmtypes.MsgEthereumTx)
+ ethMessage, ok := tx.GetMsgs()[transaction.MsgIndex].(*evmtypes.MsgEthereumTx)
  if !ok {
  b.logger.Debug("invalid transaction type", "type", fmt.Sprintf("%T", tx))
  return nil, fmt.Errorf("invalid transaction type %T", tx)