Problem: The unlucky transactions are not shown in json rpc (#458)

* Problem: The unlucky transactions are not shown in json rpc

Closes: #455
- add patch-tx-result command
- test in integration test

* support psql indexer

* fix integration test

* tag dependency

* Update cmd/cronosd/cmd/fix-unlucky-tx.go

* Apply suggestions from code review

* create a refresh app to replay tx

* enable lazy loading

* add dry-run flag

* fix lint

* more convinient flags

* Update cmd/cronosd/cmd/fix-unlucky-tx.go

Co-authored-by: Thomas Nguy <81727899+thomas-nguy@users.noreply.github.com>

* fix integration test

* fix lint

* Apply suggestions from code review

* add a warning in help text

Co-authored-by: Thomas Nguy <81727899+thomas-nguy@users.noreply.github.com>

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 - [#402](https://github.com/crypto-org-chain/cronos/pull/402) Update ethermint to include bug fixes: a) add an configurable upper bound to gasWanted to prevent DoS attack, b) fix eth_sha3 rpc api, c) json-roc server always use local tm rpc client, d) fix tx hash in pending tx filter response.
 
+### Improvements
+
+- [#458](https://github.com/crypto-org-chain/cronos/pull/458) Add `fix-unlucky-tx` command to patch the false-failed transactions in blocks before the v0.7.0 upgrade.
+
 *March 8, 2022*
 
 ## v0.6.9

cmd/cronosd/cmd/fix-unlucky-tx.go

@@ -0,0 +1,366 @@
+package cmd
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+
+ "github.com/spf13/cobra"
+
+ "github.com/cosmos/cosmos-sdk/baseapp"
+ "github.com/cosmos/cosmos-sdk/client"
+ "github.com/cosmos/cosmos-sdk/client/flags"
+ "github.com/cosmos/cosmos-sdk/server"
+ "github.com/cosmos/cosmos-sdk/store/rootmulti"
+ sdk "github.com/cosmos/cosmos-sdk/types"
+ abci "github.com/tendermint/tendermint/abci/types"
+ tmcfg "github.com/tendermint/tendermint/config"
+ "github.com/tendermint/tendermint/libs/log"
+ tmnode "github.com/tendermint/tendermint/node"
+ tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+ sm "github.com/tendermint/tendermint/state"
+ "github.com/tendermint/tendermint/state/indexer/sink/psql"
+ "github.com/tendermint/tendermint/state/txindex"
+ "github.com/tendermint/tendermint/state/txindex/kv"
+ tmstore "github.com/tendermint/tendermint/store"
+ tmtypes "github.com/tendermint/tendermint/types"
+ dbm "github.com/tendermint/tm-db"
+ evmtypes "github.com/tharsis/ethermint/x/evm/types"
+
+ "github.com/crypto-org-chain/cronos/app"
+ "github.com/crypto-org-chain/cronos/x/cronos/rpc"
+)
+
+const (
+ FlagPrintBlockNumbers = "print-block-numbers"
+ FlagBlocksFile = "blocks-file"
+ FlagStartBlock = "start-block"
+ FlagEndBlock = "end-block"
+)
+
+// FixUnluckyTxCmd update the tx execution result of false-failed tx in tendermint db
+func FixUnluckyTxCmd() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "fix-unlucky-tx",
+ Short: "Fix tx execution result of false-failed tx before v0.7.0 upgrade.",
+ Long: "Fix tx execution result of false-failed tx before v0.7.0 upgrade.\nWARNING: don't use this command to patch blocks generated after v0.7.0 upgrade",
+ Args: cobra.ExactArgs(0),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ ctx := server.GetServerContextFromCmd(cmd)
+ clientCtx := client.GetClientContextFromCmd(cmd)
+
+ chainID, err := cmd.Flags().GetString(flags.FlagChainID)
+ if err != nil {
+ return err
+ }
+ dryRun, err := cmd.Flags().GetBool(flags.FlagDryRun)
+ if err != nil {
+ return err
+ }
+ printBlockNumbers, err := cmd.Flags().GetBool(FlagPrintBlockNumbers)
+ if err != nil {
+ return err
+ }
+
+ tmDB, err := openTMDB(ctx.Config, chainID)
+ if err != nil {
+ return err
+ }
+
+ state, err := tmDB.stateStore.Load()
+ if err != nil {
+ return err
+ }
+
+ appDB, err := openAppDB(ctx.Config.RootDir)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ if err := appDB.Close(); err != nil {
+ ctx.Logger.With("error", err).Error("error closing db")
+ }
+ }()
+
+ encCfg := app.MakeEncodingConfig()
+
+ appCreator := func() *app.App {
+ cms := rootmulti.NewStore(appDB)
+ cms.SetLazyLoading(true)
+ return app.New(
+ log.NewNopLogger(), appDB, nil, false, nil,
+ ctx.Config.RootDir, 0, encCfg, ctx.Viper,
+ func(baseApp *baseapp.BaseApp) { baseApp.SetCMS(cms) },
+ )
+ }
+
+ // replay and patch a single block
+ processBlock := func(height int64) error {
+ blockResult, err := tmDB.stateStore.LoadABCIResponses(height)
+ if err != nil {
+ return err
+ }
+
+ txIndex := findUnluckyTx(blockResult)
+ if txIndex < 0 {
+ // no unlucky tx in the block
+ return nil
+ }
+
+ if printBlockNumbers {
+ fmt.Println(height, txIndex)
+ return nil
+ }
+
+ result, err := tmDB.replayTx(appCreator, height, txIndex, state.InitialHeight)
+ if err != nil {
+ return err
+ }
+
+ if dryRun {
+ return clientCtx.PrintProto(result)
+ }
+
+ if err := tmDB.patchDB(blockResult, result); err != nil {
+ return err
+ }
+
+ // decode the tx to get eth tx hashes to log
+ tx, err := clientCtx.TxConfig.TxDecoder()(result.Tx)
+ if err != nil {
+ fmt.Println("can't parse the patched tx", result.Height, result.Index)
+ return nil
+ }
+ for _, msg := range tx.GetMsgs() {
+ ethMsg, ok := msg.(*evmtypes.MsgEthereumTx)
+ if ok {
+ fmt.Println("patched", ethMsg.Hash, result.Height, result.Index)
+ }
+ }
+ return nil
+ }
+
+ blocksFile, err := cmd.Flags().GetString(FlagBlocksFile)
+ if err != nil {
+ return err
+ }
+ if len(blocksFile) > 0 {
+ // read block numbers from file, one number per line
+ file, err := os.Open(blocksFile)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ blockNumber, err := strconv.ParseInt(scanner.Text(), 10, 64)
+ if err != nil {
+ return err
+ }
+ if err := processBlock(blockNumber); err != nil {
+ return err
+ }
+ }
+ } else {
+ startHeight, err := cmd.Flags().GetInt(FlagStartBlock)
+ if err != nil {
+ return err
+ }
+ endHeight, err := cmd.Flags().GetInt(FlagEndBlock)
+ if err != nil {
+ return err
+ }
+ if startHeight < 1 {
+ return fmt.Errorf("invalid start-block: %d", startHeight)
+ }
+ if endHeight < startHeight {
+ return fmt.Errorf("invalid end-block %d, smaller than start-block", endHeight)
+ }
+
+ for height := startHeight; height <= endHeight; height++ {
+ if err := processBlock(int64(height)); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+ },
+ }
+ cmd.Flags().String(flags.FlagChainID, "cronosmainnet_25-1", "network chain ID, only useful for psql tx indexer backend")
+ cmd.Flags().Bool(flags.FlagDryRun, false, "Print the execution result of the problematic txs without patch the database")
+ cmd.Flags().Bool(FlagPrintBlockNumbers, false, "Print the problematic block number and tx index without replay and patch")
+ cmd.Flags().String(FlagBlocksFile, "", "Read block numbers from a file instead of iterating all the blocks")
+ cmd.Flags().Int(FlagStartBlock, 1, "The start of the block range to iterate, inclusive")
+ cmd.Flags().Int(FlagEndBlock, -1, "The end of the block range to iterate, inclusive")
+
+ return cmd
+}
+
+func openAppDB(rootDir string) (dbm.DB, error) {
+ dataDir := filepath.Join(rootDir, "data")
+ return sdk.NewLevelDB("application", dataDir)
+}
+
+type tmDB struct {
+ blockStore *tmstore.BlockStore
+ stateStore sm.Store
+ txIndexer txindex.TxIndexer
+}
+
+func openTMDB(cfg *tmcfg.Config, chainID string) (*tmDB, error) {
+ // open tendermint db
+ tmdb, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "blockstore", Config: cfg})
+ if err != nil {
+ return nil, err
+ }
+ blockStore := tmstore.NewBlockStore(tmdb)
+
+ stateDB, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "state", Config: cfg})
+ if err != nil {
+ return nil, err
+ }
+ stateStore := sm.NewStore(stateDB)
+
+ txIndexer, err := newTxIndexer(cfg, chainID)
+ if err != nil {
+ return nil, err
+ }
+
+ return &tmDB{
+ blockStore, stateStore, txIndexer,
+ }, nil
+}
+
+func newTxIndexer(config *tmcfg.Config, chainID string) (txindex.TxIndexer, error) {
+ switch config.TxIndex.Indexer {
+ case "kv":
+ store, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "tx_index", Config: config})
+ if err != nil {
+ return nil, err
+ }
+
+ return kv.NewTxIndex(store), nil
+ case "psql":
+ if config.TxIndex.PsqlConn == "" {
+ return nil, errors.New(`no psql-conn is set for the "psql" indexer`)
+ }
+ es, err := psql.NewEventSink(config.TxIndex.PsqlConn, chainID)
+ if err != nil {
+ return nil, fmt.Errorf("creating psql indexer: %w", err)
+ }
+ return es.TxIndexer(), nil
+ default:
+ return nil, fmt.Errorf("unsupported tx indexer backend %s", config.TxIndex.Indexer)
+ }
+}
+
+func findUnluckyTx(blockResult *tmstate.ABCIResponses) int {
+ for txIndex, txResult := range blockResult.DeliverTxs {
+ if rpc.TxExceedsBlockGasLimit(txResult) {
+ return txIndex
+ }
+ }
+ return -1
+}
+
+// replay the tx and return the result
+func (db *tmDB) replayTx(appCreator func() *app.App, height int64, txIndex int, initialHeight int64) (*abci.TxResult, error) {
+ block := db.blockStore.LoadBlock(height)
+ if block == nil {
+ return nil, fmt.Errorf("block %d not found", height)
+ }
+ anApp := appCreator()
+ if err := anApp.LoadHeight(block.Header.Height - 1); err != nil {
+ return nil, err
+ }
+
+ pbh := block.Header.ToProto()
+ if pbh == nil {
+ return nil, errors.New("nil header")
+ }
+
+ byzVals := make([]abci.Evidence, 0)
+ for _, evidence := range block.Evidence.Evidence {
+ byzVals = append(byzVals, evidence.ABCI()...)
+ }
+
+ commitInfo := getBeginBlockValidatorInfo(block, db.stateStore, initialHeight)
+
+ _ = anApp.BeginBlock(abci.RequestBeginBlock{
+ Hash: block.Hash(),
+ Header: *pbh,
+ ByzantineValidators: byzVals,
+ LastCommitInfo: commitInfo,
+ })
+
+ // replay with infinite block gas meter, before v0.7.0 upgrade those unlucky txs are committed successfully.
+ anApp.WithBlockGasMeter(sdk.NewInfiniteGasMeter())
+
+ // run the predecessor txs
+ for _, tx := range block.Txs[:txIndex] {
+ anApp.DeliverTx(abci.RequestDeliverTx{Tx: tx})
+ }
+
+ rsp := anApp.DeliverTx(abci.RequestDeliverTx{Tx: block.Txs[txIndex]})
+ return &abci.TxResult{
+ Height: block.Header.Height,
+ Index: uint32(txIndex),
+ Tx: block.Txs[txIndex],
+ Result: rsp,
+ }, nil
+}
+
+func (db *tmDB) patchDB(blockResult *tmstate.ABCIResponses, result *abci.TxResult) error {
+ if err := db.txIndexer.Index(result); err != nil {
+ return err
+ }
+ blockResult.DeliverTxs[result.Index] = &result.Result
+ if err := db.stateStore.SaveABCIResponses(result.Height, blockResult); err != nil {
+ return err
+ }
+ return nil
+}
+
+func getBeginBlockValidatorInfo(block *tmtypes.Block, store sm.Store,
+ initialHeight int64) abci.LastCommitInfo {
+ voteInfos := make([]abci.VoteInfo, block.LastCommit.Size())
+ // Initial block -> LastCommitInfo.Votes are empty.
+ // Remember that the first LastCommit is intentionally empty, so it makes
+ // sense for LastCommitInfo.Votes to also be empty.
+ if block.Height > initialHeight {
+ lastValSet, err := store.LoadValidators(block.Height - 1)
+ if err != nil {
+ panic(err)
+ }
+
+ // Sanity check that commit size matches validator set size - only applies
+ // after first block.
+ var (
+ commitSize = block.LastCommit.Size()
+ valSetLen = len(lastValSet.Validators)
+ )
+ if commitSize != valSetLen {
+ panic(fmt.Sprintf(
+ "commit size (%d) doesn't match valset length (%d) at height %d\n\n%v\n\n%v",
+ commitSize, valSetLen, block.Height, block.LastCommit.Signatures, lastValSet.Validators,
+ ))
+ }
+
+ for i, val := range lastValSet.Validators {
+ commitSig := block.LastCommit.Signatures[i]
+ voteInfos[i] = abci.VoteInfo{
+ Validator: tmtypes.TM2PB.Validator(val),
+ SignedLastBlock: !commitSig.Absent(),
+ }
+ }
+ }
+
+ return abci.LastCommitInfo{
+ Round: block.LastCommit.Round,
+ Votes: voteInfos,
+ }
+}

cmd/cronosd/cmd/root.go

@@ -136,6 +136,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) {
 
  // add rosetta
  rootCmd.AddCommand(server.RosettaCommand(encodingConfig.InterfaceRegistry, encodingConfig.Marshaler))
+ rootCmd.AddCommand(FixUnluckyTxCmd())
 }
 
 func addModuleInitFlags(startCmd *cobra.Command) {

go.mod

@@ -162,9 +162,9 @@ replace github.com/cosmos/iavl => github.com/cosmos/iavl v0.17.3
 replace github.com/ethereum/go-ethereum => github.com/crypto-org-chain/go-ethereum v1.10.3-patched
 
 // TODO: remove when ibc-go and ethermint upgrades cosmos-sdk
-replace github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.44.6-cronos-revert
+replace github.com/cosmos/cosmos-sdk => github.com/crypto-org-chain/cosmos-sdk v0.44.6-cronos-revert-2
 
-replace github.com/tharsis/ethermint => github.com/yihuang/ethermint v0.7.2-cronos-9.0.20220427040028-1d16a8af7dfc
+replace github.com/tharsis/ethermint => github.com/crypto-org-chain/ethermint v0.7.2-cronos-15
 
 // Note: gorocksdb bindings for OptimisticTransactionDB are not merged upstream, so we use a fork
 // See https://github.com/tecbot/gorocksdb/pull/216