From 3ded42cb26b557b8dc3278ace2171719b87c4707 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 6 Jun 2022 10:16:13 +0800 Subject: [PATCH 1/6] Problem: no command to patch the duplicated tx situation Closes: #521 Solution: - add reindex-duplicated-tx command --- CHANGELOG.md | 1 + cmd/cronosd/cmd/reindex-duplicated-tx.go | 251 +++++++++++++++++++++++ cmd/cronosd/cmd/root.go | 1 + 3 files changed, 253 insertions(+) create mode 100644 cmd/cronosd/cmd/reindex-duplicated-tx.go diff --git a/CHANGELOG.md b/CHANGELOG.md index b85626e78c..26a6211d0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [cronos#489](https://github.com/crypto-org-chain/cronos/pull/489) Enable jemalloc memory allocator, and update rocksdb src to `v6.29.5`. - [#513](https://github.com/crypto-org-chain/cronos/pull/513) Add `fix-unlucky-tx` command to patch txs post v0.7.0 upgrade. +- [cronos#522](https://github.com/crypto-org-chain/cronos/pull/522) Add `reindex-duplicated-tx` command to handle the tendermint tx duplicated issue. *May 3, 2022* diff --git a/cmd/cronosd/cmd/reindex-duplicated-tx.go b/cmd/cronosd/cmd/reindex-duplicated-tx.go new file mode 100644 index 0000000000..dd51771071 --- /dev/null +++ b/cmd/cronosd/cmd/reindex-duplicated-tx.go @@ -0,0 +1,251 @@ +package cmd + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "runtime" + "strconv" + "sync" + + "github.com/spf13/cobra" + + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + abci "github.com/tendermint/tendermint/abci/types" + tmcfg "github.com/tendermint/tendermint/config" + tmnode "github.com/tendermint/tendermint/node" + sm "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/state/indexer/sink/psql" + "github.com/tendermint/tendermint/state/txindex" + "github.com/tendermint/tendermint/state/txindex/kv" + "github.com/tendermint/tendermint/state/txindex/null" + tmstore "github.com/tendermint/tendermint/store" +) + +const ( + FlagPrintTxs = "print-txs" + FlagBlocksFile = "blocks-file" + FlagStartBlock = "start-block" + FlagEndBlock = "end-block" + FlagConcurrency = "concurrency" +) + +// ReindexDuplicatedTx update the tx execution result of false-failed tx in tendermint db +func ReindexDuplicatedTx() *cobra.Command { + cmd := &cobra.Command{ + Use: "reindex-duplicated-tx", + Short: "Reindex tx that suffer from tendermint duplicated tx issue", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := server.GetServerContextFromCmd(cmd) + + chainID, err := cmd.Flags().GetString(flags.FlagChainID) + if err != nil { + return err + } + printTxs, err := cmd.Flags().GetBool(FlagPrintTxs) + if err != nil { + return err + } + + tmDB, err := openTMDB(ctx.Config, chainID) + if err != nil { + return err + } + + // iterate and patch a single block + processBlock := func(height int64) error { + blockResult, err := tmDB.stateStore.LoadABCIResponses(height) + if err != nil { + return err + } + block := tmDB.blockStore.LoadBlock(height) + if err != nil { + return err + } + if block == nil { + return fmt.Errorf("block not found: %d", height) + } + + for txIndex, txResult := range blockResult.DeliverTxs { + tx := block.Txs[txIndex] + txHash := tx.Hash() + indexed, err := tmDB.txIndexer.Get(txHash) + if err != nil { + return err + } + if txResult.Code == 0 && txResult.Code != indexed.Result.Code { + if printTxs { + fmt.Println(height, txIndex) + continue + } + // a success tx but indexed wrong, reindex the tx + result := &abci.TxResult{ + Height: height, + Index: uint32(txIndex), + Tx: tx, + Result: *txResult, + } + + if err := tmDB.txIndexer.Index(result); err != nil { + return err + } + } + } + + return nil + } + + concurrency, err := cmd.Flags().GetInt(FlagConcurrency) + if err != nil { + return err + } + + blockChan := make(chan int64, concurrency) + var wg sync.WaitGroup + ctCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func(wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case <-ctCtx.Done(): + return + case blockNum, ok := <-blockChan: + if !ok { + return + } + + if err := processBlock(blockNum); err != nil { + fmt.Fprintf(os.Stderr, "process block failed: %d, %+v", blockNum, err) + cancel() + return + } + } + } + }(&wg) + } + + blocksFile, err := cmd.Flags().GetString(FlagBlocksFile) + if err != nil { + return err + } + findBlock := func() error { + if len(blocksFile) > 0 { + // read block numbers from file, one number per line + file, err := os.Open(blocksFile) + if err != nil { + return err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + blockNumber, err := strconv.ParseInt(scanner.Text(), 10, 64) + if err != nil { + return err + } + blockChan <- blockNumber + } + } else { + startHeight, err := cmd.Flags().GetInt(FlagStartBlock) + if err != nil { + return err + } + endHeight, err := cmd.Flags().GetInt(FlagEndBlock) + if err != nil { + return err + } + if startHeight < 1 { + return fmt.Errorf("invalid start-block: %d", startHeight) + } + if endHeight < startHeight { + return fmt.Errorf("invalid end-block %d, smaller than start-block", endHeight) + } + + for height := startHeight; height <= endHeight; height++ { + blockChan <- int64(height) + } + } + return nil + } + + go func() { + if err := findBlock(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + close(blockChan) + }() + + wg.Wait() + + return ctCtx.Err() + }, + } + cmd.Flags().String(flags.FlagChainID, "cronosmainnet_25-1", "network chain ID, only useful for psql tx indexer backend") + cmd.Flags().Bool(FlagPrintTxs, false, "Print the block number and tx indexes of the duplicated txs without patch") + cmd.Flags().String(FlagBlocksFile, "", "Read block numbers from a file instead of iterating all the blocks") + cmd.Flags().Int(FlagStartBlock, 1, "The start of the block range to iterate, inclusive") + cmd.Flags().Int(FlagEndBlock, -1, "The end of the block range to iterate, inclusive") + cmd.Flags().Int(FlagConcurrency, runtime.NumCPU(), "Define how many workers run in concurrency") + + return cmd +} + +type tmDB struct { + blockStore *tmstore.BlockStore + stateStore sm.Store + txIndexer txindex.TxIndexer +} + +func openTMDB(cfg *tmcfg.Config, chainID string) (*tmDB, error) { + // open tendermint db + tmdb, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "blockstore", Config: cfg}) + if err != nil { + return nil, err + } + blockStore := tmstore.NewBlockStore(tmdb) + + stateDB, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "state", Config: cfg}) + if err != nil { + return nil, err + } + stateStore := sm.NewStore(stateDB) + + txIndexer, err := newTxIndexer(cfg, chainID) + if err != nil { + return nil, err + } + + return &tmDB{ + blockStore, stateStore, txIndexer, + }, nil +} + +func newTxIndexer(config *tmcfg.Config, chainID string) (txindex.TxIndexer, error) { + switch config.TxIndex.Indexer { + case "kv": + store, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "tx_index", Config: config}) + if err != nil { + return nil, err + } + + return kv.NewTxIndex(store), nil + case "psql": + if config.TxIndex.PsqlConn == "" { + return nil, errors.New(`no psql-conn is set for the "psql" indexer`) + } + es, err := psql.NewEventSink(config.TxIndex.PsqlConn, chainID) + if err != nil { + return nil, fmt.Errorf("creating psql indexer: %w", err) + } + return es.TxIndexer(), nil + default: + return &null.TxIndex{}, nil + } +} diff --git a/cmd/cronosd/cmd/root.go b/cmd/cronosd/cmd/root.go index 7b7dfa87e4..278d670183 100644 --- a/cmd/cronosd/cmd/root.go +++ b/cmd/cronosd/cmd/root.go @@ -136,6 +136,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { txCommand(), ethermintclient.KeyCommands(app.DefaultNodeHome), FixUnluckyTxCmd(), + ReindexDuplicatedTx(), ) // add rosetta From a7096e9a9391b3c39c803974a94b30237b98e822 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 6 Jun 2022 10:24:57 +0800 Subject: [PATCH 2/6] reuse tmDB from fix-unlucky-tx --- cmd/cronosd/cmd/reindex-duplicated-tx.go | 62 ------------------------ 1 file changed, 62 deletions(-) diff --git a/cmd/cronosd/cmd/reindex-duplicated-tx.go b/cmd/cronosd/cmd/reindex-duplicated-tx.go index dd51771071..4f87ef629c 100644 --- a/cmd/cronosd/cmd/reindex-duplicated-tx.go +++ b/cmd/cronosd/cmd/reindex-duplicated-tx.go @@ -3,7 +3,6 @@ package cmd import ( "bufio" "context" - "errors" "fmt" "os" "runtime" @@ -15,14 +14,6 @@ import ( "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/server" abci "github.com/tendermint/tendermint/abci/types" - tmcfg "github.com/tendermint/tendermint/config" - tmnode "github.com/tendermint/tendermint/node" - sm "github.com/tendermint/tendermint/state" - "github.com/tendermint/tendermint/state/indexer/sink/psql" - "github.com/tendermint/tendermint/state/txindex" - "github.com/tendermint/tendermint/state/txindex/kv" - "github.com/tendermint/tendermint/state/txindex/null" - tmstore "github.com/tendermint/tendermint/store" ) const ( @@ -196,56 +187,3 @@ func ReindexDuplicatedTx() *cobra.Command { return cmd } - -type tmDB struct { - blockStore *tmstore.BlockStore - stateStore sm.Store - txIndexer txindex.TxIndexer -} - -func openTMDB(cfg *tmcfg.Config, chainID string) (*tmDB, error) { - // open tendermint db - tmdb, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "blockstore", Config: cfg}) - if err != nil { - return nil, err - } - blockStore := tmstore.NewBlockStore(tmdb) - - stateDB, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "state", Config: cfg}) - if err != nil { - return nil, err - } - stateStore := sm.NewStore(stateDB) - - txIndexer, err := newTxIndexer(cfg, chainID) - if err != nil { - return nil, err - } - - return &tmDB{ - blockStore, stateStore, txIndexer, - }, nil -} - -func newTxIndexer(config *tmcfg.Config, chainID string) (txindex.TxIndexer, error) { - switch config.TxIndex.Indexer { - case "kv": - store, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "tx_index", Config: config}) - if err != nil { - return nil, err - } - - return kv.NewTxIndex(store), nil - case "psql": - if config.TxIndex.PsqlConn == "" { - return nil, errors.New(`no psql-conn is set for the "psql" indexer`) - } - es, err := psql.NewEventSink(config.TxIndex.PsqlConn, chainID) - if err != nil { - return nil, fmt.Errorf("creating psql indexer: %w", err) - } - return es.TxIndexer(), nil - default: - return &null.TxIndex{}, nil - } -} From e605d3c1eda766aaecaafeee7ffe0ccbd98e6521 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 6 Jun 2022 11:17:25 +0800 Subject: [PATCH 3/6] Update cmd/cronosd/cmd/reindex-duplicated-tx.go --- cmd/cronosd/cmd/reindex-duplicated-tx.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/cronosd/cmd/reindex-duplicated-tx.go b/cmd/cronosd/cmd/reindex-duplicated-tx.go index 4f87ef629c..6d54afb6c1 100644 --- a/cmd/cronosd/cmd/reindex-duplicated-tx.go +++ b/cmd/cronosd/cmd/reindex-duplicated-tx.go @@ -38,9 +38,6 @@ func ReindexDuplicatedTx() *cobra.Command { return err } printTxs, err := cmd.Flags().GetBool(FlagPrintTxs) - if err != nil { - return err - } tmDB, err := openTMDB(ctx.Config, chainID) if err != nil { From 33ed5704c0155d487e24e253be44d72bd574f833 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 6 Jun 2022 11:22:45 +0800 Subject: [PATCH 4/6] Revert "Update cmd/cronosd/cmd/reindex-duplicated-tx.go" This reverts commit e605d3c1eda766aaecaafeee7ffe0ccbd98e6521. --- cmd/cronosd/cmd/reindex-duplicated-tx.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/cronosd/cmd/reindex-duplicated-tx.go b/cmd/cronosd/cmd/reindex-duplicated-tx.go index 6d54afb6c1..4f87ef629c 100644 --- a/cmd/cronosd/cmd/reindex-duplicated-tx.go +++ b/cmd/cronosd/cmd/reindex-duplicated-tx.go @@ -38,6 +38,9 @@ func ReindexDuplicatedTx() *cobra.Command { return err } printTxs, err := cmd.Flags().GetBool(FlagPrintTxs) + if err != nil { + return err + } tmDB, err := openTMDB(ctx.Config, chainID) if err != nil { From 003d905ff2f496e9cbffc8d5033e807d73814dc5 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Mon, 6 Jun 2022 11:25:07 +0800 Subject: [PATCH 5/6] fix lint --- cmd/cronosd/cmd/reindex-duplicated-tx.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/cronosd/cmd/reindex-duplicated-tx.go b/cmd/cronosd/cmd/reindex-duplicated-tx.go index 4f87ef629c..ab9996bdda 100644 --- a/cmd/cronosd/cmd/reindex-duplicated-tx.go +++ b/cmd/cronosd/cmd/reindex-duplicated-tx.go @@ -54,9 +54,6 @@ func ReindexDuplicatedTx() *cobra.Command { return err } block := tmDB.blockStore.LoadBlock(height) - if err != nil { - return err - } if block == nil { return fmt.Errorf("block not found: %d", height) } From 1b7698e85d4139d2c9b7e6e7cbb1543fac59455d Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 6 Jun 2022 12:12:47 +0800 Subject: [PATCH 6/6] Update cmd/cronosd/cmd/reindex-duplicated-tx.go --- cmd/cronosd/cmd/reindex-duplicated-tx.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cronosd/cmd/reindex-duplicated-tx.go b/cmd/cronosd/cmd/reindex-duplicated-tx.go index ab9996bdda..8722d7a094 100644 --- a/cmd/cronosd/cmd/reindex-duplicated-tx.go +++ b/cmd/cronosd/cmd/reindex-duplicated-tx.go @@ -28,7 +28,7 @@ const ( func ReindexDuplicatedTx() *cobra.Command { cmd := &cobra.Command{ Use: "reindex-duplicated-tx", - Short: "Reindex tx that suffer from tendermint duplicated tx issue", + Short: "Reindex tx that suffer from tendermint duplicated tx issue, it can handle both v0.6.x and v0.7.x blocks.", Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { ctx := server.GetServerContextFromCmd(cmd)