Main patch from tm-v0.34.17 (#389)

* crypto: Remove build flags from secp256k1 (#8051)

Manual backport of #7823.

* remove cgo build flags
* remove nocgo file

Co-authored-by: Marko <marbar3778@yahoo.com>

* consensus: start the timeout ticker before replay (backport #7844) (#8079)

Co-authored-by: M. J. Fromberger <fromberger@interchain.io>
Co-authored-by: Sam Kleinman <garen@tychoish.com>

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

* cmd: make reset more safe (backport #8081) (#8089)

Backport notes:

- Revert command declaration to the old explicit format.
- Remove threading of the keyType argument.
- Fix function naming collision.
- Fix error handling.
- Restore snake-case deprecation warnings.

* Fix merge-bug: `cmd: make reset more safe (backport #8081) (#8089)`

* Fix lint: `cmd: make reset more safe (backport #8081) (#8089)`

* Fix codecov: `cmd/ostracon/commands/reset_priv_validator_test.go`

* consensus: change lock handling in 'handleMsg' method (#7992)

* change lock handling in consensus state file

* add comment explaining the unlock

* comment fix

* Update consensus/state.go

Co-authored-by: M. J. Fromberger <fromberger@interchain.io>

* spelling fix

Co-authored-by: M. J. Fromberger <fromberger@interchain.io>

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

* consensus: change lock handling in reactor for RoundState (#7994)

This change updates the lock handling in the consensus reactor. The consensus reactor now periodically fetches the RoundState and the gossip routines operate on this fetched copy instead of fetching the latest copy in each iteration of the gossip routine.

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

* stop using %w in testing errors (#8241)

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

* consensus: change lock to handle panics (#8237)

Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

Co-authored-by: Marko <marbar3778@yahoo.com>
Co-authored-by: William Banfield <4561443+williambanfield@users.noreply.github.com>

cmd/ostracon/commands/reset_priv_validator.go

@@ -2,6 +2,7 @@ package commands
 
 import (
  "os"
+ "path/filepath"
 
  "github.com/spf13/cobra"
 
@@ -16,12 +17,22 @@ var ResetAllCmd = &cobra.Command{
  Use: "unsafe-reset-all",
  Aliases: []string{"unsafe_reset_all"},
  Short: "(unsafe) Remove all the data and WAL, reset this node's validator to genesis state",
- Run:  resetAll,
+ RunE: resetAllCmd,
  PreRun: deprecateSnakeCase,
 }
 
 var keepAddrBook bool
 
+// ResetStateCmd removes the database of the specified Tendermint core instance.
+var ResetStateCmd = &cobra.Command{
+ Use: "reset-state",
+ Short: "Remove all the data and WAL",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return resetState(config.DBDir(), logger)
+ },
+ PreRun: deprecateSnakeCase,
+}
+
 func init() {
  ResetAllCmd.Flags().BoolVar(&keepAddrBook, "keep-addr-book", false, "keep the address book intact")
  ResetAllCmd.Flags().String("priv_key_type", config.PrivKeyType,
@@ -41,12 +52,9 @@ var ResetPrivValidatorCmd = &cobra.Command{
 
 // XXX: this is totally unsafe.
 // it's only suitable for testnets.
-func resetAll(cmd *cobra.Command, args []string) {
- err := ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorKeyFile(),
+func resetAllCmd(cmd *cobra.Command, args []string) error {
+ return resetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorKeyFile(),
  config.PrivValidatorStateFile(), config.PrivValidatorKeyType(), logger)
- if err != nil {
- panic(err)
- }
 }
 
 // XXX: this is totally unsafe.
@@ -59,9 +67,8 @@ func resetPrivValidator(cmd *cobra.Command, args []string) {
  }
 }
 
-// ResetAll removes address book files plus all data, and resets the privValdiator data.
-// Exported so other CLI tools can use it.
-func ResetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile, privKeyType string, logger log.Logger) error {
+// resetAll removes address book files plus all data, and resets the privValdiator data.
+func resetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile, privKeyType string, logger log.Logger) error {
  if keepAddrBook {
  logger.Info("The address book remains intact")
  } else {
@@ -72,13 +79,76 @@ func ResetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile, privKeyType
  } else {
  logger.Error("Error removing all blockchain history", "dir", dbDir, "err", err)
  }
+
  // recreate the dbDir since the privVal state needs to live there
  if err := tmos.EnsureDir(dbDir, 0700); err != nil {
  logger.Error("unable to recreate dbDir", "err", err)
  }
  return resetFilePV(privValKeyFile, privValStateFile, privKeyType, logger)
 }
 
+// resetState removes address book files plus all databases.
+func resetState(dbDir string, logger log.Logger) error {
+ blockdb := filepath.Join(dbDir, "blockstore.db")
+ state := filepath.Join(dbDir, "state.db")
+ wal := filepath.Join(dbDir, "cs.wal")
+ evidence := filepath.Join(dbDir, "evidence.db")
+ txIndex := filepath.Join(dbDir, "tx_index.db")
+ peerstore := filepath.Join(dbDir, "peerstore.db")
+
+ if tmos.FileExists(blockdb) {
+ if err := os.RemoveAll(blockdb); err == nil {
+ logger.Info("Removed all blockstore.db", "dir", blockdb)
+ } else {
+ logger.Error("error removing all blockstore.db", "dir", blockdb, "err", err)
+ }
+ }
+
+ if tmos.FileExists(state) {
+ if err := os.RemoveAll(state); err == nil {
+ logger.Info("Removed all state.db", "dir", state)
+ } else {
+ logger.Error("error removing all state.db", "dir", state, "err", err)
+ }
+ }
+
+ if tmos.FileExists(wal) {
+ if err := os.RemoveAll(wal); err == nil {
+ logger.Info("Removed all cs.wal", "dir", wal)
+ } else {
+ logger.Error("error removing all cs.wal", "dir", wal, "err", err)
+ }
+ }
+
+ if tmos.FileExists(evidence) {
+ if err := os.RemoveAll(evidence); err == nil {
+ logger.Info("Removed all evidence.db", "dir", evidence)
+ } else {
+ logger.Error("error removing all evidence.db", "dir", evidence, "err", err)
+ }
+ }
+
+ if tmos.FileExists(txIndex) {
+ if err := os.RemoveAll(txIndex); err == nil {
+ logger.Info("Removed tx_index.db", "dir", txIndex)
+ } else {
+ logger.Error("error removing tx_index.db", "dir", txIndex, "err", err)
+ }
+ }
+
+ if tmos.FileExists(peerstore) {
+ if err := os.RemoveAll(peerstore); err == nil {
+ logger.Info("Removed peerstore.db", "dir", peerstore)
+ } else {
+ logger.Error("error removing peerstore.db", "dir", peerstore, "err", err)
+ }
+ }
+ if err := tmos.EnsureDir(dbDir, 0700); err != nil {
+ logger.Error("unable to recreate dbDir", "err", err)
+ }
+ return nil
+}
+
 func resetFilePV(privValKeyFile, privValStateFile, privKeyType string, logger log.Logger) error {
  if _, err := os.Stat(privValKeyFile); err == nil {
  pv := privval.LoadFilePVEmptyState(privValKeyFile, privValStateFile)
@@ -90,7 +160,9 @@ func resetFilePV(privValKeyFile, privValStateFile, privKeyType string, logger lo
  if err != nil {
  return err
  }
- pv.Save()
+ if pv != nil {
+ pv.Save()
+ }
  logger.Info("Generated private validator file", "keyFile", privValKeyFile,
  "stateFile", privValStateFile)
  }

cmd/ostracon/commands/reset_priv_validator_test.go

@@ -0,0 +1,33 @@
+package commands
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func setupResetCmd(t *testing.T) {
+ clearConfig(defaultRoot)
+ config.SetRoot(defaultRoot)
+ require.NoError(t, os.MkdirAll(filepath.Dir(config.PrivValidatorKeyFile()), 0755))
+ require.NoError(t, os.MkdirAll(filepath.Dir(config.PrivValidatorStateFile()), 0755))
+}
+
+func TestResetAllCmd(t *testing.T) {
+ setupResetCmd(t)
+ err := ResetAllCmd.RunE(ResetAllCmd, nil)
+ require.NoError(t, err)
+}
+
+func TestResetStateCmd(t *testing.T) {
+ setupResetCmd(t)
+ err := ResetStateCmd.RunE(ResetStateCmd, nil)
+ require.NoError(t, err)
+}
+
+func TestResetPrivValidatorCmd(t *testing.T) {
+ setupResetCmd(t)
+ ResetPrivValidatorCmd.Run(ResetPrivValidatorCmd, nil)
+}

cmd/ostracon/main.go

@@ -21,6 +21,7 @@ func main() {
  cmd.ReplayConsoleCmd,
  cmd.ResetAllCmd,
  cmd.ResetPrivValidatorCmd,
+ cmd.ResetStateCmd,
  cmd.ShowValidatorCmd,
  cmd.TestnetFilesCmd,
  cmd.ShowNodeIDCmd,

consensus/reactor.go

@@ -46,6 +46,7 @@ type Reactor struct {
  mtx tmsync.RWMutex
  waitSync bool
  eventBus *types.EventBus
+ rs *cstypes.RoundState
 
  Metrics *Metrics
 }
@@ -58,6 +59,7 @@ func NewReactor(consensusState *State, waitSync bool, async bool, recvBufSize in
  conR := &Reactor{
  conS: consensusState,
  waitSync: waitSync,
+ rs: consensusState.GetRoundState(),
  Metrics: NopMetrics(),
  }
  conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR, async, recvBufSize)
@@ -84,6 +86,7 @@ func (conR *Reactor) OnStart() error {
  go conR.peerStatsRoutine()
 
  conR.subscribeToBroadcastEvents()
+ go conR.updateRoundStateRoutine()
 
  if !conR.WaitSync() {
  err := conR.conS.Start()
@@ -488,11 +491,31 @@ func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage)
 }
 
 func (conR *Reactor) sendNewRoundStepMessage(peer p2p.Peer) {
- rs := conR.conS.GetRoundState()
+ rs := conR.getRoundState()
  nrsMsg := makeRoundStepMessage(rs)
  peer.Send(StateChannel, MustEncode(nrsMsg))
 }
 
+func (conR *Reactor) updateRoundStateRoutine() {
+ t := time.NewTicker(100 * time.Microsecond)
+ defer t.Stop()
+ for range t.C {
+ if !conR.IsRunning() {
+ return
+ }
+ rs := conR.conS.GetRoundState()
+ conR.mtx.Lock()
+ conR.rs = rs
+ conR.mtx.Unlock()
+ }
+}
+
+func (conR *Reactor) getRoundState() *cstypes.RoundState {
+ conR.mtx.RLock()
+ defer conR.mtx.RUnlock()
+ return conR.rs
+}
+
 func (conR *Reactor) gossipDataRoutine(peer p2p.Peer, ps *PeerState) {
  logger := conR.Logger.With("peer", peer)
 
@@ -502,7 +525,7 @@ OUTER_LOOP:
  if !peer.IsRunning() || !conR.IsRunning() {
  return
  }
- rs := conR.conS.GetRoundState()
+ rs := conR.getRoundState()
  prs := ps.GetRoundState()
 
  // Send proposal Block parts?
@@ -645,7 +668,7 @@ OUTER_LOOP:
  if !peer.IsRunning() || !conR.IsRunning() {
  return
  }
- rs := conR.conS.GetRoundState()
+ rs := conR.getRoundState()
  prs := ps.GetRoundState()
 
  switch sleeping {
@@ -779,7 +802,7 @@ OUTER_LOOP:
 
  // Maybe send Height/Round/Prevotes
  {
- rs := conR.conS.GetRoundState()
+ rs := conR.getRoundState()
  prs := ps.GetRoundState()
  if rs.Height == prs.Height {
  if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok {
@@ -796,7 +819,7 @@ OUTER_LOOP:
 
  // Maybe send Height/Round/Precommits
  {
- rs := conR.conS.GetRoundState()
+ rs := conR.getRoundState()
  prs := ps.GetRoundState()
  if rs.Height == prs.Height {
  if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok {
@@ -813,7 +836,7 @@ OUTER_LOOP:
 
  // Maybe send Height/Round/ProposalPOL
  {
- rs := conR.conS.GetRoundState()
+ rs := conR.getRoundState()
  prs := ps.GetRoundState()
  if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 {
  if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok {