fix validator set recovery after restart (ethereum#38)

* fix validator set recovery after restart

* remove gov.GetNextValidators()

Co-authored-by: Qi Zhou <qzhou64@gmail.com>

consensus/tendermint/gov/gov.go

@@ -3,15 +3,53 @@ package gov
 import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 type Governance struct {
-	epoch uint64
-	chain consensus.ChainHeaderReader
+	config *params.TendermintConfig
+	chain  consensus.ChainHeaderReader
 }
 
-func New(epoch uint64, chain consensus.ChainHeaderReader) *Governance {
-	return &Governance{epoch: epoch, chain: chain}
+func New(config *params.TendermintConfig, chain consensus.ChainHeaderReader) *Governance {
+	return &Governance{config: config, chain: chain}
+}
+
+// Returns the validator sets for last, current, next blocks
+func (g *Governance) GetValidatorSets(height uint64) (*types.ValidatorSet, *types.ValidatorSet, *types.ValidatorSet) {
+	if height == 0 {
+		panic("cannot get genesis validator set")
+	}
+
+	last := g.GetValidatorSet(height-1, nil)
+	current := g.GetValidatorSet(height, last)
+	next := g.GetValidatorSet(height+1, current)
+	return last, current, next
+}
+
+// GetValidatorSet returns the validator set of a height
+func (g *Governance) GetValidatorSet(height uint64, lastVals *types.ValidatorSet) *types.ValidatorSet {
+	if height == 0 {
+		return &types.ValidatorSet{}
+	}
+
+	idxInEpoch := (height - 1) % g.config.Epoch
+	if idxInEpoch != 0 && lastVals != nil {
+		// use cached version if we do not have a validator change
+		cvals := lastVals.Copy()
+		cvals.IncrementProposerPriority(1)
+		return cvals
+	}
+
+	epochNumber := height - 1 - idxInEpoch
+	epochHeader := g.chain.GetHeaderByNumber(epochNumber)
+	epochVals := types.NewValidatorSet(epochHeader.NextValidators, types.U64ToI64Array(epochHeader.NextValidatorPowers), int64(g.config.Epoch))
+	if idxInEpoch != 0 {
+		epochVals.IncrementProposerPriority(int32(idxInEpoch))
+	}
+
+	return epochVals
 }
 
 // EpochValidators returns the current epoch validators that height belongs to
@@ -29,7 +67,7 @@ func (g *Governance) EpochValidatorPowers(height uint64) []uint64 {
 }
 
 func (g *Governance) NextValidators(height uint64) []common.Address {
-	if height%g.epoch != 0 {
+	if height%g.config.Epoch != 0 {
 		return []common.Address{}
 	}
 
@@ -73,7 +111,7 @@ func CompareValidatorPowers(lhs, rhs []uint64) bool {
 }
 
 func (g *Governance) NextValidatorPowers(height uint64) []uint64 {
-	if height%g.epoch != 0 {
+	if height%g.config.Epoch != 0 {
 		return []uint64{}
 	}
 

consensus/tendermint/tendermint.go

@@ -147,6 +147,10 @@ func (c *Tendermint) Init(chain *core.BlockChain, makeBlock func(parent common.H
 	store := adapter.NewStore(chain, c.VerifyHeader, makeBlock)
 
 	// p2p key
+	if TestMode {
+		// Use "" to indicate a temp key
+		c.config.NodeKeyPath = ""
+	}
 	p2pPriv, err := getOrCreateNodeKey(c.config.NodeKeyPath)
 	if err != nil {
 		return
@@ -167,26 +171,20 @@ func (c *Tendermint) Init(chain *core.BlockChain, makeBlock func(parent common.H
 		}
 	}()
 
-	gov := gov.New(c.config.Epoch, chain)
+	gov := gov.New(c.config, chain)
+
 	block := chain.CurrentHeader()
 	number := block.Number.Uint64()
-	var lastValidators []common.Address
-	var lastValidatorPowers []uint64
-	if number != 0 {
-		lastValidators = gov.EpochValidators(number - 1)
-		lastValidatorPowers = gov.EpochValidatorPowers(number - 1)
-	}
+	last, current, next := gov.GetValidatorSets(number + 1)
+
 	gcs := pbftconsensus.MakeChainState(
 		c.config.NetworkID,
 		number,
 		block.Hash(),
 		block.TimeMs,
-		gov.EpochValidators(number),
-		types.U64ToI64Array(gov.EpochValidatorPowers(number)),
-		gov.NextValidators(number),
-		types.U64ToI64Array(gov.NextValidatorPowers(number)),
-		lastValidators,
-		types.U64ToI64Array(lastValidatorPowers),
+		last,
+		current,
+		next,
 		c.config.Epoch,
 		int64(c.config.ProposerRepetition),
 	)
@@ -232,17 +230,17 @@ func EnableTestMode() {
 }
 
 func getOrCreateNodeKey(path string) (p2pcrypto.PrivKey, error) {
-	if TestMode {
-		path = ""
-	}
 	if path == "" {
+		// Create a temp key
+		log.Info("Create a temp node key")
+
 		priv, _, err := p2pcrypto.GenerateKeyPair(p2pcrypto.Ed25519, -1)
 		if err != nil {
 			panic(err)
 		}
-		// don't save priv in test mode
 		return priv, nil
 	}
+
 	b, err := ioutil.ReadFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -333,7 +331,7 @@ func (c *Tendermint) verifyHeader(chain consensus.ChainHeaderReader, header *typ
 		return consensus.ErrFutureBlock
 	}
 
-	governance := gov.New(c.config.Epoch, chain)
+	governance := gov.New(c.config, chain)
 	if !gov.CompareValidators(header.NextValidators, governance.NextValidators(number)) {
 		return errors.New("NextValidators is incorrect")
 	}
@@ -432,7 +430,7 @@ func (c *Tendermint) Prepare(chain consensus.ChainHeaderReader, header *types.He
 	header.Time = timestamp / 1000
 	header.Difficulty = big.NewInt(1)
 
-	governance := gov.New(c.config.Epoch, chain)
+	governance := gov.New(c.config, chain)
 	header.NextValidators = governance.NextValidators(number)
 	header.NextValidatorPowers = governance.NextValidatorPowers(number)
 

go.mod

@@ -4,7 +4,7 @@ go 1.15
 
 require (
 	github.com/Azure/azure-storage-blob-go v0.7.0
-	github.com/QuarkChain/go-minimal-pbft v0.0.0-20220309025448-1a13edd735c1
+	github.com/QuarkChain/go-minimal-pbft v0.0.0-20220309221209-4a8b73a5d901
 	github.com/VictoriaMetrics/fastcache v1.6.0
 	github.com/aws/aws-sdk-go-v2 v1.2.0
 	github.com/aws/aws-sdk-go-v2/config v1.1.1

go.sum

@@ -89,6 +89,8 @@ github.com/QuarkChain/go-minimal-pbft v0.0.0-20220308184838-5baea63c2394 h1:HaaW
 github.com/QuarkChain/go-minimal-pbft v0.0.0-20220308184838-5baea63c2394/go.mod h1:nC/gDMz8G2h+K5KGDiEwEGJ//D70G8mwOBgHraRUab8=
 github.com/QuarkChain/go-minimal-pbft v0.0.0-20220309025448-1a13edd735c1 h1:z/d1/Numk5zfU+VJGtSJ7j200zs7LoWRAWldHZrwTQI=
 github.com/QuarkChain/go-minimal-pbft v0.0.0-20220309025448-1a13edd735c1/go.mod h1:nC/gDMz8G2h+K5KGDiEwEGJ//D70G8mwOBgHraRUab8=
+github.com/QuarkChain/go-minimal-pbft v0.0.0-20220309221209-4a8b73a5d901 h1:2FhIbg26oKAl10utfICfNpnEUzgeSJRiYTXN9DrZXGM=
+github.com/QuarkChain/go-minimal-pbft v0.0.0-20220309221209-4a8b73a5d901/go.mod h1:nC/gDMz8G2h+K5KGDiEwEGJ//D70G8mwOBgHraRUab8=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=