fix(prover): ensure L2 reorg finished before generating proofs && add…

… `verificationCheckTicker` (#277)

prover/prover.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -235,19 +236,35 @@ func (p *Prover) eventLoop() {
 		}
 	}
 
+	lastLatestVerifiedL1Height := p.latestVerifiedL1Height
+
 	// If there is too many (TaikoData.Config.maxNumBlocks) pending blocks in TaikoL1 contract, there will be no new
 	// BlockProposed temporarily, so except the BlockProposed subscription, we need another trigger to start
 	// fetching the proposed blocks.
 	forceProvingTicker := time.NewTicker(15 * time.Second)
 	defer forceProvingTicker.Stop()
 
+	// If there is no new block verification in `proofCooldownPeriod * 2` seconeds, and the current prover is
+	// a special prover, we will go back to try proving the block whose id is `lastVerifiedBlockId + 1`.
+	verificationCheckTicker := time.NewTicker(
+		time.Duration(p.protocolConfigs.ProofCooldownPeriod.Uint64()*2) * time.Second,
+	)
+	defer verificationCheckTicker.Stop()
+
 	// Call reqProving() right away to catch up with the latest state.
 	reqProving()
 
 	for {
 		select {
 		case <-p.ctx.Done():
 			return
+		case <-verificationCheckTicker.C:
+			if err := backoff.Retry(
+				func() error { return p.checkChainVerification(lastLatestVerifiedL1Height) },
+				backoff.NewExponentialBackOff(),
+			); err != nil {
+				log.Error("Check chain verification error", "error", err)
+			}
 		case proofWithHeader := <-p.proofGenerationCh:
 			p.submitProofOp(p.ctx, proofWithHeader)
 		case <-p.proveNotify:
@@ -345,6 +362,27 @@ func (p *Prover) onBlockProposed(
 	if event.Id.Uint64() <= p.lastHandledBlockID {
 		return nil
 	}
+
+	currentL1OriginHeader, err := p.rpc.L1.HeaderByNumber(ctx, new(big.Int).SetUint64(event.Meta.L1Height))
+	if err != nil {
+		return err
+	}
+
+	if currentL1OriginHeader.Hash() != event.Meta.L1Hash {
+		log.Warn(
+			"L1 block hash mismatch due to L1 reorg",
+			"height", event.Meta.L1Height,
+			"currentL1OriginHeader", currentL1OriginHeader.Hash(),
+			"L1HashInEvent", event.Meta.L1Hash,
+		)
+
+		return fmt.Errorf(
+			"L1 block hash mismatch due to L1 reorg: %s != %s",
+			currentL1OriginHeader.Hash(),
+			event.Meta.L1Hash,
+		)
+	}
+
 	log.Info(
 		"Proposed block",
 		"L1Height", event.Raw.BlockNumber,
@@ -535,7 +573,7 @@ func (p *Prover) initL1Current(startingBlockID *big.Int) error {
 	latestVerifiedHeaderL1Origin, err := p.rpc.L2.L1OriginByID(p.ctx, startingBlockID)
 	if err != nil {
 		if err.Error() == ethereum.NotFound.Error() {
-			log.Warn("Failed to find L1Origin for blockID: %d, use latest L1 head instead", startingBlockID)
+			log.Warn("Failed to find L1Origin for blockID, use latest L1 head instead", "blockID", startingBlockID)
 			l1Head, err := p.rpc.L1.BlockNumber(p.ctx)
 			if err != nil {
 				return err
@@ -574,6 +612,33 @@ func (p *Prover) closeSubscription() {
 	p.blockProposedSub.Unsubscribe()
 }
 
+// checkChainVerification checks if there is no new block verification in protocol, if so,
+// it will let current sepecial prover to go back to try proving the block whose id is `lastVerifiedBlockId + 1`.
+func (p *Prover) checkChainVerification(lastLatestVerifiedL1Height uint64) error {
+	if (!p.cfg.SystemProver && !p.cfg.OracleProver) || lastLatestVerifiedL1Height != p.latestVerifiedL1Height {
+		return nil
+	}
+
+	log.Warn(
+		"No new block verification in `proofCooldownPeriod * 2` seconeds",
+		"latestVerifiedL1Height", p.latestVerifiedL1Height,
+		"proofCooldownPeriod", p.protocolConfigs.ProofCooldownPeriod,
+	)
+
+	stateVar, err := p.rpc.TaikoL1.GetStateVariables(nil)
+	if err != nil {
+		log.Error("Failed to get protocol state variables", "error", err)
+		return err
+	}
+
+	if err := p.initL1Current(new(big.Int).SetUint64(stateVar.LastVerifiedBlockId)); err != nil {
+		return err
+	}
+	p.lastHandledBlockID = stateVar.LastVerifiedBlockId
+
+	return nil
+}
+
 // cancelProofIfValid cancels proof only if the parentGasUsed and parentHash in the proof match what
 // is expected
 func (p *Prover) cancelProofIfValid(

prover/prover_test.go

@@ -145,6 +145,13 @@ func (s *ProverTestSuite) TestStartSubscription() {
 	s.NotPanics(s.p.closeSubscription)
 }
 
+func (s *ProverTestSuite) TestCheckChainVerification() {
+	s.Nil(s.p.checkChainVerification(0))
+	s.p.latestVerifiedL1Height = 1024
+	s.p.cfg.SystemProver = true
+	s.Nil(s.p.checkChainVerification(1024))
+}
+
 func (s *ProverTestSuite) TestStartClose() {
 	s.Nil(s.p.Start())
 	s.cancel()