Skip to content

Commit 65a0d4e

Browse files
jonastheisomerfirmakThegaram
authored
feat(permissionless batches): recovery mode after permissionless batches (#1115)
* port changes from #1013 * port changes from #1068 * go.mod tidy * fix compile error * fix goimports * fix log * address review comments * upgrade golang.org/x/net to 0.23.0 * port changes from #1018 * fix tests and linter errors * address review comments * refactor rollup sync service / verifier to use CalldataBlobSource to retrieve data from L1 * add configuration and initialize blob clients * fix unit tests * remove unused code * address review comments * address more review comments * implement first version of new da-codec and to handle multiple batches submitted in one transaction * add CommitBatchDAV7 and handle multiple commit events submitted in a single transactions * fix bug due to previous batch being empty when processing the first batch within a set of batches * Allow using MPT * update to latest da-codec * add field to CommittedBatchMeta to store LastL1MessageQueueHash for CodecV7 batches * adjust rollup verifier to support CodecV7 batches * address review comments * fix issues after merge * go mod tidy * fix unit tests * update da-codec * add test TestValidateBatchCodecV7 * go mod tidy * do not log error on shutdown * add sanity check for version to deserialization of committedBatchMetaV7 * port changes from #1073 * chore: auto version bump [bot] * address review comments * add more logs * disable ENRUpdater if DA sync mode is enabled * exit pipeline if context is cancelled * correctly handle override by setting the head of the chain to the parent's height so that created blocks will always become part of canonical chain * fix error with genesis event being nil * chore: auto version bump [bot] * chore: auto version bump [bot] * goimports --------- Co-authored-by: Ömer Faruk Irmak <omerfirmak@gmail.com> Co-authored-by: Thegaram <Thegaram@users.noreply.github.com> Co-authored-by: jonastheis <jonastheis@users.noreply.github.com> Co-authored-by: Péter Garamvölgyi <peter@scroll.io>
1 parent 60cbf64 commit 65a0d4e

File tree

14 files changed

+215
-53
lines changed

14 files changed

+215
-53
lines changed

‎cmd/geth/main.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,11 @@ var (
178178
utils.DABlockNativeAPIEndpointFlag,
179179
utils.DABlobScanAPIEndpointFlag,
180180
utils.DABeaconNodeAPIEndpointFlag,
181+
utils.DARecoveryModeFlag,
182+
utils.DARecoveryInitialL1BlockFlag,
183+
utils.DARecoveryInitialBatchFlag,
184+
utils.DARecoverySignBlocksFlag,
185+
utils.DARecoveryL2EndBlockFlag,
181186
}
182187

183188
rpcFlags = []cli.Flag{

‎cmd/utils/flags.go

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -890,22 +890,42 @@ var (
890890
}
891891

892892
// DA syncing settings
893-
DASyncEnabledFlag = &cli.BoolFlag{
893+
DASyncEnabledFlag = cli.BoolFlag{
894894
Name: "da.sync",
895895
Usage: "Enable node syncing from DA",
896896
}
897-
DABlobScanAPIEndpointFlag = &cli.StringFlag{
897+
DABlobScanAPIEndpointFlag = cli.StringFlag{
898898
Name: "da.blob.blobscan",
899899
Usage: "BlobScan blob API endpoint",
900900
}
901-
DABlockNativeAPIEndpointFlag = &cli.StringFlag{
901+
DABlockNativeAPIEndpointFlag = cli.StringFlag{
902902
Name: "da.blob.blocknative",
903903
Usage: "BlockNative blob API endpoint",
904904
}
905-
DABeaconNodeAPIEndpointFlag = &cli.StringFlag{
905+
DABeaconNodeAPIEndpointFlag = cli.StringFlag{
906906
Name: "da.blob.beaconnode",
907907
Usage: "Beacon node API endpoint",
908908
}
909+
DARecoveryModeFlag = cli.BoolFlag{
910+
Name: "da.recovery",
911+
Usage: "Enable recovery mode for DA syncing",
912+
}
913+
DARecoveryInitialL1BlockFlag = cli.Uint64Flag{
914+
Name: "da.recovery.initiall1block",
915+
Usage: "Initial L1 block to start recovery from",
916+
}
917+
DARecoveryInitialBatchFlag = cli.Uint64Flag{
918+
Name: "da.recovery.initialbatch",
919+
Usage: "Initial batch to start recovery from",
920+
}
921+
DARecoverySignBlocksFlag = cli.BoolFlag{
922+
Name: "da.recovery.signblocks",
923+
Usage: "Sign blocks during recovery (requires correct Clique signer key and history of blocks with Clique signatures)",
924+
}
925+
DARecoveryL2EndBlockFlag = cli.Uint64Flag{
926+
Name: "da.recovery.l2endblock",
927+
Usage: "End L2 block to recover to",
928+
}
909929
)
910930

911931
// MakeDataDir retrieves the currently requested data directory, terminating
@@ -1658,6 +1678,21 @@ func setDA(ctx *cli.Context, cfg *ethconfig.Config) {
16581678
if ctx.IsSet(DABeaconNodeAPIEndpointFlag.Name) {
16591679
cfg.DA.BeaconNodeAPIEndpoint = ctx.String(DABeaconNodeAPIEndpointFlag.Name)
16601680
}
1681+
if ctx.IsSet(DARecoveryModeFlag.Name) {
1682+
cfg.DA.RecoveryMode = ctx.Bool(DARecoveryModeFlag.Name)
1683+
}
1684+
if ctx.IsSet(DARecoveryInitialL1BlockFlag.Name) {
1685+
cfg.DA.InitialL1Block = ctx.Uint64(DARecoveryInitialL1BlockFlag.Name)
1686+
}
1687+
if ctx.IsSet(DARecoveryInitialBatchFlag.Name) {
1688+
cfg.DA.InitialBatch = ctx.Uint64(DARecoveryInitialBatchFlag.Name)
1689+
}
1690+
if ctx.IsSet(DARecoverySignBlocksFlag.Name) {
1691+
cfg.DA.SignBlocks = ctx.Bool(DARecoverySignBlocksFlag.Name)
1692+
}
1693+
if ctx.IsSet(DARecoveryL2EndBlockFlag.Name) {
1694+
cfg.DA.L2EndBlock = ctx.Uint64(DARecoveryL2EndBlockFlag.Name)
1695+
}
16611696
}
16621697

16631698
func setMaxBlockRange(ctx *cli.Context, cfg *ethconfig.Config) {

‎core/blockchain.go

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1806,15 +1806,15 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
18061806
return it.index, err
18071807
}
18081808

1809-
func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types.Header, txs types.Transactions) (WriteStatus, error) {
1809+
func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types.Header, txs types.Transactions, sign bool) (*types.Block, WriteStatus, error) {
18101810
if !bc.chainmu.TryLock() {
1811-
return NonStatTy, errInsertionInterrupted
1811+
return nil, NonStatTy, errInsertionInterrupted
18121812
}
18131813
defer bc.chainmu.Unlock()
18141814

18151815
statedb, err := state.New(parentBlock.Root(), bc.stateCache, bc.snaps)
18161816
if err != nil {
1817-
return NonStatTy, err
1817+
return nil, NonStatTy, err
18181818
}
18191819

18201820
statedb.StartPrefetcher("l1sync", nil)
@@ -1825,18 +1825,51 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types
18251825
tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil)
18261826
receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig)
18271827
if err != nil {
1828-
return NonStatTy, fmt.Errorf("error processing block: %w", err)
1828+
return nil, NonStatTy, fmt.Errorf("error processing block: %w", err)
18291829
}
18301830

18311831
// TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique
18321832
// This should be done with https://github.com/scroll-tech/go-ethereum/pull/913.
18331833

1834-
// finalize and assemble block as fullBlock
1834+
if sign {
1835+
// remember the time as Clique will override it
1836+
originalTime := header.Time
1837+
1838+
err = bc.engine.Prepare(bc, header)
1839+
if err != nil {
1840+
return nil, NonStatTy, fmt.Errorf("error preparing block %d: %w", tempBlock.Number().Uint64(), err)
1841+
}
1842+
1843+
// we want to re-sign the block: set time to original value again.
1844+
header.Time = originalTime
1845+
}
1846+
1847+
// finalize and assemble block as fullBlock: replicates consensus.FinalizeAndAssemble()
18351848
header.GasUsed = gasUsed
18361849
header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number))
18371850

18381851
fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil))
18391852

1853+
// Sign the block if requested
1854+
if sign {
1855+
resultCh, stopCh := make(chan *types.Block), make(chan struct{})
1856+
if err = bc.engine.Seal(bc, fullBlock, resultCh, stopCh); err != nil {
1857+
return nil, NonStatTy, fmt.Errorf("error sealing block %d: %w", fullBlock.Number().Uint64(), err)
1858+
}
1859+
// Clique.Seal() will only wait for a second before giving up on us. So make sure there is nothing computational heavy
1860+
// or a call that blocks between the call to Seal and the line below. Seal might introduce some delay, so we keep track of
1861+
// that artificially added delay and subtract it from overall runtime of commit().
1862+
fullBlock = <-resultCh
1863+
if fullBlock == nil {
1864+
return nil, NonStatTy, fmt.Errorf("sealing block failed %d: block is nil", header.Number.Uint64())
1865+
}
1866+
1867+
// verify the generated block with local consensus engine to make sure everything is as expected
1868+
if err = bc.engine.VerifyHeader(bc, fullBlock.Header(), true); err != nil {
1869+
return nil, NonStatTy, fmt.Errorf("error verifying signed block %d: %w", fullBlock.Number().Uint64(), err)
1870+
}
1871+
}
1872+
18401873
blockHash := fullBlock.Hash()
18411874
// manually replace the block hash in the receipts
18421875
for i, receipt := range receipts {
@@ -1856,16 +1889,17 @@ func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types
18561889
// Make sure the block body is valid e.g. ordering of L1 messages is correct and continuous.
18571890
if err = bc.validator.ValidateBody(fullBlock); err != nil {
18581891
bc.reportBlock(fullBlock, receipts, err)
1859-
return NonStatTy, fmt.Errorf("error validating block body %d: %w", fullBlock.Number().Uint64(), err)
1892+
return nil, NonStatTy, fmt.Errorf("error validating block body %d: %w", fullBlock.Number().Uint64(), err)
18601893
}
18611894

18621895
// Double check: even though we just built the block, make sure it is valid.
18631896
if err = bc.validator.ValidateState(fullBlock, statedb, receipts, gasUsed); err != nil {
18641897
bc.reportBlock(fullBlock, receipts, err)
1865-
return NonStatTy, fmt.Errorf("error validating block %d: %w", fullBlock.Number().Uint64(), err)
1898+
return nil, NonStatTy, fmt.Errorf("error validating block %d: %w", fullBlock.Number().Uint64(), err)
18661899
}
18671900

1868-
return bc.writeBlockWithState(fullBlock, receipts, logs, statedb, false)
1901+
writeStatus, err := bc.writeBlockWithState(fullBlock, receipts, logs, statedb, false)
1902+
return fullBlock, writeStatus, err
18691903
}
18701904

18711905
// insertSideChain is called when an import batch hits upon a pruned ancestor

‎eth/backend.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -588,7 +588,10 @@ func (s *Ethereum) Protocols() []p2p.Protocol {
588588
// Start implements node.Lifecycle, starting all internal goroutines needed by the
589589
// Ethereum protocol implementation.
590590
func (s *Ethereum) Start() error {
591-
eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode())
591+
// handler is not enabled when DA syncing enabled
592+
if !s.config.EnableDASyncing {
593+
eth.StartENRUpdater(s.blockchain, s.p2pServer.LocalNode())
594+
}
592595

593596
// Start the bloom bits servicing goroutines
594597
s.startBloomHandlers(params.BloomBitsBlocks)

‎params/version.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
const (
2525
VersionMajor = 5 // Major version component of the current release
2626
VersionMinor = 8 // Minor version component of the current release
27-
VersionPatch = 10 // Patch version component of the current release
27+
VersionPatch = 11 // Patch version component of the current release
2828
VersionMeta = "mainnet" // Version metadata to append to the version string
2929
)
3030

‎rollup/da_syncer/batch_queue.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ func (bq *BatchQueue) NextBatch(ctx context.Context) (da.Entry, error) {
3636
}
3737

3838
for {
39+
select {
40+
case <-ctx.Done():
41+
return nil, ctx.Err()
42+
default:
43+
}
44+
3945
daEntry, err := bq.DAQueue.NextDA(ctx)
4046
if err != nil {
4147
return nil, err

‎rollup/da_syncer/da/calldata_blob_source.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/scroll-tech/go-ethereum/accounts/abi"
1111
"github.com/scroll-tech/go-ethereum/common"
1212
"github.com/scroll-tech/go-ethereum/ethdb"
13+
"github.com/scroll-tech/go-ethereum/log"
1314
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client"
1415
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
1516
"github.com/scroll-tech/go-ethereum/rollup/l1"
@@ -65,6 +66,8 @@ func (ds *CalldataBlobSource) NextData() (Entries, error) {
6566
to = min(to, ds.l1Finalized)
6667
}
6768

69+
log.Debug("Fetching rollup events", "from", ds.l1Height, "to", to, "finalized", ds.l1Finalized)
70+
6871
if ds.l1Height > to {
6972
return nil, ErrSourceExhausted
7073
}
@@ -194,7 +197,7 @@ func (ds *CalldataBlobSource) getCommitBatchDA(commitEvents []*l1.CommitBatchEve
194197
}
195198

196199
if commitEvents[0].BatchIndex().Uint64() == 0 {
197-
return Entries{NewCommitBatchDAV0Empty()}, nil
200+
return Entries{NewCommitBatchDAV0Empty(commitEvents[0])}, nil
198201
}
199202

200203
firstCommitEvent := commitEvents[0]

‎rollup/da_syncer/da/commitV0.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/scroll-tech/go-ethereum/core/rawdb"
1111
"github.com/scroll-tech/go-ethereum/core/types"
1212
"github.com/scroll-tech/go-ethereum/ethdb"
13+
"github.com/scroll-tech/go-ethereum/log"
1314
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
1415
"github.com/scroll-tech/go-ethereum/rollup/l1"
1516
)
@@ -65,9 +66,10 @@ func NewCommitBatchDAV0WithChunks(db ethdb.Database,
6566
}, nil
6667
}
6768

68-
func NewCommitBatchDAV0Empty() *CommitBatchDAV0 {
69+
func NewCommitBatchDAV0Empty(event *l1.CommitBatchEvent) *CommitBatchDAV0 {
6970
return &CommitBatchDAV0{
7071
batchIndex: 0,
72+
event: event,
7173
}
7274
}
7375

@@ -172,6 +174,7 @@ func getL1Messages(db ethdb.Database, parentTotalL1MessagePopped uint64, skipped
172174
}
173175
l1Tx := rawdb.ReadL1Message(db, currentIndex)
174176
if l1Tx == nil {
177+
log.Info("L1 message not yet available", "index", currentIndex)
175178
// message not yet available
176179
// we return serrors.EOFError as this will be handled in the syncing pipeline with a backoff and retry
177180
return nil, serrors.EOFError

‎rollup/da_syncer/da/commitV7.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99

1010
"github.com/scroll-tech/go-ethereum/core/rawdb"
1111
"github.com/scroll-tech/go-ethereum/core/types"
12+
"github.com/scroll-tech/go-ethereum/log"
1213
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client"
1314
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
1415
"github.com/scroll-tech/go-ethereum/rollup/l1"
@@ -168,6 +169,7 @@ func getL1MessagesV7(db ethdb.Database, blocks []encoding.DABlock, initialL1Mess
168169
for i := messageIndex; i < messageIndex+uint64(block.NumL1Messages()); i++ {
169170
l1Tx := rawdb.ReadL1Message(db, i)
170171
if l1Tx == nil {
172+
log.Info("L1 message not yet available", "index", i)
171173
// message not yet available
172174
// we return serrors.EOFError as this will be handled in the syncing pipeline with a backoff and retry
173175
return nil, serrors.EOFError

‎rollup/da_syncer/da_queue.go

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,43 +4,56 @@ import (
44
"context"
55
"errors"
66

7+
"github.com/scroll-tech/go-ethereum/log"
78
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
89
"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
910
)
1011

1112
// DAQueue is a pipeline stage that reads DA entries from a DataSource and provides them to the next stage.
1213
type DAQueue struct {
13-
l1height uint64
14+
l1height uint64
15+
initialBatch uint64
16+
1417
dataSourceFactory *DataSourceFactory
1518
dataSource DataSource
1619
da da.Entries
1720
}
1821

19-
func NewDAQueue(l1height uint64, dataSourceFactory *DataSourceFactory) *DAQueue {
22+
func NewDAQueue(l1height uint64, initialBatch uint64, dataSourceFactory *DataSourceFactory) *DAQueue {
2023
return &DAQueue{
2124
l1height: l1height,
25+
initialBatch: initialBatch,
2226
dataSourceFactory: dataSourceFactory,
2327
dataSource: nil,
2428
da: make(da.Entries, 0),
2529
}
2630
}
2731

2832
func (dq *DAQueue) NextDA(ctx context.Context) (da.Entry, error) {
29-
for len(dq.da) == 0 {
33+
for {
3034
select {
3135
case <-ctx.Done():
3236
return nil, ctx.Err()
3337
default:
3438
}
3539

36-
err := dq.getNextData(ctx)
37-
if err != nil {
38-
return nil, err
40+
for len(dq.da) == 0 {
41+
err := dq.getNextData(ctx)
42+
if err != nil {
43+
return nil, err
44+
}
45+
}
46+
47+
daEntry := dq.da[0]
48+
dq.da = dq.da[1:]
49+
50+
if daEntry.BatchIndex() < dq.initialBatch {
51+
log.Debug("Skipping DA entry due to initial batch requirement", "batchIndex", daEntry.BatchIndex(), "initialBatch", dq.initialBatch)
52+
continue
3953
}
54+
55+
return daEntry, nil
4056
}
41-
daEntry := dq.da[0]
42-
dq.da = dq.da[1:]
43-
return daEntry, nil
4457
}
4558

4659
func (dq *DAQueue) getNextData(ctx context.Context) error {

0 commit comments

Comments
 (0)