op-e2e: 4844 DA support action-tests (#9104)

* op-e2e: 4844 DA support action-tests

* op-service: MakeSidecar unit-test

* op-e2e: implement fixes for reviews

* op-e2e: action test default batcher config update

op-e2e/actions/eip4844_test.go

@@ -0,0 +1,147 @@
+package actions
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+
+	batcherFlags "github.com/ethereum-optimism/optimism/op-batcher/flags"
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/sync"
+	"github.com/ethereum-optimism/optimism/op-service/testlog"
+)
+
+func setupEIP4844Test(t Testing, log log.Logger) (*e2eutils.SetupData, *e2eutils.DeployParams, *L1Miner, *L2Sequencer, *L2Engine, *L2Verifier, *L2Engine) {
+	dp := e2eutils.MakeDeployParams(t, defaultRollupTestParams)
+	genesisActivation := hexutil.Uint64(0)
+	dp.DeployConfig.L1CancunTimeOffset = &genesisActivation
+	dp.DeployConfig.L2GenesisCanyonTimeOffset = &genesisActivation
+	dp.DeployConfig.L2GenesisDeltaTimeOffset = &genesisActivation
+	dp.DeployConfig.L2GenesisEcotoneTimeOffset = &genesisActivation
+
+	sd := e2eutils.Setup(t, dp, defaultAlloc)
+	miner, seqEngine, sequencer := setupSequencerTest(t, sd, log)
+	miner.ActL1SetFeeRecipient(common.Address{'A'})
+	sequencer.ActL2PipelineFull(t)
+	verifEngine, verifier := setupVerifier(t, sd, log, miner.L1Client(t, sd.RollupCfg), miner.BlobStore(), &sync.Config{})
+	return sd, dp, miner, sequencer, seqEngine, verifier, verifEngine
+}
+
+func setupBatcher(t Testing, log log.Logger, sd *e2eutils.SetupData, dp *e2eutils.DeployParams, miner *L1Miner,
+	sequencer *L2Sequencer, engine *L2Engine, daType batcherFlags.DataAvailabilityType) *L2Batcher {
+	return NewL2Batcher(log, sd.RollupCfg, &BatcherCfg{
+		MinL1TxSize:          0,
+		MaxL1TxSize:          128_000,
+		BatcherKey:           dp.Secrets.Batcher,
+		DataAvailabilityType: daType,
+	}, sequencer.RollupClient(), miner.EthClient(), engine.EthClient(), engine.EngineClient(t, sd.RollupCfg))
+}
+
+func TestEIP4844DataAvailability(gt *testing.T) {
+	t := NewDefaultTesting(gt)
+
+	log := testlog.Logger(t, log.LvlDebug)
+	sd, dp, miner, sequencer, seqEngine, verifier, _ := setupEIP4844Test(t, log)
+
+	batcher := setupBatcher(t, log, sd, dp, miner, sequencer, seqEngine, batcherFlags.BlobsType)
+
+	sequencer.ActL2PipelineFull(t)
+	verifier.ActL2PipelineFull(t)
+
+	// build empty L1 block
+	miner.ActEmptyBlock(t)
+	// finalize it, so the L1 geth blob pool doesn't log errors about missing finality
+	miner.ActL1SafeNext(t)
+	miner.ActL1FinalizeNext(t)
+
+	// Create L2 blocks, and reference the L1 head as origin
+	sequencer.ActL1HeadSignal(t)
+	sequencer.ActBuildToL1Head(t)
+
+	// submit all new L2 blocks
+	batcher.ActSubmitAll(t)
+	batchTx := batcher.LastSubmitted
+
+	// new L1 block with L2 batch
+	miner.ActL1StartBlock(12)(t)
+	miner.ActL1IncludeTxByHash(batchTx.Hash())(t)
+	miner.ActL1EndBlock(t)
+
+	require.Equal(t, uint8(types.BlobTxType), batchTx.Type(), "batch tx must be blob-tx")
+
+	// verifier picks up the L2 chain that was submitted
+	verifier.ActL1HeadSignal(t)
+	verifier.ActL2PipelineFull(t)
+	require.Equal(t, verifier.L2Safe(), sequencer.L2Unsafe(), "verifier syncs from sequencer via L1")
+	require.NotEqual(t, sequencer.L2Safe(), sequencer.L2Unsafe(), "sequencer has not processed L1 yet")
+}
+
+func TestEIP4844DataAvailabilitySwitch(gt *testing.T) {
+	t := NewDefaultTesting(gt)
+
+	log := testlog.Logger(t, log.LvlDebug)
+	sd, dp, miner, sequencer, seqEngine, verifier, _ := setupEIP4844Test(t, log)
+
+	oldBatcher := setupBatcher(t, log, sd, dp, miner, sequencer, seqEngine, batcherFlags.CalldataType)
+
+	sequencer.ActL2PipelineFull(t)
+	verifier.ActL2PipelineFull(t)
+
+	// build empty L1 block
+	miner.ActEmptyBlock(t)
+	// finalize it, so the L1 geth blob pool doesn't log errors about missing finality
+	miner.ActL1SafeNext(t)
+	miner.ActL1FinalizeNext(t)
+
+	// Create L2 blocks, and reference the L1 head as origin
+	sequencer.ActL1HeadSignal(t)
+	sequencer.ActBuildToL1Head(t)
+
+	// submit all new L2 blocks, with legacy calldata DA
+	oldBatcher.ActSubmitAll(t)
+	batchTx := oldBatcher.LastSubmitted
+
+	// new L1 block with L2 batch
+	miner.ActL1StartBlock(12)(t)
+	miner.ActL1IncludeTxByHash(batchTx.Hash())(t)
+	miner.ActL1EndBlock(t)
+
+	require.Equal(t, uint8(types.DynamicFeeTxType), batchTx.Type(), "batch tx must be eip1559 tx")
+
+	// verifier picks up the L2 chain that was submitted
+	verifier.ActL1HeadSignal(t)
+	verifier.ActL2PipelineFull(t)
+	require.Equal(t, verifier.L2Safe(), sequencer.L2Unsafe(), "verifier syncs from sequencer via L1")
+	require.NotEqual(t, sequencer.L2Safe(), sequencer.L2Unsafe(), "sequencer has not processed L1 yet")
+
+	newBatcher := setupBatcher(t, log, sd, dp, miner, sequencer, seqEngine, batcherFlags.BlobsType)
+
+	// build empty L1 block
+	miner.ActEmptyBlock(t)
+
+	// Create L2 blocks, and reference the L1 head as origin
+	sequencer.ActL1HeadSignal(t)
+	sequencer.ActBuildToL1Head(t)
+
+	// submit all new L2 blocks, now with Blobs DA!
+	newBatcher.ActSubmitAll(t)
+	batchTx = newBatcher.LastSubmitted
+
+	// new L1 block with L2 batch
+	miner.ActL1StartBlock(12)(t)
+	miner.ActL1IncludeTxByHash(batchTx.Hash())(t)
+	miner.ActL1EndBlock(t)
+
+	require.Equal(t, uint8(types.BlobTxType), batchTx.Type(), "batch tx must be blob-tx")
+
+	// verifier picks up the L2 chain that was submitted
+	verifier.ActL1HeadSignal(t)
+	verifier.ActL2PipelineFull(t)
+	require.Equal(t, verifier.L2Safe(), sequencer.L2Unsafe(), "verifier syncs from sequencer via L1")
+	require.NotEqual(t, sequencer.L2Safe(), sequencer.L2Unsafe(), "sequencer has not processed L1 yet")
+}

op-e2e/actions/l1_miner.go

@@ -3,20 +3,28 @@ package actions
 import (
 	"math/big"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
+	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
-	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/optimism/op-e2e/e2eutils"
+	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
 )
 
 // L1Miner wraps a L1Replica with instrumented block building ability.
 type L1Miner struct {
 	L1Replica
 
+	blobStore *e2eutils.BlobsStore
+
 	// L1 block building preferences
 	prefCoinbase common.Address
 
@@ -29,16 +37,23 @@ type L1Miner struct {
 	l1Receipts       []*types.Receipt          // collect receipts of ongoing building
 	l1Building       bool
 	l1TxFailed       []*types.Transaction // log of failed transactions which could not be included
+	// sidecars that come with the transactions
+	l1BuildingBlobSidecars []*types.BlobTxSidecar
 }
 
 // NewL1Miner creates a new L1Replica that can also build blocks.
 func NewL1Miner(t Testing, log log.Logger, genesis *core.Genesis) *L1Miner {
 	rep := NewL1Replica(t, log, genesis)
 	return &L1Miner{
 		L1Replica: *rep,
+		blobStore: e2eutils.NewBlobStore(),
 	}
 }
 
+func (s *L1Miner) BlobStore() derive.L1BlobsFetcher {
+	return s.blobStore
+}
+
 // ActL1StartBlock returns an action to build a new L1 block on top of the head block,
 // with timeDelta added to the head block time.
 func (s *L1Miner) ActL1StartBlock(timeDelta uint64) Action {
@@ -77,11 +92,9 @@ func (s *L1Miner) ActL1StartBlock(timeDelta uint64) Action {
 			header.WithdrawalsHash = &types.EmptyWithdrawalsHash
 		}
 		if s.l1Cfg.Config.IsCancun(header.Number, header.Time) {
-			var root common.Hash
-			var zero uint64
-			header.BlobGasUsed = &zero
-			header.ExcessBlobGas = &zero
-			header.ParentBeaconRoot = &root
+			header.BlobGasUsed = new(uint64)
+			header.ExcessBlobGas = new(uint64)
+			header.ParentBeaconRoot = new(common.Hash)
 		}
 
 		s.l1Building = true
@@ -90,6 +103,7 @@ func (s *L1Miner) ActL1StartBlock(timeDelta uint64) Action {
 		s.l1Receipts = make([]*types.Receipt, 0)
 		s.l1Transactions = make([]*types.Transaction, 0)
 		s.pendingIndices = make(map[common.Address]uint64)
+		s.l1BuildingBlobSidecars = make([]*types.BlobTxSidecar, 0)
 
 		s.l1GasPool = new(core.GasPool).AddGas(header.GasLimit)
 	}
@@ -111,6 +125,22 @@ func (s *L1Miner) ActL1IncludeTx(from common.Address) Action {
 	}
 }
 
+// ActL1IncludeTxByHash tries to include a tx by tx-hash.
+func (s *L1Miner) ActL1IncludeTxByHash(txHash common.Hash) Action {
+	return func(t Testing) {
+		if !s.l1Building {
+			t.InvalidAction("no tx inclusion when not building l1 block")
+			return
+		}
+		tx := s.eth.TxPool().Get(txHash)
+		require.NotNil(t, tx, "cannot find tx %s", txHash)
+		s.IncludeTx(t, tx)
+		from, err := s.l1Signer.Sender(tx)
+		require.NoError(t, err)
+		s.pendingIndices[from] = s.pendingIndices[from] + 1 // won't retry the tx
+	}
+}
+
 func (s *L1Miner) IncludeTx(t Testing, tx *types.Transaction) {
 	from, err := s.l1Signer.Sender(tx)
 	require.NoError(t, err)
@@ -124,13 +154,21 @@ func (s *L1Miner) IncludeTx(t Testing, tx *types.Transaction) {
 	}
 	s.l1BuildingState.SetTxContext(tx.Hash(), len(s.l1Transactions))
 	receipt, err := core.ApplyTransaction(s.l1Cfg.Config, s.l1Chain, &s.l1BuildingHeader.Coinbase,
-		s.l1GasPool, s.l1BuildingState, s.l1BuildingHeader, tx, &s.l1BuildingHeader.GasUsed, *s.l1Chain.GetVMConfig())
+		s.l1GasPool, s.l1BuildingState, s.l1BuildingHeader, tx.WithoutBlobTxSidecar(), &s.l1BuildingHeader.GasUsed, *s.l1Chain.GetVMConfig())
 	if err != nil {
 		s.l1TxFailed = append(s.l1TxFailed, tx)
 		t.Fatalf("failed to apply transaction to L1 block (tx %d): %v", len(s.l1Transactions), err)
 	}
 	s.l1Receipts = append(s.l1Receipts, receipt)
-	s.l1Transactions = append(s.l1Transactions, tx)
+	s.l1Transactions = append(s.l1Transactions, tx.WithoutBlobTxSidecar())
+	if tx.Type() == types.BlobTxType {
+		require.True(t, s.l1Cfg.Config.IsCancun(s.l1BuildingHeader.Number, s.l1BuildingHeader.Time), "L1 must be cancun to process blob tx")
+		sidecar := tx.BlobTxSidecar()
+		if sidecar != nil {
+			s.l1BuildingBlobSidecars = append(s.l1BuildingBlobSidecars, sidecar)
+		}
+		*s.l1BuildingHeader.BlobGasUsed += receipt.BlobGasUsed
+	}
 }
 
 func (s *L1Miner) ActL1SetFeeRecipient(coinbase common.Address) {
@@ -154,6 +192,19 @@ func (s *L1Miner) ActL1EndBlock(t Testing) {
 	if s.l1Cfg.Config.IsShanghai(s.l1BuildingHeader.Number, s.l1BuildingHeader.Time) {
 		block = block.WithWithdrawals(make([]*types.Withdrawal, 0))
 	}
+	if s.l1Cfg.Config.IsCancun(s.l1BuildingHeader.Number, s.l1BuildingHeader.Time) {
+		parent := s.l1Chain.GetHeaderByHash(s.l1BuildingHeader.ParentHash)
+		var (
+			parentExcessBlobGas uint64
+			parentBlobGasUsed   uint64
+		)
+		if parent.ExcessBlobGas != nil {
+			parentExcessBlobGas = *parent.ExcessBlobGas
+			parentBlobGasUsed = *parent.BlobGasUsed
+		}
+		excessBlobGas := eip4844.CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed)
+		s.l1BuildingHeader.ExcessBlobGas = &excessBlobGas
+	}
 
 	// Write state changes to db
 	root, err := s.l1BuildingState.Commit(s.l1BuildingHeader.Number.Uint64(), s.l1Cfg.Config.IsEIP158(s.l1BuildingHeader.Number))
@@ -163,7 +214,13 @@ func (s *L1Miner) ActL1EndBlock(t Testing) {
 	if err := s.l1BuildingState.Database().TrieDB().Commit(root, false); err != nil {
 		t.Fatalf("l1 trie write error: %v", err)
 	}
-
+	// now that the blob txs are in a canonical block, flush them to the blob store
+	for _, sidecar := range s.l1BuildingBlobSidecars {
+		for i, h := range sidecar.BlobHashes() {
+			blob := (*eth.Blob)(&sidecar.Blobs[i])
+			s.blobStore.StoreBlob(block.Hash(), h, blob)
+		}
+	}
 	_, err = s.l1Chain.InsertChain(types.Blocks{block})
 	if err != nil {
 		t.Fatalf("failed to insert block into l1 chain")