Merge branch 'zkevm' into zkevm

cmd/rpcdaemon/commands/zkevm_api.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ledgerwatch/erigon/zk/witness"
 	"github.com/ledgerwatch/erigon/zkevm/hex"
 	"github.com/ledgerwatch/erigon/zkevm/jsonrpc/client"
+	zktx "github.com/ledgerwatch/erigon/zk/tx"
 )
 
 var sha3UncleHash = common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347")
@@ -282,12 +283,15 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B
 	// collect blocks in batch
 	batch.Blocks = []interface{}{}
 	batch.Transactions = []interface{}{}
+	var batchBlocks []*eritypes.Block
+	var batchTxs []eritypes.Transaction
 	// handle genesis - not in the hermez tables so requires special treament
 	if batchNumber == 0 {
 		blk, err := api.ethApi.BaseAPI.blockByNumberWithSenders(tx, 0)
 		if err != nil {
 			return nil, err
 		}
+		batchBlocks = append(batchBlocks, blk)
 		batch.Blocks = append(batch.Blocks, blk.Hash())
 		// no txs in genesis
 	}
@@ -296,8 +300,10 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B
 		if err != nil {
 			return nil, err
 		}
+		batchBlocks = append(batchBlocks, blk)
 		batch.Blocks = append(batch.Blocks, blk.Hash())
 		for _, btx := range blk.Transactions() {
+			batchTxs = append(batchTxs, btx)
 			batch.Transactions = append(batch.Transactions, btx.Hash())
 		}
 	}
@@ -362,13 +368,6 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B
 		batch.VerifyBatchTxHash = &ver.L1TxHash
 	}
 
-	// batch l2 data
-	batchL2Data, err := hermezDb.GetL1BatchData(bn)
-	if err != nil {
-		return nil, err
-	}
-	batch.BatchL2Data = batchL2Data
-
 	// exit roots
 	infoTreeUpdate, err := hermezDb.GetL1InfoTreeUpdateByGer(batch.GlobalExitRoot)
 	if err != nil {
@@ -379,6 +378,50 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B
 		batch.RollupExitRoot = infoTreeUpdate.RollupExitRoot
 	}
 
+	if infoTreeUpdate == nil {
+		return nil, errors.New("infoTreeUpdate is nil")
+	}
+
+	// batch l2 data - must build on the fly
+	forkId, err := hermezDb.GetForkId(bn)
+	if err != nil {
+		return nil, err
+	}
+
+	// last batch last block for deltaTimestamp calc
+	lastBlockNoInPreviousBatch := batchBlocks[0].NumberU64() - 1
+	lastBlockInPreviousBatch, err := rawdb.ReadBlockByNumber(tx, lastBlockNoInPreviousBatch)
+	if err != nil {
+		return nil, err
+	}
+
+	var batchL2Data []byte
+	for i := 0; i < len(batchBlocks); i++ {
+		var dTs uint32
+		if i == 0 {
+			dTs = uint32(batchBlocks[i].Time() - lastBlockInPreviousBatch.Time())
+		} else {
+			dTs = uint32(batchBlocks[i].Time() - batchBlocks[i-1].Time())
+		}
+		iti, err := hermezDb.GetBlockL1InfoTreeIndex(batchBlocks[i].NumberU64())
+
+		egTx := make(map[common.Hash]uint8)
+		for _, txn := range batchBlocks[i].Transactions() {
+			eg, err := hermezDb.GetEffectiveGasPricePercentage(txn.Hash())
+			if err != nil {
+				return nil, err
+			}
+			egTx[txn.Hash()] = eg
+		}
+
+		bl2d, err := zktx.GenerateBlockBatchL2Data(uint16(forkId), dTs, uint32(iti), batchBlocks[i].Transactions(), egTx)
+		if err != nil {
+			return nil, err
+		}
+		batchL2Data = append(batchL2Data, bl2d...)
+	}
+	batch.BatchL2Data = batchL2Data
+
 	// currently gives 'error execution reverted' when calling the L1
 	//oaih, err := api.l1Syncer.GetOldAccInputHash(ctx, &api.config.AddressRollup, ApiRollupId, bn+1)
 	//if err != nil {

core/rawdb/accessors_chain_zkevm.go

@@ -131,6 +131,28 @@ func WriteHeader_zkEvm(db kv.Putter, header *types.Header) error {
 	return nil
 }
 
+// WriteHeader stores a block header into the database and also stores the hash-
+// to-number mapping.
+func WriteHeaderWithhash(db kv.Putter, hash libcommon.Hash, header *types.Header) error {
+	var (
+		number  = header.Number.Uint64()
+		encoded = hexutility.EncodeTs(number)
+	)
+	if err := db.Put(kv.HeaderNumber, hash[:], encoded); err != nil {
+		return fmt.Errorf("failed to store hash to number mapping: %W", err)
+	}
+	// Write the encoded header
+	data, err := rlp.EncodeToBytes(header)
+	if err != nil {
+		return fmt.Errorf("failed to RLP encode header: %W", err)
+	}
+	if err := db.Put(kv.Headers, dbutils.HeaderKey(number, hash), data); err != nil {
+		return fmt.Errorf("failed to store header: %W", err)
+	}
+
+	return nil
+}
+
 // ReadReceipts retrieves all the transaction receipts belonging to a block, including
 // its corresponding metadata fields. If it is unable to populate these metadata
 // fields then nil is returned.

eth/stagedsync/stage_execute_zkevm.go

@@ -121,7 +121,7 @@ Loop:
 		}
 
 		//fetch values pre execute
-		preExecuteHeaderHash, block, senders, err := getPreexecuteValues(cfg, ctx, tx, blockNum, prevBlockHash)
+		datastreamBlockHash, block, senders, err := getPreexecuteValues(cfg, ctx, tx, blockNum, prevBlockHash)
 		if err != nil {
 			stoppedErr = err
 			break
@@ -135,15 +135,15 @@ Loop:
 		execRs, err := executeBlockZk(block, &prevBlockRoot, tx, batch, cfg, *cfg.vmConfig, writeChangeSets, writeReceipts, writeCallTraces, initialCycle, stateStream, hermezDb)
 		if err != nil {
 			if !errors.Is(err, context.Canceled) {
-				log.Warn(fmt.Sprintf("[%s] Execution failed", s.LogPrefix()), "block", blockNum, "hash", block.Hash().String(), "err", err)
+				log.Warn(fmt.Sprintf("[%s] Execution failed", s.LogPrefix()), "block", blockNum, "hash", datastreamBlockHash.Hex(), "err", err)
 				if cfg.hd != nil {
-					cfg.hd.ReportBadHeaderPoS(preExecuteHeaderHash, block.ParentHash())
+					cfg.hd.ReportBadHeaderPoS(datastreamBlockHash, block.ParentHash())
 				}
 				if cfg.badBlockHalt {
 					return err
 				}
 			}
-			u.UnwindTo(blockNum-1, block.Hash())
+			u.UnwindTo(blockNum-1, datastreamBlockHash)
 			break Loop
 		}
 
@@ -196,7 +196,7 @@ Loop:
 		}
 
 		//commit values post execute
-		if err := postExecuteCommitValues(cfg, tx, eridb, batch, preExecuteHeaderHash, block, senders); err != nil {
+		if err := postExecuteCommitValues(s.LogPrefix(), cfg, tx, eridb, batch, datastreamBlockHash, block, senders); err != nil {
 			return err
 		}
 	}
@@ -313,24 +313,38 @@ func getPreexecuteValues(cfg ExecuteBlockCfg, ctx context.Context, tx kv.RwTx, b
 }
 
 func postExecuteCommitValues(
+	logPrefix string,
 	cfg ExecuteBlockCfg,
 	tx kv.RwTx,
 	eridb *erigon_db.ErigonDb,
 	batch ethdb.DbWithPendingMutations,
-	preExecuteHeaderHash common.Hash,
+	datastreamBlockHash common.Hash,
 	block *types.Block,
 	senders []common.Address,
 ) error {
 	header := block.Header()
-	headerHash := header.Hash()
+	blockHash := header.Hash()
 	blockNum := block.NumberU64()
 
-	if err := rawdbZk.DeleteSenders(tx, preExecuteHeaderHash, blockNum); err != nil {
-		return fmt.Errorf("failed to delete senders: %v", err)
-	}
+	// if datastream hash was wrong, remove old data
+	if blockHash != datastreamBlockHash {
+		if cfg.chainConfig.IsForkId9Elderberry2(blockNum) {
+			log.Warn(fmt.Sprintf("[%s] Blockhash mismatch", logPrefix), "blockNumber", blockNum, "datastreamBlockHash", datastreamBlockHash, "calculatedBlockHash", blockHash)
+		}
+		if err := rawdbZk.DeleteSenders(tx, datastreamBlockHash, blockNum); err != nil {
+			return fmt.Errorf("failed to delete senders: %v", err)
+		}
+
+		if err := rawdbZk.DeleteHeader(tx, datastreamBlockHash, blockNum); err != nil {
+			return fmt.Errorf("failed to delete header: %v", err)
+		}
 
-	if err := rawdbZk.DeleteHeader(tx, preExecuteHeaderHash, blockNum); err != nil {
-		return fmt.Errorf("failed to delete header: %v", err)
+		// [zkevm] senders were saved in stage_senders for headerHashes based on incomplete headers
+		// in stage execute we complete the headers and senders should be moved to the correct headerHash
+		// also we should delete other data based on the old hash, since it is unaccessable now
+		if err := rawdb.WriteSenders(tx, blockHash, blockNum, senders); err != nil {
+			return fmt.Errorf("failed to write senders: %v", err)
+		}
 	}
 
 	// TODO: how can we store this data right first time?  Or mop up old data as we're currently duping storage
@@ -356,24 +370,16 @@ func postExecuteCommitValues(
 	if err := rawdb.WriteHeader_zkEvm(tx, header); err != nil {
 		return fmt.Errorf("failed to write header: %v", err)
 	}
-	if err := rawdb.WriteHeadHeaderHash(tx, headerHash); err != nil {
+	if err := rawdb.WriteHeadHeaderHash(tx, blockHash); err != nil {
 		return err
 	}
-	if err := rawdb.WriteCanonicalHash(tx, headerHash, blockNum); err != nil {
+	if err := rawdb.WriteCanonicalHash(tx, blockHash, blockNum); err != nil {
 		return fmt.Errorf("failed to write header: %v", err)
 	}
-
-	if err := eridb.WriteBody(block.Number(), headerHash, block.Transactions()); err != nil {
+	if err := eridb.WriteBody(block.Number(), blockHash, block.Transactions()); err != nil {
 		return fmt.Errorf("failed to write body: %v", err)
 	}
 
-	// [zkevm] senders were saved in stage_senders for headerHashes based on incomplete headers
-	// in stage execute we complete the headers and senders should be moved to the correct headerHash
-	// also we should delete other ata based on the old hash, since it is unaccessable now
-	if err := rawdb.WriteSenders(tx, headerHash, blockNum, senders); err != nil {
-		return fmt.Errorf("failed to write senders: %v", err)
-	}
-
 	// write the new block lookup entries
 	if err := rawdb.WriteTxLookupEntries_zkEvm(tx, block); err != nil {
 		return fmt.Errorf("failed to write tx lookup entries: %v", err)

zk/datastream/types/l2block_proto.go

@@ -1,9 +1,9 @@
 package types
 
 import (
+	libcommon "github.com/gateway-fm/cdk-erigon-lib/common"
 	"github.com/ledgerwatch/erigon/zk/datastream/proto/github.com/0xPolygonHermez/zkevm-node/state/datastream"
 	"google.golang.org/protobuf/proto"
-	libcommon "github.com/gateway-fm/cdk-erigon-lib/common"
 )
 
 type L2BlockProto struct {
@@ -56,7 +56,7 @@ func UnmarshalL2Block(data []byte) (*FullL2Block, error) {
 		GlobalExitRoot:  libcommon.BytesToHash(block.GlobalExitRoot),
 		Coinbase:        libcommon.BytesToAddress(block.Coinbase),
 		L1BlockHash:     libcommon.BytesToHash(block.L1Blockhash),
-		L2Blockhash:     libcommon.Hash{},
+		L2Blockhash:     libcommon.BytesToHash(block.Hash),
 		StateRoot:       libcommon.BytesToHash(block.StateRoot),
 		BlockGasLimit:   block.BlockGasLimit,
 		BlockInfoRoot:   libcommon.BytesToHash(block.BlockInfoRoot),

zk/erigon_db/db.go

@@ -25,6 +25,7 @@ func NewErigonDb(tx kv.RwTx) *ErigonDb {
 
 func (db ErigonDb) WriteHeader(
 	blockNo *big.Int,
+	blockHash common.Hash,
 	stateRoot, txHash, parentHash common.Hash,
 	coinbase common.Address,
 	ts, gasLimit uint64,
@@ -42,9 +43,11 @@ func (db ErigonDb) WriteHeader(
 		Extra:      make([]byte, 0),
 	}
 
-	rawdb.WriteHeader(db.tx, h)
-	err := rawdb.WriteCanonicalHash(db.tx, h.Hash(), blockNo.Uint64())
-	if err != nil {
+	if err := rawdb.WriteHeaderWithhash(db.tx, blockHash, h); err != nil {
+		return nil, fmt.Errorf("failed to write header: %w", err)
+
+	}
+	if err := rawdb.WriteCanonicalHash(db.tx, blockHash, blockNo.Uint64()); err != nil {
 		return nil, fmt.Errorf("failed to write canonical hash: %w", err)
 	}
 	return h, nil

zk/stages/stage_batches.go

@@ -34,7 +34,7 @@ const (
 )
 
 type ErigonDb interface {
-	WriteHeader(batchNo *big.Int, stateRoot, txHash, parentHash common.Hash, coinbase common.Address, ts, gasLimit uint64) (*ethTypes.Header, error)
+	WriteHeader(batchNo *big.Int, blockHash common.Hash, stateRoot, txHash, parentHash common.Hash, coinbase common.Address, ts, gasLimit uint64) (*ethTypes.Header, error)
 	WriteBody(batchNo *big.Int, headerHash common.Hash, txs []ethTypes.Transaction) error
 }
 
@@ -785,7 +785,7 @@ func writeL2Block(eriDb ErigonDb, hermezDb HermezDb, l2Block *types.FullL2Block,
 
 	gasLimit := utils.GetBlockGasLimitForFork(l2Block.ForkId)
 
-	h, err := eriDb.WriteHeader(bn, l2Block.StateRoot, txHash, l2Block.ParentHash, l2Block.Coinbase, uint64(l2Block.Timestamp), gasLimit)
+	_, err := eriDb.WriteHeader(bn, l2Block.L2Blockhash, l2Block.StateRoot, txHash, l2Block.ParentHash, l2Block.Coinbase, uint64(l2Block.Timestamp), gasLimit)
 	if err != nil {
 		return fmt.Errorf("write header error: %v", err)
 	}
@@ -867,7 +867,7 @@ func writeL2Block(eriDb ErigonDb, hermezDb HermezDb, l2Block *types.FullL2Block,
 		}
 	}
 
-	if err := eriDb.WriteBody(bn, h.Hash(), txs); err != nil {
+	if err := eriDb.WriteBody(bn, l2Block.L2Blockhash, txs); err != nil {
 		return fmt.Errorf("write body error: %v", err)
 	}
 

zk/tx/tx.go

@@ -382,7 +382,7 @@ func GetDecodedV(tx types.Transaction, v *uint256.Int) *uint256.Int {
 	return result
 }
 
-func GenerateBlockBatchL2Data(forkId uint16, deltaTimestamp uint32, l1InfoTreeIndex uint32, transactions []types.Transaction) ([]byte, error) {
+func GenerateBlockBatchL2Data(forkId uint16, deltaTimestamp uint32, l1InfoTreeIndex uint32, transactions []types.Transaction, egTx map[common.Hash]uint8) ([]byte, error) {
 	var result []byte
 
 	// add in the changeL2Block transaction
@@ -391,7 +391,7 @@ func GenerateBlockBatchL2Data(forkId uint16, deltaTimestamp uint32, l1InfoTreeIn
 	result = binary.BigEndian.AppendUint32(result, l1InfoTreeIndex)
 
 	for _, transaction := range transactions {
-		encoded, err := TransactionToL2Data(transaction, forkId, MaxEffectivePercentage)
+		encoded, err := TransactionToL2Data(transaction, forkId, egTx[transaction.Hash()])
 		if err != nil {
 			return nil, err
 		}

zk/tx/tx_test.go

@@ -423,7 +423,7 @@ func Test_BlockBatchL2DataEncode(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	batchL2Data, err := GenerateBlockBatchL2Data(7, 1, 2, []types.Transaction{tx})
+	batchL2Data, err := GenerateBlockBatchL2Data(7, 1, 2, []types.Transaction{tx}, map[common.Hash]uint8{tx.Hash(): 255})
 	if err != nil {
 		t.Fatal(err)
 	}