Caplin: Fixed EL<->CL communication for Deneb

cl/persistence/block_saver_test.go

@@ -37,7 +37,7 @@ func (m *mockEngine) FrozenBlocks() uint64 {
 	panic("unimplemented")
 }
 
-func (m *mockEngine) NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash) (bool, error) {
+func (m *mockEngine) NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash, expectedBlobsHashes *[]libcommon.Hash) (bool, error) {
 	panic("unimplemented")
 }
 

cl/phase1/execution_client/execution_client_direct.go

@@ -23,7 +23,7 @@ func NewExecutionClientDirect(ctx context.Context, chainRW eth1_chain_reader.Cha
 	}, nil
 }
 
-func (cc *ExecutionClientDirect) NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash) (invalid bool, err error) {
+func (cc *ExecutionClientDirect) NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash, versionedHashes *[]libcommon.Hash) (invalid bool, err error) {
 	if payload == nil {
 		return
 	}
@@ -39,7 +39,7 @@ func (cc *ExecutionClientDirect) NewPayload(payload *cltypes.Eth1Block, beaconPa
 		return true, err
 	}
 
-	if err := cc.chainRW.InsertBlockAndWait(types.NewBlockFromStorage(payload.BlockHash, header, txs, nil, body.Withdrawals)); err != nil {
+	if err := cc.chainRW.InsertBlockAndWait(types.NewBlockFromStorage(payload.BlockHash, header, txs, nil, body.Withdrawals), versionedHashes); err != nil {
 		return false, err
 	}
 
@@ -75,7 +75,7 @@ func (cc *ExecutionClientDirect) InsertBlocks(blks []*types.Block) error {
 }
 
 func (cc *ExecutionClientDirect) InsertBlock(blk *types.Block) error {
-	return cc.chainRW.InsertBlockAndWait(blk)
+	return cc.chainRW.InsertBlockAndWait(blk, nil)
 }
 
 func (cc *ExecutionClientDirect) IsCanonicalHash(hash libcommon.Hash) (bool, error) {

cl/phase1/execution_client/execution_client_rpc.go

@@ -3,12 +3,13 @@ package execution_client
 import (
 	"context"
 	"fmt"
-	"github.com/ledgerwatch/erigon-lib/common/hexutil"
 	"math/big"
 	"net/http"
 	"strings"
 	"time"
 
+	"github.com/ledgerwatch/erigon-lib/common/hexutil"
+
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -53,7 +54,7 @@ func NewExecutionClientRPC(ctx context.Context, jwtSecret []byte, addr string, p
 	}, nil
 }
 
-func (cc *ExecutionClientRpc) NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash) (invalid bool, err error) {
+func (cc *ExecutionClientRpc) NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash, versionedHashes *[]libcommon.Hash) (invalid bool, err error) {
 	if payload == nil {
 		return
 	}
@@ -111,7 +112,11 @@ func (cc *ExecutionClientRpc) NewPayload(payload *cltypes.Eth1Block, beaconParen
 
 	payloadStatus := &engine_types.PayloadStatus{} // As it is done in the rpcdaemon
 	log.Debug("[ExecutionClientRpc] Calling EL", "method", engineMethod)
-	err = cc.client.CallContext(cc.ctx, &payloadStatus, engineMethod, request)
+	args := []interface{}{request}
+	if versionedHashes != nil {
+		args = append(args, *versionedHashes, *beaconParentRoot)
+	}
+	err = cc.client.CallContext(cc.ctx, &payloadStatus, engineMethod, args...)
 	if err != nil {
 		err = fmt.Errorf("execution Client RPC failed to retrieve the NewPayload status response, err: %w", err)
 		return

cl/phase1/execution_client/interface.go

@@ -12,7 +12,7 @@ var errContextExceeded = "rpc error: code = DeadlineExceeded desc = context dead
 // ExecutionEngine is used only for syncing up very close to chain tip and to stay in sync.
 // It pretty much mimics engine API.
 type ExecutionEngine interface {
-	NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash) (bool, error)
+	NewPayload(payload *cltypes.Eth1Block, beaconParentRoot *libcommon.Hash, versionedHashes *[]libcommon.Hash) (bool, error)
 	ForkChoiceUpdate(finalized libcommon.Hash, head libcommon.Hash) error
 	SupportInsertion() bool
 	InsertBlocks([]*types.Block) error

cl/phase1/forkchoice/on_block.go

@@ -7,14 +7,29 @@ import (
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/log/v3"
 
+	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
 	"github.com/ledgerwatch/erigon/cl/cltypes/solid"
 	"github.com/ledgerwatch/erigon/cl/freezer"
 	"github.com/ledgerwatch/erigon/cl/phase1/core/state"
 	"github.com/ledgerwatch/erigon/cl/phase1/forkchoice/fork_graph"
 	"github.com/ledgerwatch/erigon/cl/transition/impl/eth2/statechange"
+	"github.com/ledgerwatch/erigon/cl/utils"
 )
 
+const VERSIONED_HASH_VERSION_KZG byte = byte(1)
+
+func kzgCommitmentToVersionedHash(kzgCommitment *cltypes.KZGCommitment) (libcommon.Hash, error) {
+	versionedHash := [32]byte{}
+	kzgCommitmentHash := utils.Sha256(kzgCommitment[:])
+
+	buf := append([]byte{}, VERSIONED_HASH_VERSION_KZG)
+	buf = append(buf, kzgCommitmentHash[1:]...)
+	copy(versionedHash[:], buf)
+
+	return versionedHash, nil
+}
+
 func (f *ForkChoiceStore) OnBlock(block *cltypes.SignedBeaconBlock, newPayload, fullValidation bool) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -32,10 +47,23 @@ func (f *ForkChoiceStore) OnBlock(block *cltypes.SignedBeaconBlock, newPayload,
 	if block.Block.Slot <= finalizedSlot {
 		return nil
 	}
+	// Now we find the versioned hashes
+	var versionedHashes *[]libcommon.Hash
+	if newPayload && f.engine != nil && block.Version() >= clparams.DenebVersion {
+		versionedHashes = &[]libcommon.Hash{}
+		solid.RangeErr[*cltypes.KZGCommitment](block.Block.Body.BlobKzgCommitments, func(i1 int, k *cltypes.KZGCommitment, i2 int) error {
+			versionedHash, err := kzgCommitmentToVersionedHash(k)
+			if err != nil {
+				return err
+			}
+			*versionedHashes = append(*versionedHashes, versionedHash)
+			return nil
+		})
+	}
 
 	var invalidBlock bool
 	if newPayload && f.engine != nil {
-		if invalidBlock, err = f.engine.NewPayload(block.Block.Body.ExecutionPayload, &block.Block.ParentRoot); err != nil {
+		if invalidBlock, err = f.engine.NewPayload(block.Block.Body.ExecutionPayload, &block.Block.ParentRoot, versionedHashes); err != nil {
 			if invalidBlock {
 				f.forkGraph.MarkHeaderAsInvalid(blockRoot)
 			}

cl/transition/impl/eth2/operations.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"reflect"
 	"time"
 
 	"github.com/ledgerwatch/erigon-lib/metrics"
@@ -24,7 +23,6 @@ import (
 	"github.com/ledgerwatch/erigon/cl/cltypes"
 	"github.com/ledgerwatch/erigon/cl/fork"
 	"github.com/ledgerwatch/erigon/cl/utils"
-	"github.com/ledgerwatch/erigon/core/types"
 )
 
 func (I *impl) ProcessProposerSlashing(s abstract.BeaconState, propSlashing *cltypes.ProposerSlashing) error {
@@ -451,39 +449,6 @@ func (I *impl) ProcessBlsToExecutionChange(s abstract.BeaconState, signedChange
 	return nil
 }
 
-func (I *impl) VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) {
-	if I.FullValidation {
-		return true, nil
-	}
-	allVersionedHashes := []common.Hash{}
-	transactions.ForEach(func(tx []byte, idx, total int) bool {
-		if tx[0] != types.BlobTxType {
-			return true
-		}
-
-		allVersionedHashes = append(allVersionedHashes, txPeekBlobVersionedHashes(tx)...)
-		return true
-	})
-
-	commitmentVersionedHash := []common.Hash{}
-	var err error
-	var versionedHash common.Hash
-	kzgCommitments.Range(func(index int, value *cltypes.KZGCommitment, length int) bool {
-		versionedHash, err = kzgCommitmentToVersionedHash(value)
-		if err != nil {
-			return false
-		}
-
-		commitmentVersionedHash = append(commitmentVersionedHash, versionedHash)
-		return true
-	})
-	if err != nil {
-		return false, err
-	}
-
-	return reflect.DeepEqual(allVersionedHashes, commitmentVersionedHash), nil
-}
-
 func (I *impl) ProcessAttestations(s abstract.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error {
 	attestingIndiciesSet := make([][]uint64, attestations.Len())
 	h := metrics.NewHistTimer("beacon_process_attestations")

cl/transition/impl/eth2/utils.go

@@ -6,58 +6,9 @@ import (
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 
 	"github.com/ledgerwatch/erigon/cl/abstract"
-	"github.com/ledgerwatch/erigon/cl/cltypes"
 	"github.com/ledgerwatch/erigon/cl/utils"
-	"github.com/ledgerwatch/erigon/core/types"
 )
 
-const VERSIONED_HASH_VERSION_KZG byte = byte(1)
-
-func kzgCommitmentToVersionedHash(kzgCommitment *cltypes.KZGCommitment) (libcommon.Hash, error) {
-	versionedHash := [32]byte{}
-	kzgCommitmentHash := utils.Sha256(kzgCommitment[:])
-
-	buf := append([]byte{}, VERSIONED_HASH_VERSION_KZG)
-	buf = append(buf, kzgCommitmentHash[1:]...)
-	copy(versionedHash[:], buf)
-
-	return versionedHash, nil
-}
-
-func txPeekBlobVersionedHashes(txBytes []byte) []libcommon.Hash {
-	if txBytes[0] != types.BlobTxType {
-		return []libcommon.Hash{}
-	}
-
-	messageOffset := 1 + binary.LittleEndian.Uint32(txBytes[1:5])
-
-	/*
-		https://gist.github.com/protolambda/23bd106b66f6d4bb854ce46044aa3ca3
-		chain_id: 32 bytes
-		nonce: 8 bytes
-		priority_fee_per_gas: 32 bytes
-		max_basefee_per_gas: 32 bytes
-		gas: 8 bytes
-		to: 4 bytes - offset to B (relative to A)
-		value: 32 bytes
-		data: 4 bytes - offset to C (relative to A)
-		access_list: 4 bytes - offset to D (relative to A)
-		max_fee_per_blob_gas: 32 bytes
-		blob_versioned_hashes: 4 bytes - offset to E (relative to A)
-	*/
-	// field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188
-	blobVersionedHashes := messageOffset + binary.LittleEndian.Uint32(txBytes[messageOffset+188:messageOffset+192])
-
-	versionedHashes := make([]libcommon.Hash, len(txBytes[blobVersionedHashes:])/32)
-	for pos, i := blobVersionedHashes, 0; int(pos) < len(txBytes) && i < len(versionedHashes); pos += 32 {
-		versionedHash := libcommon.BytesToHash(txBytes[pos : pos+32])
-		versionedHashes[i] = versionedHash
-		i++
-	}
-
-	return versionedHashes
-}
-
 func computeSigningRootEpoch(epoch uint64, domain []byte) (libcommon.Hash, error) {
 	b := make([]byte, 32)
 	binary.LittleEndian.PutUint64(b, epoch)

cl/transition/impl/funcmap/impl.go

@@ -10,22 +10,21 @@ import (
 var _ machine.Interface = (*Impl)(nil)
 
 type Impl struct {
-	FnVerifyBlockSignature                    func(s abstract.BeaconState, block *cltypes.SignedBeaconBlock) error
-	FnVerifyTransition                        func(s abstract.BeaconState, block *cltypes.BeaconBlock) error
-	FnProcessSlots                            func(s abstract.BeaconState, slot uint64) error
-	FnProcessBlockHeader                      func(s abstract.BeaconState, block *cltypes.BeaconBlock) error
-	FnProcessWithdrawals                      func(s abstract.BeaconState, withdrawals *solid.ListSSZ[*cltypes.Withdrawal]) error
-	FnProcessExecutionPayload                 func(s abstract.BeaconState, payload *cltypes.Eth1Block) error
-	FnProcessRandao                           func(s abstract.BeaconState, randao [96]byte, proposerIndex uint64) error
-	FnProcessEth1Data                         func(state abstract.BeaconState, eth1Data *cltypes.Eth1Data) error
-	FnProcessSyncAggregate                    func(s abstract.BeaconState, sync *cltypes.SyncAggregate) error
-	FnVerifyKzgCommitmentsAgainstTransactions func(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error)
-	FnProcessProposerSlashing                 func(s abstract.BeaconState, propSlashing *cltypes.ProposerSlashing) error
-	FnProcessAttesterSlashing                 func(s abstract.BeaconState, attSlashing *cltypes.AttesterSlashing) error
-	FnProcessAttestations                     func(s abstract.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error
-	FnProcessDeposit                          func(s abstract.BeaconState, deposit *cltypes.Deposit) error
-	FnProcessVoluntaryExit                    func(s abstract.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit) error
-	FnProcessBlsToExecutionChange             func(state abstract.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange) error
+	FnVerifyBlockSignature        func(s abstract.BeaconState, block *cltypes.SignedBeaconBlock) error
+	FnVerifyTransition            func(s abstract.BeaconState, block *cltypes.BeaconBlock) error
+	FnProcessSlots                func(s abstract.BeaconState, slot uint64) error
+	FnProcessBlockHeader          func(s abstract.BeaconState, block *cltypes.BeaconBlock) error
+	FnProcessWithdrawals          func(s abstract.BeaconState, withdrawals *solid.ListSSZ[*cltypes.Withdrawal]) error
+	FnProcessExecutionPayload     func(s abstract.BeaconState, payload *cltypes.Eth1Block) error
+	FnProcessRandao               func(s abstract.BeaconState, randao [96]byte, proposerIndex uint64) error
+	FnProcessEth1Data             func(state abstract.BeaconState, eth1Data *cltypes.Eth1Data) error
+	FnProcessSyncAggregate        func(s abstract.BeaconState, sync *cltypes.SyncAggregate) error
+	FnProcessProposerSlashing     func(s abstract.BeaconState, propSlashing *cltypes.ProposerSlashing) error
+	FnProcessAttesterSlashing     func(s abstract.BeaconState, attSlashing *cltypes.AttesterSlashing) error
+	FnProcessAttestations         func(s abstract.BeaconState, attestations *solid.ListSSZ[*solid.Attestation]) error
+	FnProcessDeposit              func(s abstract.BeaconState, deposit *cltypes.Deposit) error
+	FnProcessVoluntaryExit        func(s abstract.BeaconState, signedVoluntaryExit *cltypes.SignedVoluntaryExit) error
+	FnProcessBlsToExecutionChange func(state abstract.BeaconState, signedChange *cltypes.SignedBLSToExecutionChange) error
 }
 
 func (i Impl) VerifyBlockSignature(s abstract.BeaconState, block *cltypes.SignedBeaconBlock) error {
@@ -60,10 +59,6 @@ func (i Impl) ProcessSyncAggregate(s abstract.BeaconState, sync *cltypes.SyncAgg
 	return i.FnProcessSyncAggregate(s, sync)
 }
 
-func (i Impl) VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error) {
-	return i.FnVerifyKzgCommitmentsAgainstTransactions(transactions, kzgCommitments)
-}
-
 func (i Impl) ProcessProposerSlashing(s abstract.BeaconState, propSlashing *cltypes.ProposerSlashing) error {
 	return i.FnProcessProposerSlashing(s, propSlashing)
 }

cl/transition/machine/block.go

@@ -56,15 +56,7 @@ func ProcessBlock(impl BlockProcessor, s abstract.BeaconState, signedBlock *clty
 			return fmt.Errorf("processBlock: failed to process sync aggregate: %v", err)
 		}
 	}
-	if version >= clparams.DenebVersion {
-		verified, err := impl.VerifyKzgCommitmentsAgainstTransactions(block.Body.ExecutionPayload.Transactions, block.Body.BlobKzgCommitments)
-		if err != nil {
-			return fmt.Errorf("processBlock: failed to process blob kzg commitments: %w", err)
-		}
-		if !verified {
-			return fmt.Errorf("processBlock: failed to process blob kzg commitments: commitments are not equal")
-		}
-	}
+
 	h.PutSince()
 	return nil
 }

cl/transition/machine/machine.go

@@ -34,7 +34,6 @@ type BlockHeaderProcessor interface {
 	ProcessRandao(s abstract.BeaconState, randao [96]byte, proposerIndex uint64) error
 	ProcessEth1Data(state abstract.BeaconState, eth1Data *cltypes.Eth1Data) error
 	ProcessSyncAggregate(s abstract.BeaconState, sync *cltypes.SyncAggregate) error
-	VerifyKzgCommitmentsAgainstTransactions(transactions *solid.TransactionsSSZ, kzgCommitments *solid.ListSSZ[*cltypes.KZGCommitment]) (bool, error)
 }
 
 type BlockOperationProcessor interface {

erigon-lib/go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/erigontech/mdbx-go v0.27.21
 	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240115083615-b5feeb63e191
-	github.com/ledgerwatch/interfaces v0.0.0-20240113123344-7100723dbd63
+	github.com/ledgerwatch/interfaces v0.0.0-20240122095607-549d80de3670
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0
 )

erigon-lib/go.sum

@@ -295,8 +295,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240115083615-b5feeb63e191 h1:X/mHEyh0xEuhixj6hKCNQl04NuNDToYWJ08vr66e6L0=
 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240115083615-b5feeb63e191/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
-github.com/ledgerwatch/interfaces v0.0.0-20240113123344-7100723dbd63 h1:g7786bCGyulcAD4lmE8rusla/k2gna+5Z3z54wbrsq8=
-github.com/ledgerwatch/interfaces v0.0.0-20240113123344-7100723dbd63/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
+github.com/ledgerwatch/interfaces v0.0.0-20240122095607-549d80de3670 h1:/ye+TmuN4DTjUlJGeu9+dCC9sYafgbG0saGg9NXnL3E=
+github.com/ledgerwatch/interfaces v0.0.0-20240122095607-549d80de3670/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
 github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
 github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=