simulators/ethereum/engine: Cross-client Payload Validation

simulators/ethereum/engine/client/engine.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rpc"
 )
 
 type Eth interface {
@@ -21,6 +22,7 @@ type Eth interface {
 	StorageAt(ctx context.Context, account common.Address, key common.Hash, blockNumber *big.Int) ([]byte, error)
 	TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
 	NonceAt(ctx context.Context, account common.Address, blockNumber *big.Int) (uint64, error)
+	CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error)
 }
 
 type Engine interface {
@@ -38,8 +40,10 @@ type Engine interface {
 type EngineClient interface {
 	// General Methods
 	ID() string
+	ClientType() string
 	Close() error
 	EnodeURL() (string, error)
+	RPC() *rpc.Client
 
 	// Local Test Account Management
 	GetNextAccountNonce(testCtx context.Context, account common.Address) (uint64, error)

simulators/ethereum/engine/client/hive_rpc/hive_rpc.go

@@ -147,6 +147,7 @@ type HiveRPCEngineClient struct {
 	h              *hivesim.Client
 	c              *rpc.Client
 	cEth           *rpc.Client
+	clientType     string
 	ttd            *big.Int
 	JWTSecretBytes []byte
 
@@ -191,6 +192,14 @@ func (ec *HiveRPCEngineClient) ID() string {
 	return ec.h.Container
 }
 
+func (ec *HiveRPCEngineClient) ClientType() string {
+	return ec.clientType
+}
+
+func (ec *HiveRPCEngineClient) RPC() *rpc.Client {
+	return ec.c
+}
+
 func (ec *HiveRPCEngineClient) EnodeURL() (string, error) {
 	return ec.h.EnodeURL()
 }

simulators/ethereum/engine/client/node/node.go

@@ -806,6 +806,14 @@ func (n *GethNode) NonceAt(ctx context.Context, account common.Address, blockNum
 	return stateDB.GetNonce(account), nil
 }
 
+func (n *GethNode) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) {
+	stateDB, err := n.getStateDB(ctx, blockNumber)
+	if err != nil {
+		return nil, err
+	}
+	return stateDB.GetCode(account), nil
+}
+
 func (n *GethNode) GetBlockTotalDifficulty(ctx context.Context, hash common.Hash) (*big.Int, error) {
 	block := n.eth.BlockChain().GetBlockByHash(hash)
 	if block == nil {
@@ -836,6 +844,14 @@ func (n *GethNode) ID() string {
 	return n.node.Config().Name
 }
 
+func (n *GethNode) ClientType() string {
+	return n.node.Config().Name
+}
+
+func (n *GethNode) RPC() *rpc.Client {
+	return nil
+}
+
 func (n *GethNode) GetNextAccountNonce(testCtx context.Context, account common.Address) (uint64, error) {
 	// First get the current head of the client where we will send the tx
 	head, err := n.eth.APIBackend.BlockByNumber(testCtx, LatestBlockNumber)

simulators/ethereum/engine/clmock/clmock.go

@@ -331,6 +331,9 @@ func (cl *CLMocker) broadcastNextNewPayload() {
 				if resp.ExecutePayloadResponse.LatestValidHash != nil && *resp.ExecutePayloadResponse.LatestValidHash != (common.Hash{}) {
 					cl.Fatalf("CLMocker: NewPayload returned ACCEPTED status with incorrect LatestValidHash==%v", resp.ExecutePayloadResponse.LatestValidHash)
 				}
+			} else if resp.ExecutePayloadResponse.Status == api.INVALID {
+				// At any point during the CLMock workflow there mustn't be any INVALID payload
+				cl.Fatalf("CLMocker: An invalid payload was produced by one of the clients during the payload building process: Payload builder=%s, invalidating client=%s, hash=%s", cl.NextBlockProducer.ID(), resp.Container, cl.LatestPayloadBuilt.BlockHash)
 			} else {
 				cl.Logf("CLMocker: BroadcastNewPayload Response (%v): %v\n", resp.Container, resp.ExecutePayloadResponse)
 			}
@@ -356,7 +359,10 @@ func (cl *CLMocker) broadcastLatestForkchoice() {
 			if resp.ForkchoiceResponse.PayloadID != nil {
 				cl.Fatalf("CLMocker: Expected empty PayloadID: %v\n", resp.Container, resp.ForkchoiceResponse.PayloadID)
 			}
-		} else if resp.ForkchoiceResponse.PayloadStatus.Status != api.VALID {
+		} else if resp.ForkchoiceResponse.PayloadStatus.Status == api.INVALID {
+			// At any point during the CLMock workflow there mustn't be any INVALID payload
+			cl.Fatalf("CLMocker: An invalid payload was produced by one of the clients during the payload building process (ForkchoiceUpdated): Payload builder=%s, invalidating client=%s, hash=%s", cl.NextBlockProducer.ID(), resp.Container, cl.LatestPayloadBuilt.BlockHash)
+		} else {
 			cl.Logf("CLMocker: BroadcastForkchoiceUpdated Response (%v): %v\n", resp.Container, resp.ForkchoiceResponse)
 		}
 	}

simulators/ethereum/engine/globals/globals.go

@@ -13,10 +13,11 @@ import (
 var (
 
 	// Test chain parameters
-	ChainID     = big.NewInt(7)
-	GasPrice    = big.NewInt(30 * params.GWei)
-	GasTipPrice = big.NewInt(1 * params.GWei)
-	NetworkID   = big.NewInt(7)
+	ChainID   = big.NewInt(7)
+	GasPrice  = big.NewInt(30 * params.GWei)
+	GasFeeCap = big.NewInt(30 * params.GWei)
+	GasTipCap = big.NewInt(1 * params.GWei)
+	NetworkID = big.NewInt(7)
 
 	// RPC Timeout for every call
 	RPCTimeout = 10 * time.Second

simulators/ethereum/engine/helper/helper.go

@@ -2,6 +2,7 @@ package helper
 
 import (
 	"context"
+	"strings"
 	"sync"
 	"time"
 
@@ -17,6 +18,8 @@ import (
 	"os"
 
 	api "github.com/ethereum/go-ethereum/core/beacon"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/hive/simulators/ethereum/engine/client"
 	"github.com/ethereum/hive/simulators/ethereum/engine/globals"
 
@@ -350,13 +353,13 @@ func MakeTransaction(nonce uint64, recipient *common.Address, gasLimit uint64, a
 			Data:     payload,
 		}
 	case types.DynamicFeeTxType:
-		gasFeeCap := new(big.Int).Set(globals.GasPrice)
-		gasTipCap := new(big.Int).Set(globals.GasTipPrice)
+		gasFeeCap := new(big.Int).Set(globals.GasFeeCap)
+		gasTipCap := new(big.Int).Set(globals.GasTipCap)
 		newTxData = &types.DynamicFeeTx{
 			Nonce:     nonce,
 			Gas:       gasLimit,
-			GasTipCap: gasTipCap,
 			GasFeeCap: gasFeeCap,
+			GasTipCap: gasTipCap,
 			To:        recipient,
 			Value:     amount,
 			Data:      payload,
@@ -371,6 +374,37 @@ func MakeTransaction(nonce uint64, recipient *common.Address, gasLimit uint64, a
 	return signedTx, nil
 }
 
+func MakeContractTransaction(nonce uint64, gasLimit uint64, amount *big.Int, contractCode []byte, txType TestTransactionType) (common.Address, *types.Transaction, error) {
+	// Create a contract based on the contract code without any specialized initialization
+	var initCode []byte
+	if len(contractCode) > 32 {
+		return common.Address{}, nil, fmt.Errorf("contract too big")
+	} else {
+		// Push the entire contract code onto the stack and init from there
+		initCode = []byte{ // Push contract code onto stack
+			byte(vm.PUSH1) + byte(len(contractCode)-1)}
+		initCode = append(initCode, contractCode...)
+		initCode = append(initCode, []byte{
+			byte(vm.PUSH1), 0x0, // memory start on stack
+			byte(vm.MSTORE),
+			byte(vm.PUSH1), byte(len(contractCode)), // size
+			byte(vm.PUSH1), byte(32 - len(contractCode)), // offset
+			byte(vm.RETURN),
+		}...)
+	}
+
+	contractAddress := crypto.CreateAddress(globals.VaultAccountAddress, nonce)
+	tx, err := MakeTransaction(nonce, nil, gasLimit, amount, initCode, txType)
+	return contractAddress, tx, err
+}
+
+// Determines if the error we got from sending the raw tx is because the client
+// already knew the tx (might happen if we produced a re-org where the tx was
+// unwind back into the txpool)
+func SentTxAlreadyKnown(err error) bool {
+	return strings.Contains(err.Error(), "already known")
+}
+
 func SendNextTransaction(testCtx context.Context, node client.EngineClient, recipient common.Address, amount *big.Int, payload []byte, txType TestTransactionType) (*types.Transaction, error) {
 	nonce, err := node.GetNextAccountNonce(testCtx, globals.VaultAccountAddress)
 	if err != nil {
@@ -383,6 +417,8 @@ func SendNextTransaction(testCtx context.Context, node client.EngineClient, reci
 		err := node.SendTransaction(ctx, tx)
 		if err == nil {
 			return tx, nil
+		} else if SentTxAlreadyKnown(err) {
+			return tx, nil
 		}
 		select {
 		case <-time.After(time.Second):

simulators/ethereum/engine/helper/payload.go

@@ -250,7 +250,7 @@ func GenerateInvalidPayload(basePayload *api.ExecutableDataV1, payloadField Inva
 		case InvalidTransactionGasPrice:
 			customTxData.GasPriceOrGasFeeCap = common.Big0
 		case InvalidTransactionGasTipPrice:
-			invalidGasTip := new(big.Int).Set(globals.GasTipPrice)
+			invalidGasTip := new(big.Int).Set(globals.GasTipCap)
 			invalidGasTip.Mul(invalidGasTip, big.NewInt(2))
 			customTxData.GasTipCap = invalidGasTip
 		case InvalidTransactionValue:

simulators/ethereum/engine/main.go

@@ -81,23 +81,25 @@ func addTestsToSuite(suite *hivesim.Suite, tests []test.Spec) {
 		if currentTest.DisableMining {
 			delete(newParams, "HIVE_MINER")
 		}
-		suite.Add(hivesim.ClientTestSpec{
+		suite.Add(hivesim.TestSpec{
 			Name:        currentTest.Name,
 			Description: currentTest.About,
-			Parameters:  newParams,
-			Files:       testFiles,
-			Run: func(t *hivesim.T, c *hivesim.Client) {
-				t.Logf("Start test (%s): %s", c.Type, currentTest.Name)
+			Run: func(t *hivesim.T) {
+				testClientTypes, err := t.Sim.ClientTypes()
+				if err != nil {
+					t.Fatalf("No client types")
+				}
+				t.Logf("Start test (%s): %s", testClientTypes[0].Name, currentTest.Name)
 				defer func() {
-					t.Logf("End test (%s): %s", c.Type, currentTest.Name)
+					t.Logf("End test (%s): %s", testClientTypes[0].Name, currentTest.Name)
 				}()
 				timeout := globals.DefaultTestCaseTimeout
 				// If a test.Spec specifies a timeout, use that instead
 				if currentTest.TimeoutSeconds != 0 {
 					timeout = time.Second * time.Duration(currentTest.TimeoutSeconds)
 				}
 				// Run the test case
-				test.Run(currentTest.Name, big.NewInt(ttd), currentTest.SlotsToSafe, currentTest.SlotsToFinalized, timeout, t, c, currentTest.Run, newParams, testFiles, currentTest.TestTransactionType, currentTest.SafeSlotsToImportOptimistically)
+				test.Run(currentTest.Name, big.NewInt(ttd), currentTest.SlotsToSafe, currentTest.SlotsToFinalized, timeout, t, currentTest.Run, newParams, testFiles, currentTest.TestTransactionType, currentTest.SafeSlotsToImportOptimistically)
 			},
 		})
 	}

simulators/ethereum/engine/suites/auth/tests.go

@@ -2,11 +2,13 @@ package suite_auth
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	api "github.com/ethereum/go-ethereum/core/beacon"
+	"github.com/ethereum/hive/simulators/ethereum/engine/client/hive_rpc"
 	"github.com/ethereum/hive/simulators/ethereum/engine/globals"
 	"github.com/ethereum/hive/simulators/ethereum/engine/test"
 )
@@ -100,12 +102,16 @@ func GenerateAuthTestSpec(authTestSpec AuthTestSpec) test.Spec {
 			if authTestSpec.TimeDriftSeconds != 0 {
 				testTime = testTime.Add(time.Second * time.Duration(authTestSpec.TimeDriftSeconds))
 			}
-			if err := t.HiveEngine.PrepareAuthCallToken(testSecret, testTime); err != nil {
+			hiveEngine, ok := t.Engine.(*hive_rpc.HiveRPCEngineClient)
+			if !ok {
+				panic(fmt.Errorf("Invalid cast to HiveRPCEngineClient"))
+			}
+			if err := hiveEngine.PrepareAuthCallToken(testSecret, testTime); err != nil {
 				t.Fatalf("FAIL (%s): Unable to prepare the auth token: %v", t.TestName, err)
 			}
 			ctx, cancel := context.WithTimeout(t.TestContext, globals.RPCTimeout)
 			defer cancel()
-			_, err := t.HiveEngine.ExchangeTransitionConfigurationV1(ctx, &tConf)
+			_, err := hiveEngine.ExchangeTransitionConfigurationV1(ctx, &tConf)
 			if (authTestSpec.AuthOk && err == nil) || (!authTestSpec.AuthOk && err != nil) {
 				// Test passed
 				return

simulators/ethereum/engine/suites/engine/tests.go

@@ -1,6 +1,7 @@
 package suite_engine
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"math/big"
@@ -9,7 +10,6 @@ import (
 
 	api "github.com/ethereum/go-ethereum/core/beacon"
 	"github.com/ethereum/hive/simulators/ethereum/engine/client"
-	"github.com/ethereum/hive/simulators/ethereum/engine/client/hive_rpc"
 	"github.com/ethereum/hive/simulators/ethereum/engine/client/node"
 	"github.com/ethereum/hive/simulators/ethereum/engine/clmock"
 	"github.com/ethereum/hive/simulators/ethereum/engine/globals"
@@ -18,6 +18,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
 )
 
 // Execution specification reference:
@@ -1192,6 +1193,11 @@ var Tests = []test.Spec{
 		TTD:                 10,
 		TestTransactionType: helper.DynamicFeeTxOnly,
 	},
+	{
+		Name:           "High PrevRandao Transaction Count (Payload Cross-Check)",
+		Run:            highTxPrevRandaoOpcodeTx,
+		TimeoutSeconds: 400,
+	},
 }
 
 // Invalid Terminal Block in ForkchoiceUpdated: Client must reject ForkchoiceUpdated directives if the referenced HeadBlockHash does not meet the TTD requirement.
@@ -1697,10 +1703,7 @@ func invalidPayloadTestCaseGen(payloadField helper.InvalidPayloadBlockField, syn
 
 		if syncing {
 			// To allow sending the primary engine client into SYNCING state, we need a secondary client to guide the payload creation
-			secondaryClient, err := hive_rpc.HiveRPCEngineStarter{}.StartClient(t.T, t.TestContext, t.ClientParams, t.ClientFiles)
-			if err != nil {
-				t.Fatalf("FAIL (%s): Unable to spawn a secondary client: %v", t.TestName, err)
-			}
+			secondaryClient := t.StartNextClient(t.ClientParams, t.ClientFiles)
 			t.CLMock.AddEngineClient(secondaryClient)
 		}
 
@@ -3168,10 +3171,7 @@ func inOrderPayloads(t *test.Env) {
 	r.ExpectBalanceEqual(expectedBalance)
 
 	// Start a second client to send newPayload consecutively without fcU
-	secondaryClient, err := hive_rpc.HiveRPCEngineStarter{}.StartClient(t.T, t.TestContext, t.ClientParams, t.ClientFiles)
-	if err != nil {
-		t.Fatalf("FAIL (%s): Unable to start secondary client: %v", t.TestName, err)
-	}
+	secondaryClient := t.StartNextClient(t.ClientParams, t.ClientFiles)
 	secondaryTestEngineClient := test.NewTestEngineClient(t, secondaryClient)
 
 	// Send the forkchoiceUpdated with the LatestExecutedPayload hash, we should get SYNCING back
@@ -3219,12 +3219,7 @@ func validPayloadFcUSyncingClient(t *test.Env) {
 	)
 	{
 		// To allow sending the primary engine client into SYNCING state, we need a secondary client to guide the payload creation
-		var err error
-		secondaryClient, err = hive_rpc.HiveRPCEngineStarter{}.StartClient(t.T, t.TestContext, t.ClientParams, t.ClientFiles)
-
-		if err != nil {
-			t.Fatalf("FAIL (%s): Unable to spawn a secondary client: %v", t.TestName, err)
-		}
+		secondaryClient = t.StartNextClient(t.ClientParams, t.ClientFiles)
 		t.CLMock.AddEngineClient(secondaryClient)
 	}
 
@@ -3304,11 +3299,7 @@ func missingFcu(t *test.Env) {
 
 	var secondaryEngineTest *test.TestEngineClient
 	{
-		secondaryEngine, err := hive_rpc.HiveRPCEngineStarter{}.StartClient(t.T, t.TestContext, t.ClientParams, t.ClientFiles)
-
-		if err != nil {
-			t.Fatalf("FAIL (%s): Unable to spawn a secondary client: %v", t.TestName, err)
-		}
+		secondaryEngine := t.StartNextClient(t.ClientParams, t.ClientFiles)
 		secondaryEngineTest = test.NewTestEngineClient(t, secondaryEngine)
 		t.CLMock.AddEngineClient(secondaryEngine)
 	}
@@ -3345,11 +3336,7 @@ func missingFcu(t *test.Env) {
 // P <- INV_P, newPayload(INV_P), fcU(head: P, payloadAttributes: attrs) + getPayload(…)
 func payloadBuildAfterNewInvalidPayload(t *test.Env) {
 	// Add a second client to build the invalid payload
-	secondaryEngine, err := hive_rpc.HiveRPCEngineStarter{}.StartClient(t.T, t.TestContext, t.ClientParams, t.ClientFiles)
-
-	if err != nil {
-		t.Fatalf("FAIL (%s): Unable to spawn a secondary client: %v", t.TestName, err)
-	}
+	secondaryEngine := t.StartNextClient(t.ClientParams, t.ClientFiles)
 	secondaryEngineTest := test.NewTestEngineClient(t, secondaryEngine)
 	t.CLMock.AddEngineClient(secondaryEngine)
 
@@ -3368,7 +3355,10 @@ func payloadBuildAfterNewInvalidPayload(t *test.Env) {
 			if t.CLMock.NextBlockProducer == invalidPayloadProducer.Engine {
 				invalidPayloadProducer = secondaryEngineTest
 			}
-			var inv_p *api.ExecutableDataV1
+			var (
+				inv_p *api.ExecutableDataV1
+				err   error
+			)
 
 			{
 				// Get a payload from the invalid payload producer and invalidate it
@@ -3586,7 +3576,7 @@ func prevRandaoOpcodeTx(t *test.Env) {
 			expectedPrevRandao := t.CLMock.PrevRandaoHistory[t.CLMock.LatestHeader.Number.Uint64()+1]
 			ctx, cancel := context.WithTimeout(t.TestContext, globals.RPCTimeout)
 			defer cancel()
-			if err := helper.DebugPrevRandaoTransaction(ctx, t.Client.RPC(), t.Client.Type, txs[currentTxIndex-1],
+			if err := helper.DebugPrevRandaoTransaction(ctx, t.Engine.RPC(), t.Engine.ClientType(), txs[currentTxIndex-1],
 				&expectedPrevRandao); err != nil {
 				t.Fatalf("FAIL (%s): Error during transaction tracing: %v", t.TestName, err)
 			}
@@ -3604,3 +3594,101 @@ func prevRandaoOpcodeTx(t *test.Env) {
 	}
 
 }
+
+// Multi-client High Transaction Count PrevRandao
+func highTxPrevRandaoOpcodeTx(t *test.Env) {
+	// Wait for TTD to be reached
+	t.CLMock.WaitForTTD()
+
+	// Start a secondary client to cross check payloads
+	secondaryClient := t.StartNextClient(t.ClientParams, t.ClientFiles, t.Engine)
+	t.CLMock.AddEngineClient(secondaryClient)
+
+	// Create a smart contract that puts something in the log bloom
+	contractCode := []byte{
+		// Launch an event with the prevRandao as topic
+		byte(vm.DIFFICULTY), // topic0
+		byte(vm.PUSH1), 0x0, // length
+		byte(vm.PUSH1), 0x0, // offset
+		byte(vm.LOG1), // log1
+		// Also store prevrandao to storage
+		byte(vm.DIFFICULTY), // value
+		byte(vm.NUMBER),     // key
+		byte(vm.SSTORE),     // Set slot[block.Number]=prevRandao
+	}
+
+	var (
+		nonce                   = uint64(0)
+		blockCount              = 128
+		MaxTransactionsPerBlock = 100
+		totalTxsIncluded        = 0
+		contractAddress         common.Address
+	)
+
+	t.CLMock.ProduceSingleBlock(clmock.BlockProcessCallbacks{
+		OnPayloadProducerSelected: func() {
+			var (
+				tx  *types.Transaction
+				err error
+			)
+			contractAddress, tx, err = helper.MakeContractTransaction(nonce, 75000, big0, contractCode, t.TestTransactionType)
+			if err != nil {
+				t.Fatalf("FAIL (%s): Error trying to create contract transaction: %v", t.TestName, err)
+			}
+			ctx, cancel := context.WithTimeout(t.TestContext, globals.RPCTimeout)
+			defer cancel()
+			err = t.Engine.SendTransaction(ctx, tx)
+			if err != nil && !helper.SentTxAlreadyKnown(err) {
+				t.Fatalf("FAIL (%s): Error trying to send transaction: %v", t.TestName, err)
+			}
+			nonce++
+		},
+	})
+	ctx, cancel := context.WithTimeout(t.TestContext, globals.RPCTimeout)
+	defer cancel()
+	code, err := t.Engine.CodeAt(ctx, contractAddress, nil)
+	if err != nil {
+		t.Fatalf("FAIL (%s): Unable to get contract code: %v", t.TestName, err)
+	}
+	if bytes.Compare(code, contractCode) != 0 {
+		t.Fatalf("FAIL (%s): Contract not set correctly: %s", t.TestName, common.Bytes2Hex(code))
+	}
+
+	t.CLMock.ProduceBlocks(blockCount, clmock.BlockProcessCallbacks{
+		OnPayloadProducerSelected: func() {
+			// Limit the number of transactions per block to limit the basefee
+			blockTxCount := MaxTransactionsPerBlock
+			if (t.CLMock.LatestExecutedPayload.Number % 2) == 0 {
+				blockTxCount /= 2
+			}
+			for i := 0; i < blockTxCount; i++ {
+				tx, _ := helper.MakeTransaction(nonce, &contractAddress, 75000, big0, nil, t.TestTransactionType)
+				// Send transaction to both clients
+				ctx, cancel := context.WithTimeout(t.TestContext, globals.RPCTimeout)
+				defer cancel()
+				err := t.Engine.SendTransaction(ctx, tx)
+				if err != nil && !helper.SentTxAlreadyKnown(err) {
+					t.Fatalf("FAIL (%s): Error trying to send transaction: %v", t.TestName, err)
+				}
+				ctx, cancel = context.WithTimeout(t.TestContext, globals.RPCTimeout)
+				defer cancel()
+				err = secondaryClient.SendTransaction(ctx, tx)
+				if err != nil && !helper.SentTxAlreadyKnown(err) {
+					t.Fatalf("FAIL (%s): Error trying to send transaction: %v", t.TestName, err)
+				}
+				nonce++
+			}
+		},
+		OnForkchoiceBroadcast: func() {
+			// Verify the payload built contained transactions
+			if len(t.CLMock.LatestPayloadBuilt.Transactions) == 0 {
+				t.Fatalf("FAIL (%s): Payload %d contained no transactions", t.TestName, t.CLMock.LatestPayloadBuilt.Number)
+			}
+			t.Logf("INFO (%s): Transactions in payload %d (Producer %s): %d", t.TestName, t.CLMock.LatestPayloadBuilt.Number, t.CLMock.NextBlockProducer.ID(), len(t.CLMock.LatestPayloadBuilt.Transactions))
+			totalTxsIncluded += len(t.CLMock.LatestPayloadBuilt.Transactions)
+			t.Logf("INFO (%s): Total Transactions Included: %d", t.TestName, totalTxsIncluded)
+			t.Logf("INFO (%s): Total Transactions Sent: %d", t.TestName, nonce+1)
+		},
+	})
+
+}

simulators/ethereum/engine/suites/sync/tests.go

@@ -8,6 +8,7 @@ import (
 
 	api "github.com/ethereum/go-ethereum/core/beacon"
 	"github.com/ethereum/hive/hivesim"
+	"github.com/ethereum/hive/simulators/ethereum/engine/client"
 	"github.com/ethereum/hive/simulators/ethereum/engine/client/hive_rpc"
 	"github.com/ethereum/hive/simulators/ethereum/engine/clmock"
 	"github.com/ethereum/hive/simulators/ethereum/engine/globals"
@@ -39,69 +40,115 @@ func AddSyncTestsToSuite(sim *hivesim.Simulation, suite *hivesim.Suite, tests []
 		panic(err)
 	}
 	for _, currentTest := range tests {
-		for _, clientDef := range clientDefs {
-			clientSyncVariantGenerator, ok := ClientToSyncVariantGenerator[clientDef.Name]
-			if !ok {
-				clientSyncVariantGenerator = DefaultSyncVariantGenerator{}
-			}
-			currentTest := currentTest
-			genesisPath := "./init/genesis.json"
-			// If the test.Spec specified a custom genesis file, use that instead.
-			if currentTest.GenesisFile != "" {
-				genesisPath = "./init/" + currentTest.GenesisFile
-			}
-			testFiles := hivesim.Params{"/genesis.json": genesisPath}
-			// Calculate and set the TTD for this test
-			ttd := helper.CalculateRealTTD(genesisPath, currentTest.TTD)
-			newParams := globals.DefaultClientEnv.Set("HIVE_TERMINAL_TOTAL_DIFFICULTY", fmt.Sprintf("%d", ttd))
-			if currentTest.ChainFile != "" {
-				// We are using a Proof of Work chain file, remove all clique-related settings
-				// TODO: Nethermind still requires HIVE_MINER for the Engine API
-				// delete(newParams, "HIVE_MINER")
-				delete(newParams, "HIVE_CLIQUE_PRIVATEKEY")
-				delete(newParams, "HIVE_CLIQUE_PERIOD")
-				// Add the new file to be loaded as chain.rlp
-			}
-			for _, variant := range clientSyncVariantGenerator.Configure(big.NewInt(ttd), genesisPath, currentTest.ChainFile) {
-				variant := variant
-				clientDef := clientDef
-				suite.Add(hivesim.TestSpec{
-					Name:        fmt.Sprintf("%s (%s, sync/%s)", currentTest.Name, clientDef.Name, variant.Name),
-					Description: currentTest.About,
-					Run: func(t *hivesim.T) {
-
-						mainClientParams := newParams.Copy()
-						for k, v := range variant.MainClientConfig {
-							mainClientParams = mainClientParams.Set(k, v)
-						}
-						mainClientFiles := testFiles.Copy()
-						if currentTest.ChainFile != "" {
-							mainClientFiles = mainClientFiles.Set("/chain.rlp", "./chains/"+currentTest.ChainFile)
-						}
-						c := t.StartClient(clientDef.Name, mainClientParams, hivesim.WithStaticFiles(mainClientFiles))
+		clientSyncVariantGenerator, ok := ClientToSyncVariantGenerator[clientDefs[0].Name]
+		if !ok {
+			clientSyncVariantGenerator = DefaultSyncVariantGenerator{}
+		}
+		currentTest := currentTest
+		genesisPath := "./init/genesis.json"
+		// If the test.Spec specified a custom genesis file, use that instead.
+		if currentTest.GenesisFile != "" {
+			genesisPath = "./init/" + currentTest.GenesisFile
+		}
+		testFiles := hivesim.Params{"/genesis.json": genesisPath}
+		// Calculate and set the TTD for this test
+		ttd := helper.CalculateRealTTD(genesisPath, currentTest.TTD)
+		newParams := globals.DefaultClientEnv.Set("HIVE_TERMINAL_TOTAL_DIFFICULTY", fmt.Sprintf("%d", ttd))
+		if currentTest.ChainFile != "" {
+			// We are using a Proof of Work chain file, remove all clique-related settings
+			// TODO: Nethermind still requires HIVE_MINER for the Engine API
+			// delete(newParams, "HIVE_MINER")
+			delete(newParams, "HIVE_CLIQUE_PRIVATEKEY")
+			delete(newParams, "HIVE_CLIQUE_PERIOD")
+			// Add the new file to be loaded as chain.rlp
+		}
+		for _, variant := range clientSyncVariantGenerator.Configure(big.NewInt(ttd), genesisPath, currentTest.ChainFile) {
+			variant := variant
+			clientDef := clientDefs[0]
+			suite.Add(hivesim.TestSpec{
+				Name:        fmt.Sprintf("%s (%s, sync/%s)", currentTest.Name, clientDef.Name, variant.Name),
+				Description: currentTest.About,
+				Run: func(t *hivesim.T) {
 
-						t.Logf("Start test (%s, %s, sync/%s)", c.Type, currentTest.Name, variant.Name)
-						defer func() {
-							t.Logf("End test (%s, %s, sync/%s)", c.Type, currentTest.Name, variant.Name)
-						}()
+					mainClientParams := newParams.Copy()
+					for k, v := range variant.MainClientConfig {
+						mainClientParams = mainClientParams.Set(k, v)
+					}
+					mainClientFiles := testFiles.Copy()
+					if currentTest.ChainFile != "" {
+						mainClientFiles = mainClientFiles.Set("/chain.rlp", "./chains/"+currentTest.ChainFile)
+					}
+					// c := t.StartClient(clientDef.Name, mainClientParams, hivesim.WithStaticFiles(mainClientFiles))
 
-						timeout := globals.DefaultTestCaseTimeout
-						// If a test.Spec specifies a timeout, use that instead
-						if currentTest.TimeoutSeconds != 0 {
-							timeout = time.Second * time.Duration(currentTest.TimeoutSeconds)
-						}
+					t.Logf("Start test (%s, %s, sync/%s)", clientDef.Name, currentTest.Name, variant.Name)
+					defer func() {
+						t.Logf("End test (%s, %s, sync/%s)", clientDef.Name, currentTest.Name, variant.Name)
+					}()
 
-						// Prepare sync client parameters
-						syncClientParams := newParams.Copy()
-						for k, v := range variant.SyncClientConfig {
-							syncClientParams = syncClientParams.Set(k, v)
+					timeout := globals.DefaultTestCaseTimeout
+					// If a test.Spec specifies a timeout, use that instead
+					if currentTest.TimeoutSeconds != 0 {
+						timeout = time.Second * time.Duration(currentTest.TimeoutSeconds)
+					}
+
+					// Prepare sync client parameters
+					syncClientParams := newParams.Copy()
+					for k, v := range variant.SyncClientConfig {
+						syncClientParams = syncClientParams.Set(k, v)
+					}
+
+					// Setup the CL Mocker for this test
+					clMocker := clmock.NewCLMocker(t, currentTest.SlotsToSafe, currentTest.SlotsToFinalized, big.NewInt(currentTest.SafeSlotsToImportOptimistically))
+					// Defer closing all clients
+					defer func() {
+						clMocker.CloseClients()
+					}()
+
+					// Set up test context, which has a few more seconds to finish up after timeout happens
+					ctx, cancel := context.WithTimeout(context.Background(), timeout+(time.Second*10))
+					defer cancel()
+					clMocker.TestContext = ctx
+
+					env := &test.Env{
+						T:                   t,
+						TestName:            currentTest.Name,
+						Clients:             make([]client.EngineClient, 0),
+						CLMock:              clMocker,
+						ClientParams:        syncClientParams,
+						ClientFiles:         testFiles.Copy(),
+						TestTransactionType: currentTest.TestTransactionType,
+						TestContext:         ctx,
+					}
+
+					// Setup the main test client
+					var ec client.EngineClient
+					ec = env.StartNextClient(mainClientParams, mainClientFiles)
+					defer ec.Close()
+
+					// Create the test-expect object
+					env.TestEngine = test.NewTestEngineClient(env, ec)
+
+					// Add main client to CLMocker
+					clMocker.AddEngineClient(ec)
+
+					// Setup context timeout
+					ctx, cancel = context.WithTimeout(ctx, timeout)
+					defer cancel()
+					env.TimeoutContext = ctx
+					clMocker.TimeoutContext = ctx
+
+					// Defer producing one last block to verify Execution client did not break after the test
+					defer func() {
+						// Only run if the TTD was reached during test, and test had not failed at this point.
+						if clMocker.TTDReached && !t.Failed() {
+							clMocker.ProduceSingleBlock(clmock.BlockProcessCallbacks{})
 						}
+					}()
 
-						// Run the test case
-						test.Run(currentTest.Name, big.NewInt(ttd), currentTest.SlotsToSafe, currentTest.SlotsToFinalized, timeout, t, c, currentTest.Run, syncClientParams, testFiles.Copy(), currentTest.TestTransactionType, currentTest.SafeSlotsToImportOptimistically)
-					},
-				})
-			}
+					// Run the test
+					currentTest.Run(env)
+				},
+			})
 		}
 	}
 

simulators/ethereum/engine/test/env.go

@@ -3,7 +3,6 @@ package test
 import (
 	"context"
 	"math/big"
-	"net/http"
 	"time"
 
 	"github.com/ethereum/hive/simulators/ethereum/engine/client"
@@ -20,7 +19,6 @@ import (
 type Env struct {
 	*hivesim.T
 	TestName string
-	Client   *hivesim.Client
 
 	// Timeout context signals that the test must wrap up its execution
 	TimeoutContext context.Context
@@ -29,10 +27,10 @@ type Env struct {
 	TestContext context.Context
 
 	// RPC Clients
+	Clients    []client.EngineClient
 	Engine     client.EngineClient
 	Eth        client.Eth
 	TestEngine *TestEngineClient
-	HiveEngine *hive_rpc.HiveRPCEngineClient
 
 	// Consensus Layer Mocker Instance
 	CLMock *clmock.CLMocker
@@ -45,58 +43,48 @@ type Env struct {
 	TestTransactionType helper.TestTransactionType
 }
 
-func Run(testName string, ttd *big.Int, slotsToSafe *big.Int, slotsToFinalized *big.Int, timeout time.Duration, t *hivesim.T, c *hivesim.Client, fn func(*Env), cParams hivesim.Params, cFiles hivesim.Params, testTransactionType helper.TestTransactionType, safeSlotsToImportOptimistically int64) {
+func Run(testName string, ttd *big.Int, slotsToSafe *big.Int, slotsToFinalized *big.Int, timeout time.Duration, t *hivesim.T, fn func(*Env), cParams hivesim.Params, cFiles hivesim.Params, testTransactionType helper.TestTransactionType, safeSlotsToImportOptimistically int64) {
+
 	// Setup the CL Mocker for this test
 	clMocker := clmock.NewCLMocker(t, slotsToSafe, slotsToFinalized, big.NewInt(safeSlotsToImportOptimistically))
 	// Defer closing all clients
 	defer func() {
 		clMocker.CloseClients()
 	}()
 
-	// Create Engine client from main hivesim.Client to be used by tests
-	ec := hive_rpc.NewHiveRPCEngineClient(c, globals.EnginePortHTTP, globals.EthPortHTTP, globals.DefaultJwtTokenSecretBytes, ttd, &helper.LoggingRoundTrip{
-		T:     t,
-		Hc:    c,
-		Inner: http.DefaultTransport,
-	})
-	defer ec.Close()
-
-	// Add main client to CLMocker
-	clMocker.AddEngineClient(ec)
+	// Set up test context, which has a few more seconds to finish up after timeout happens
+	ctx, cancel := context.WithTimeout(context.Background(), timeout+(time.Second*10))
+	defer cancel()
+	clMocker.TestContext = ctx
 
 	env := &Env{
 		T:                   t,
 		TestName:            testName,
-		Client:              c,
-		Engine:              ec,
-		Eth:                 ec,
-		HiveEngine:          ec,
+		Clients:             make([]client.EngineClient, 0),
 		CLMock:              clMocker,
 		ClientParams:        cParams,
 		ClientFiles:         cFiles,
 		TestTransactionType: testTransactionType,
+		TestContext:         ctx,
 	}
 
-	// Before running the test, make sure Eth and Engine ports are open for the client
-	if err := hive_rpc.CheckEthEngineLive(c); err != nil {
-		t.Fatalf("FAIL (%s): Ports were never open for client: %v", env.TestName, err)
-	}
+	// Setup the main test client
+	var ec client.EngineClient
+	ec = env.StartNextClient(cParams, cFiles)
+	defer ec.Close()
 
-	// Full test context has a few more seconds to finish up after timeout happens
-	ctx, cancel := context.WithTimeout(context.Background(), timeout+(time.Second*10))
-	defer cancel()
-	env.TestContext = ctx
-	clMocker.TestContext = ctx
+	// Create the test-expect object
+	env.TestEngine = NewTestEngineClient(env, ec)
+
+	// Add main client to CLMocker
+	clMocker.AddEngineClient(ec)
 
 	// Setup context timeout
 	ctx, cancel = context.WithTimeout(ctx, timeout)
 	defer cancel()
 	env.TimeoutContext = ctx
 	clMocker.TimeoutContext = ctx
 
-	// Create the test-expect object
-	env.TestEngine = NewTestEngineClient(env, ec)
-
 	// Defer producing one last block to verify Execution client did not break after the test
 	defer func() {
 		// Only run if the TTD was reached during test, and test had not failed at this point.
@@ -113,6 +101,51 @@ func (t *Env) MainTTD() *big.Int {
 	return t.Engine.TerminalTotalDifficulty()
 }
 
+func (t *Env) NextClientType() *hivesim.ClientDefinition {
+	testClientTypes, err := t.Sim.ClientTypes()
+	if err != nil {
+		t.Fatalf("No client types")
+	}
+	nextClientTypeIndex := len(t.Clients) % len(testClientTypes)
+	return testClientTypes[nextClientTypeIndex]
+}
+
+func (t *Env) ClientStarter(clientType string) client.EngineStarter {
+	return hive_rpc.HiveRPCEngineStarter{
+		ClientType: clientType,
+	}
+}
+
+func (t *Env) NextClientStarter() client.EngineStarter {
+	return t.ClientStarter(t.NextClientType().Name)
+}
+
+func (t *Env) StartClient(clientType string, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootclients ...client.EngineClient) client.EngineClient {
+	ec, err := t.ClientStarter(clientType).StartClient(t.T, t.TestContext, ClientParams, ClientFiles, bootclients...)
+	if err != nil {
+		t.Fatalf("FAIL (%s): Unable to start client: %v", t.TestName, err)
+	}
+	if len(t.Clients) == 0 {
+		t.Engine = ec
+		t.Eth = ec
+	}
+	t.Clients = append(t.Clients, ec)
+	return ec
+}
+
+func (t *Env) StartNextClient(ClientParams hivesim.Params, ClientFiles hivesim.Params, bootclients ...client.EngineClient) client.EngineClient {
+	ec, err := t.NextClientStarter().StartClient(t.T, t.TestContext, ClientParams, ClientFiles, bootclients...)
+	if err != nil {
+		t.Fatalf("FAIL (%s): Unable to start client: %v", t.TestName, err)
+	}
+	if len(t.Clients) == 0 {
+		t.Engine = ec
+		t.Eth = ec
+	}
+	t.Clients = append(t.Clients, ec)
+	return ec
+}
+
 func (t *Env) HandleClientPostRunVerification(ec client.EngineClient) {
 	if err := ec.PostRunVerifications(); err != nil {
 		t.Fatalf("FAIL (%s): Client failed post-run verification: %v", t.TestName, err)