Merge pull request #162 from chainbound/chore/devnet/multiproofs-update

fix(devnet): better support of multiple preconfs in blocks

Justfile

@@ -102,23 +102,25 @@ dora:
   fi
 
 # manually send a preconfirmation to the bolt devnet
-send-preconf:
+send-preconf count='1':
 	cd bolt-spammer && RUST_LOG=info cargo run -- \
 		--provider-url $(kurtosis port print bolt-devnet el-1-geth-lighthouse rpc) \
 		--beacon-client-url $(kurtosis port print bolt-devnet cl-1-lighthouse-geth http) \
 		--bolt-sidecar-url http://$(kurtosis port print bolt-devnet mev-sidecar-api api)  \
 		--private-key 53321db7c1e331d93a11a41d16f004d7ff63972ec8ec7c25db329728ceeb1710 \
-		--slot head
+		--slot head \
+		--count {{count}}
 
 # manually send a blob preconfirmation to the bolt devnet
-send-blob-preconf:
+send-blob-preconf count='1':
 	cd bolt-spammer && RUST_LOG=info cargo run -- \
 		--provider-url $(kurtosis port print bolt-devnet el-1-geth-lighthouse rpc) \
 		--beacon-client-url $(kurtosis port print bolt-devnet cl-1-lighthouse-geth http) \
 		--bolt-sidecar-url http://$(kurtosis port print bolt-devnet mev-sidecar-api api)  \
 		--private-key 53321db7c1e331d93a11a41d16f004d7ff63972ec8ec7c25db329728ceeb1710 \
 		--slot head \
-		--blob
+		--blob \
+		--count {{count}} \
 
 # build all the docker images locally
 build-images:

bolt-sidecar/src/config/mod.rs

@@ -146,7 +146,7 @@ pub struct Limits {
 impl Default for Limits {
     fn default() -> Self {
         Self {
-            max_commitments_per_slot: NonZero::new(6).expect("Valid non-zero"),
+            max_commitments_per_slot: NonZero::new(128).expect("Valid non-zero"),
             max_committed_gas_per_slot: NonZero::new(10_000_000).expect("Valid non-zero"),
         }
     }

bolt-spammer/src/main.rs

@@ -33,6 +33,8 @@ struct Opts {
     blob: bool,
     #[clap(short = 's', long, default_value = "head")]
     slot: String,
+    #[clap(short = 'C', long, default_value_t = 1)]
+    count: u64,
 }
 
 #[tokio::main]
@@ -53,45 +55,47 @@ async fn main() -> Result<()> {
     let current_slot = current_slot(&beacon_api_client).await?;
     let target_slot = if opts.slot == "head" { current_slot + 2 } else { opts.slot.parse()? };
 
-    let mut tx = if opts.blob { generate_random_blob_tx() } else { generate_random_tx() };
-    tx.set_from(sender);
-    tx.set_nonce(provider.get_transaction_count(sender).await?);
-
-    let tx_signed = tx.build(&transaction_signer).await?;
-    let tx_hash = tx_signed.tx_hash().to_string();
-    let tx_rlp = hex::encode(tx_signed.encoded_2718());
-
-    let message_digest = {
-        let mut data = Vec::new();
-        data.extend_from_slice(&target_slot.to_le_bytes());
-        data.extend_from_slice(hex::decode(tx_hash.trim_start_matches("0x"))?.as_slice());
-        keccak256(data)
-    };
-
-    let signature = wallet.sign_hash(&message_digest).await?;
-    let signature = hex::encode(signature.as_bytes());
-
-    let request = prepare_rpc_request(
-        "bolt_inclusionPreconfirmation",
-        vec![serde_json::json!({
-            "slot": target_slot,
-            "tx": tx_rlp,
-            "signature": signature,
-        })],
-    );
-
-    info!("Transaction hash: {}", tx_hash);
-    info!("body: {}", serde_json::to_string(&request)?);
-
-    let client = reqwest::Client::new();
-    let response = client
-        .post(&opts.bolt_sidecar_url)
-        .header("content-type", "application/json")
-        .body(serde_json::to_string(&request)?)
-        .send()
-        .await?;
-
-    info!("Response: {:?}", response.text().await?);
+    for i in 0..opts.count {
+        let mut tx = if opts.blob { generate_random_blob_tx() } else { generate_random_tx() };
+        tx.set_from(sender);
+        tx.set_nonce(provider.get_transaction_count(sender).await? + i);
+
+        let tx_signed = tx.build(&transaction_signer).await?;
+        let tx_hash = tx_signed.tx_hash().to_string();
+        let tx_rlp = hex::encode(tx_signed.encoded_2718());
+
+        let message_digest = {
+            let mut data = Vec::new();
+            data.extend_from_slice(&target_slot.to_le_bytes());
+            data.extend_from_slice(hex::decode(tx_hash.trim_start_matches("0x"))?.as_slice());
+            keccak256(data)
+        };
+
+        let signature = wallet.sign_hash(&message_digest).await?;
+        let signature = hex::encode(signature.as_bytes());
+
+        let request = prepare_rpc_request(
+            "bolt_inclusionPreconfirmation",
+            vec![serde_json::json!({
+                "slot": target_slot,
+                "tx": tx_rlp,
+                "signature": signature,
+            })],
+        );
+
+        info!("Transaction hash: {}", tx_hash);
+        info!("body: {}", serde_json::to_string(&request)?);
+
+        let client = reqwest::Client::new();
+        let response = client
+            .post(&opts.bolt_sidecar_url)
+            .header("content-type", "application/json")
+            .body(serde_json::to_string(&request)?)
+            .send()
+            .await?;
+
+        info!("Response: {:?}", response.text().await?);
+    }
 
     Ok(())
 }

bolt-spammer/src/utils.rs

@@ -19,7 +19,7 @@ pub fn generate_random_tx() -> TransactionRequest {
         .with_to(Address::from_str(DEAD_ADDRESS).unwrap())
         .with_chain_id(KURTOSIS_CHAIN_ID)
         .with_value(U256::from(thread_rng().gen_range(1..100)))
-        .with_gas_limit(1_000_000u128)
+        .with_gas_limit(21_000u128)
         .with_gas_price(NOICE_GAS_PRICE)
 }
 
@@ -38,7 +38,7 @@ pub fn generate_random_blob_tx() -> TransactionRequest {
         .with_max_fee_per_blob_gas(100u128)
         .max_fee_per_gas(NOICE_GAS_PRICE)
         .max_priority_fee_per_gas(NOICE_GAS_PRICE / 10)
-        .with_gas_limit(1_000_000u128)
+        .with_gas_limit(42_000u128)
         .with_blob_sidecar(sidecar)
         .with_input(random_bytes)
 }

builder/Dockerfile

@@ -4,7 +4,7 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.21-alpine AS builder
+FROM golang:1.22-alpine AS builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 

builder/Dockerfile.alltools

@@ -4,7 +4,7 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.21-alpine AS builder
+FROM golang:1.22-alpine AS builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 

builder/builder/utils.go

@@ -156,8 +156,11 @@ func EmitBoltDemoEvent(message string) {
 
 func CalculateMerkleMultiProofs(
 	payloadTransactions types.Transactions,
-	constraints types.HashToConstraintDecoded,
+	HashToConstraintDecoded types.HashToConstraintDecoded,
 ) (inclusionProof *common.InclusionProof, rootNode *ssz.Node, err error) {
+	constraintsOrderedByIndex, constraintsWithoutIndex, _, _ := types.ParseConstraintsDecoded(HashToConstraintDecoded)
+	constraints := slices.Concat(constraintsOrderedByIndex, constraintsWithoutIndex)
+
 	// BOLT: generate merkle tree from payload transactions (we need raw RLP bytes for this)
 	rawTxs := make([]bellatrix.Transaction, len(payloadTransactions))
 	for i, tx := range payloadTransactions {
@@ -185,21 +188,20 @@ func CalculateMerkleMultiProofs(
 	baseGeneralizedIndex := int(math.Pow(float64(2), float64(21)))
 	generalizedIndexes := make([]int, len(constraints))
 	transactionHashes := make([]common.Hash, len(constraints))
-	i := 0
 
-	for hash := range constraints {
+	for i, constraint := range constraints {
+		tx := constraint.Tx
 		// get the index of the preconfirmed transaction in the block
-		preconfIndex := slices.IndexFunc(payloadTransactions, func(tx *types.Transaction) bool { return tx.Hash() == hash })
+		preconfIndex := slices.IndexFunc(payloadTransactions, func(payloadTx *types.Transaction) bool { return payloadTx.Hash() == tx.Hash() })
 		if preconfIndex == -1 {
-			log.Error(fmt.Sprintf("Preconfirmed transaction %s not found in block", hash))
+			log.Error(fmt.Sprintf("Preconfirmed transaction %s not found in block", tx.Hash()))
 			log.Error(fmt.Sprintf("block has %v transactions", len(payloadTransactions)))
 			continue
 		}
 
 		generalizedIndex := baseGeneralizedIndex + preconfIndex
 		generalizedIndexes[i] = generalizedIndex
-		transactionHashes[i] = hash
-		i++
+		transactionHashes[i] = tx.Hash()
 	}
 
 	log.Info(fmt.Sprintf("[BOLT]: Calculating merkle multiproof for %d preconfirmed transaction",

builder/core/types/constraints.go

@@ -1,6 +1,10 @@
 package types
 
-import "github.com/ethereum/go-ethereum/common"
+import (
+	"sort"
+
+	"github.com/ethereum/go-ethereum/common"
+)
 
 // NOTE: not the greatest place for this type but given that it uses
 // `common.Hash`, `Transaction` and it's used in both the builder
@@ -13,3 +17,46 @@ type (
 		Tx    *Transaction
 	}
 )
+
+// ParseConstraintsDecoded receives a map of constraints and returns
+// - a slice of constraints sorted by index
+// - a slice of constraints without index sorted by nonce and hash
+// - the total gas required by the constraints
+// - the total blob gas required by the constraints
+func ParseConstraintsDecoded(constraints HashToConstraintDecoded) ([]*ConstraintDecoded, []*ConstraintDecoded, uint64, uint64) {
+	// Here we initialize and track the constraints left to be executed along
+	// with their gas requirements
+	constraintsOrderedByIndex := make([]*ConstraintDecoded, 0, len(constraints))
+	constraintsWithoutIndex := make([]*ConstraintDecoded, 0, len(constraints))
+	constraintsTotalGasLeft := uint64(0)
+	constraintsTotalBlobGasLeft := uint64(0)
+
+	for _, constraint := range constraints {
+		if constraint.Index == nil {
+			constraintsWithoutIndex = append(constraintsWithoutIndex, constraint)
+		} else {
+			constraintsOrderedByIndex = append(constraintsOrderedByIndex, constraint)
+		}
+		constraintsTotalGasLeft += constraint.Tx.Gas()
+		constraintsTotalBlobGasLeft += constraint.Tx.BlobGas()
+	}
+
+	// Sorts the constraints by index ascending
+	sort.Slice(constraintsOrderedByIndex, func(i, j int) bool {
+		// By assumption, all constraints here have a non-nil index
+		return *constraintsOrderedByIndex[i].Index < *constraintsOrderedByIndex[j].Index
+	})
+
+	// Sorts the unindexed constraints by nonce ascending and by hash
+	sort.Slice(constraintsWithoutIndex, func(i, j int) bool {
+		iNonce := constraintsWithoutIndex[i].Tx.Nonce()
+		jNonce := constraintsWithoutIndex[j].Tx.Nonce()
+		// Sort by hash
+		if iNonce == jNonce {
+			return constraintsWithoutIndex[i].Tx.Hash().Cmp(constraintsWithoutIndex[j].Tx.Hash()) < 0
+		}
+		return iNonce < jNonce
+	})
+
+	return constraintsOrderedByIndex, constraintsWithoutIndex, constraintsTotalGasLeft, constraintsTotalBlobGasLeft
+}

builder/go.mod

@@ -1,6 +1,6 @@
 module github.com/ethereum/go-ethereum
 
-go 1.20
+go 1.22
 
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0
@@ -26,7 +26,7 @@ require (
 	github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127
 	github.com/ethereum/c-kzg-4844 v0.4.0
 	github.com/fatih/color v1.15.0
-	github.com/ferranbt/fastssz v0.1.3
+	github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688
 	github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e
 	github.com/fjl/memsize v0.0.2
 	github.com/flashbots/go-boost-utils v1.8.0
@@ -86,6 +86,7 @@ require (
 )
 
 require (
+	github.com/emicklei/dot v1.6.2 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/goccy/go-yaml v1.11.2 // indirect

builder/go.sum

@@ -137,6 +137,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA
 github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A=
+github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
@@ -150,8 +152,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo=
-github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE=
+github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 h1:k70X5h1haHaSbpD/9fcjtvAUEVlRlOKtdpvN7Mzhcv4=
+github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg=
 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY=
 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY=
 github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA=

builder/miner/worker.go

@@ -1033,26 +1033,10 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac
 
 	// Here we initialize and track the constraints left to be executed along
 	// with their gas requirements
-	constraintsOrderedByIndex := make([]*types.ConstraintDecoded, 0, len(constraints))
-	constraintsWithoutIndex := make([]*types.ConstraintDecoded, 0, len(constraints))
-	constraintsTotalGasLeft := uint64(0)
-	constraintsTotalBlobGasLeft := uint64(0)
-
-	for _, constraint := range constraints {
-		if constraint.Index == nil {
-			constraintsWithoutIndex = append(constraintsWithoutIndex, constraint)
-		} else {
-			constraintsOrderedByIndex = append(constraintsOrderedByIndex, constraint)
-		}
-		constraintsTotalGasLeft += constraint.Tx.Gas()
-		constraintsTotalBlobGasLeft += constraint.Tx.BlobGas()
-	}
-
-	// Sorts the constraints by index ascending
-	sort.Slice(constraintsOrderedByIndex, func(i, j int) bool {
-		// By assumption, all constraints here have a non-nil index
-		return *constraintsOrderedByIndex[i].Index < *constraintsOrderedByIndex[j].Index
-	})
+	constraintsOrderedByIndex,
+		constraintsWithoutIndex,
+		constraintsTotalGasLeft,
+		constraintsTotalBlobGasLeft := types.ParseConstraintsDecoded(constraints)
 
 	for {
 		// `env.tcount` starts from 0 so it's correct to use it as the current index
@@ -1177,7 +1161,7 @@ func (w *worker) commitTransactions(env *environment, plainTxs, blobTxs *transac
 					// As such, we can safely exist
 					break
 				}
-				candidate = candidateTx{tx: common.Pop(&constraintsWithoutIndex).Tx, isConstraint: true}
+				candidate = candidateTx{tx: common.Shift(&constraintsWithoutIndex).Tx, isConstraint: true}
 			}
 		}
 

mev-boost-relay/go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 // indirect
 	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
+	github.com/emicklei/dot v1.6.2 // indirect
 	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
@@ -91,7 +92,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/ferranbt/fastssz v0.1.3
+	github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect

mev-boost-relay/go.sum

@@ -116,6 +116,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A=
+github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
@@ -131,6 +133,8 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo=
 github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE=
+github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688 h1:k70X5h1haHaSbpD/9fcjtvAUEVlRlOKtdpvN7Mzhcv4=
+github.com/ferranbt/fastssz v0.1.4-0.20240724090034-31cd371f8688/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg=
 github.com/flashbots/go-boost-utils v1.8.0 h1:z3K1hw+Fbl9AGMNQKnK7Bvf0M/rKgjfruAEvra+Z8Mg=
 github.com/flashbots/go-boost-utils v1.8.0/go.mod h1:Ry1Rw8Lx5v1rpAR0+IvR4sV10jYAeQaGVM3vRD8mYdM=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=