Supporting driving multiple EL nodes from a Nimbus BN

Full list of changes:

* Eth1Monitor has been renamed to ELManager to match its current
  responsibilities better.

* The ELManager is no longer optional in the code (it won't have
  a nil value under any circumstances).

* The support for subscribing for headers was removed as it only
  worked with WebSockets and contributed significant complexity
  while bringing only a very minor advantage.

* Transition configuration is now exchanged 2 weeks prior to the
  Bellatrix fork (at this point, relevant only for future testnets).

* The `--web3-url` parameter has been deprecated in favor of a new
  `--el` parameter. The new parameter supports specifying the JWT
  token per connection. Each connection can also be configured with
  a different set of responsibilities (e.g. download deposits, validate
  blocks and/or produce blocks). On the command-line, these properties
  can be configured through URL properties stored in the #anchor part
  of the URL. In TOML files, they come with a very natural syntax
  (althrough the URL scheme is also supported).

* The previously scattered EL-related state and logic is now moved
  to `eth1_monitor.nim` (this module will be renamed to `el_manager.nim`
  in a follow-up commit). State is assigned properly either to the
  `ELManager` or the to individual `ELConnection` objects where
  appropriate.

  The ELManager executes all Engine API requests against all attached
  EL nodes, in parallel. It compares their results and if there is a
  disagreement regarding the validity of a certain payload, this is
  detected and the beacon node is protected from publishing a block
  with a potential execution layer consensus bug in it.

  The BN provides metrics per EL node for the number of successful or
  failed requests for each type Engine API requests. If an EL node
  goes offline and connectivity is resoted later, we report the
  problem and the remedy in edge-triggered fashion.

* The client can now produce Capella execution payloads (with empty
  withdrawal lists for now)

* The local testnet simulation script now runs Geth and Nimbus by
  default. The minimal network simulation uses a Nimbus remote signing
  node for 50% of the validators. The mainnet simulation uses the
  official web3signer in a 2 out of 3 threshold signing config.

* The local testnet simulation can now use a payload builder.
  This is currently not activated in CI due to lack of automated
  procedures for installing third-party relays or builders.

  You are adviced to use mergemock for now, but for most realistic
  results, we can create a simple builder based on the nimbus-eth1
  codebase that will be able to propose transactions from the regular
  network mempool.

* Updated the `run-geth-el.sh` script to work with the latest Capella
  and EIP-4844 builds of Geth and added the `run-nimbus-eth2-in-withdrawal-testnet.sh`
  script that can be used to launch Nimbus instances in the available
  withdrawal devnets.

Makefile

@@ -174,11 +174,12 @@ libbacktrace:
 # - --base-metrics-port + [0, --nodes)
 # - --base-vc-keymanager-port + [0, --nodes)
 # - --base-vc-metrics-port + [0, --nodes]
-# - --base-remote-signer-port + [0, --remote-signers)
+# - --base-remote-signer-port + [0, --nimbus-signer-nodes | --web3signer-nodes)
+# - --base-remote-signer-metrics-port + [0, --nimbus-signer-node | --web3signer-nodes)
 #
 # Local testnets with --run-geth or --run-nimbus (only these ports):
 # - --base-el-net-port + --el-port-offset * [0, --nodes + --light-clients)
-# - --base-el-http-port + --el-port-offset * [0, --nodes + --light-clients)
+# - --base-el-rpc-port + --el-port-offset * [0, --nodes + --light-clients)
 # - --base-el-ws-port + --el-port-offset * [0, --nodes + --light-clients)
 # - --base-el-auth-rpc-port + --el-port-offset * [0, --nodes + --light-clients)
 UNIT_TEST_BASE_PORT := 9950
@@ -192,34 +193,36 @@ restapi-test:
 		--resttest-delay 30 \
 		--kill-old-processes
 
-ifneq ($(shell uname -p), arm)
-TESTNET_EXTRA_FLAGS := --run-geth --dl-geth
-else
-TESTNET_EXTRA_FLAGS :=
-endif
-
 local-testnet-minimal:
 	./scripts/launch_local_testnet.sh \
 		--data-dir $@ \
 		--preset minimal \
 		--nodes 4 \
-		--stop-at-epoch 6 \
+		--capella-fork-epoch 4 \
+		--eip4844-fork-epoch 20 \
+		--stop-at-epoch 10 \
 		--disable-htop \
+		--remote-validators-count 512 \
+		--enable-payload-builder \
+		--nimbus-signer-nodes 1 \
+		--threshold 1 \
 		--enable-logtrace \
 		--base-port $$(( 6001 + EXECUTOR_NUMBER * 500 )) \
 		--base-rest-port $$(( 6031 + EXECUTOR_NUMBER * 500 )) \
 		--base-metrics-port $$(( 6061 + EXECUTOR_NUMBER * 500 )) \
 		--base-vc-keymanager-port $$(( 6131 + EXECUTOR_NUMBER * 500 )) \
 		--base-vc-metrics-port $$(( 6161 + EXECUTOR_NUMBER * 500 )) \
 		--base-remote-signer-port $$(( 6201 + EXECUTOR_NUMBER * 500 )) \
+		--base-remote-signer-metrics-port $$(( 6251 + EXECUTOR_NUMBER * 500 )) \
 		--base-el-net-port $$(( 6301 + EXECUTOR_NUMBER * 500 )) \
-		--base-el-http-port $$(( 6302 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-rpc-port $$(( 6302 + EXECUTOR_NUMBER * 500 )) \
 		--base-el-ws-port $$(( 6303 + EXECUTOR_NUMBER * 500 )) \
 		--base-el-auth-rpc-port $$(( 6304 + EXECUTOR_NUMBER * 500 )) \
 		--el-port-offset 5 \
 		--timeout 648 \
 		--kill-old-processes \
-		$(TESTNET_EXTRA_FLAGS) \
+		--run-geth --dl-geth \
+		--run-nimbus-eth1 --dl-nimbus-eth1 \
 		-- \
 		--verify-finalization \
 		--discv5:no
@@ -231,20 +234,25 @@ local-testnet-mainnet:
 		--stop-at-epoch 6 \
 		--disable-htop \
 		--enable-logtrace \
+		--nimbus-signer-nodes 3 \
+		--threshold 2 \
+		--remote-validators-count 512 \
 		--base-port $$(( 7001 + EXECUTOR_NUMBER * 500 )) \
 		--base-rest-port $$(( 7031 + EXECUTOR_NUMBER * 500 )) \
 		--base-metrics-port $$(( 7061 + EXECUTOR_NUMBER * 500 )) \
 		--base-vc-keymanager-port $$(( 7131 + EXECUTOR_NUMBER * 500 )) \
 		--base-vc-metrics-port $$(( 7161 + EXECUTOR_NUMBER * 500 )) \
 		--base-remote-signer-port $$(( 7201 + EXECUTOR_NUMBER * 500 )) \
+		--base-remote-signer-metrics-port $$(( 7251 + EXECUTOR_NUMBER * 500 )) \
 		--base-el-net-port $$(( 7301 + EXECUTOR_NUMBER * 500 )) \
-		--base-el-http-port $$(( 7302 + EXECUTOR_NUMBER * 500 )) \
+		--base-el-rpc-port $$(( 7302 + EXECUTOR_NUMBER * 500 )) \
 		--base-el-ws-port $$(( 7303 + EXECUTOR_NUMBER * 500 )) \
 		--base-el-auth-rpc-port $$(( 7304 + EXECUTOR_NUMBER * 500 )) \
 		--el-port-offset 5 \
 		--timeout 2784 \
 		--kill-old-processes \
-		$(TESTNET_EXTRA_FLAGS) \
+		--run-geth --dl-geth \
+		--run-nimbus-eth1 --dl-nimbus-eth1 \
 		-- \
 		--verify-finalization \
 		--discv5:no

beacon_chain/beacon_node.nim

@@ -66,7 +66,7 @@ type
     syncCommitteeMsgPool*: ref SyncCommitteeMsgPool
     lightClientPool*: ref LightClientPool
     exitPool*: ref ExitPool
-    eth1Monitor*: Eth1Monitor
+    elManager*: ELManager
     payloadBuilderRestClient*: RestClientRef
     restServer*: RestServerRef
     keymanagerHost*: ref KeymanagerHost
@@ -87,7 +87,6 @@ type
     restKeysCache*: Table[ValidatorPubKey, ValidatorIndex]
     validatorMonitor*: ref ValidatorMonitor
     stateTtlCache*: StateTtlCache
-    nextExchangeTransitionConfTime*: Moment
     router*: ref MessageRouter
     dynamicFeeRecipientsStore*: ref DynamicFeeRecipientsStore
     externalBuilderRegistrations*:

beacon_chain/beacon_node_light_client.nim

@@ -11,14 +11,12 @@ else:
   {.push raises: [].}
 
 import
-  chronicles,
+  chronicles, web3/engine_api_types,
   ./beacon_node
 
 logScope: topics = "beacnde"
 
 func shouldSyncOptimistically*(node: BeaconNode, wallSlot: Slot): bool =
-  if node.eth1Monitor == nil:
-    return false
   let optimisticHeader = node.lightClient.optimisticHeader.valueOr:
     return false
 
@@ -42,7 +40,7 @@ proc initLightClient*(
 
   let
     optimisticHandler = proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock):
-        Future[void] {.async.} =
+                             Future[void] {.async.} =
       info "New LC optimistic block",
         opt = signedBlock.toBlockId(),
         dag = node.dag.head.bid,
@@ -54,10 +52,9 @@ proc initLightClient*(
           if blck.message.is_execution_block:
             template payload(): auto = blck.message.body.execution_payload
 
-            let eth1Monitor = node.eth1Monitor
-            if eth1Monitor != nil and not payload.block_hash.isZero:
+            if not payload.block_hash.isZero:
               # engine_newPayloadV1
-              discard await eth1Monitor.newExecutionPayload(payload)
+              discard await node.elManager.newExecutionPayload(payload)
 
               # Retain optimistic head for other `forkchoiceUpdated` callers.
               # May temporarily block `forkchoiceUpdatedV1` calls, e.g., Geth:
@@ -70,10 +67,10 @@ proc initLightClient*(
 
               # engine_forkchoiceUpdatedV1
               let beaconHead = node.attestationPool[].getBeaconHead(nil)
-              discard await eth1Monitor.runForkchoiceUpdated(
-                headBlockRoot = payload.block_hash,
-                safeBlockRoot = beaconHead.safeExecutionPayloadHash,
-                finalizedBlockRoot = beaconHead.finalizedExecutionPayloadHash)
+              discard await node.elManager.forkchoiceUpdated(
+                headBlock = payload.block_hash,
+                safeBlock = beaconHead.safeExecutionPayloadHash,
+                finalizedBlock = beaconHead.finalizedExecutionPayloadHash)
           else: discard
 
     optimisticProcessor = initOptimisticProcessor(

beacon_chain/conf.nim

@@ -29,6 +29,7 @@ import
   ./spec/datatypes/base,
   ./networking/network_metadata,
   ./validators/slashing_protection_common,
+  ./eth1/el_conf,
   ./filepath
 
 from consensus_object_pools/block_pools_types_light_client
@@ -38,7 +39,7 @@ export
   uri, nat, enr,
   defaultEth2TcpPort, enabledLogLevel, ValidIpAddress,
   defs, parseCmdArg, completeCmdArg, network_metadata,
-  network, BlockHashOrNumber,
+  el_conf, network, BlockHashOrNumber,
   confTomlDefs, confTomlNet, confTomlUri
 
 declareGauge network_name, "network name", ["name"]
@@ -170,14 +171,12 @@ type
       name: "era-dir" .}: Option[InputDir]
 
     web3Urls* {.
-      desc: "One or more execution layer Web3 provider URLs"
-      name: "web3-url" .}: seq[string]
+      desc: "One or more Execution Layer Engine API URLs"
+      name: "web3-url" .}: seq[EngineApiUrlConfigValue]
 
-    web3ForcePolling* {.
-      hidden
-      defaultValue: false
-      desc: "Force the use of polling when determining the head block of Eth1"
-      name: "web3-force-polling" .}: bool
+    elUrls* {.
+      desc: "One or more Execution Layer Engine API URLs"
+      name: "el" .}: seq[EngineApiUrlConfigValue]
 
     optimistic* {.
       hidden # deprecated > 22.12
@@ -228,7 +227,7 @@ type
     # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.1/src/engine/authentication.md#key-distribution
     jwtSecret* {.
       desc: "A file containing the hex-encoded 256 bit secret key to be used for verifying/generating JWT tokens"
-      name: "jwt-secret" .}: Option[string]
+      name: "jwt-secret" .}: Option[InputFile]
 
     case cmd* {.
       command
@@ -597,10 +596,13 @@ type
         defaultValueDesc: $defaultEth2TcpPortDesc
         name: "bootstrap-port" .}: Port
 
+      genesisTime* {.
+        desc: "Unix epoch time of the network genesis"
+        name: "genesis-time" .}: Option[uint64]
+
       genesisOffset* {.
         desc: "Seconds from now to add to genesis time"
-        defaultValue: 5
-        name: "genesis-offset" .}: int
+        name: "genesis-offset" .}: Option[int]
 
       outputGenesis* {.
         desc: "Output file where to write the initial state snapshot"
@@ -1290,7 +1292,7 @@ func defaultFeeRecipient*(conf: AnyConf): Eth1Address =
 proc loadJwtSecret*(
     rng: var HmacDrbgContext,
     dataDir: string,
-    jwtSecret: Option[string],
+    jwtSecret: Option[InputFile],
     allowCreate: bool): Option[seq[byte]] =
   # Some Web3 endpoints aren't compatible with JWT, but if explicitly chosen,
   # use it regardless.
@@ -1305,8 +1307,11 @@ proc loadJwtSecret*(
   else:
     none(seq[byte])
 
-template loadJwtSecret*(
+proc loadJwtSecret*(
     rng: var HmacDrbgContext,
     config: BeaconNodeConf,
     allowCreate: bool): Option[seq[byte]] =
   rng.loadJwtSecret(string(config.dataDir), config.jwtSecret, allowCreate)
+
+proc engineApiUrls*(config: BeaconNodeConf): seq[EngineApiUrl] =
+  (config.elUrls & config.web3Urls).toFinalEngineApiUrls(config.jwtSecret)

beacon_chain/conf_light_client.nim

@@ -127,11 +127,15 @@ type LightClientConf* = object
   # Execution layer
   web3Urls* {.
     desc: "One or more execution layer Web3 provider URLs"
-    name: "web3-url" .}: seq[string]
+    name: "web3-url" .}: seq[EngineApiUrlConfigValue]
+
+  elUrls* {.
+    desc: "One or more Execution Layer Engine API URLs"
+    name: "el" .}: seq[EngineApiUrlConfigValue]
 
   jwtSecret* {.
     desc: "A file containing the hex-encoded 256 bit secret key to be used for verifying/generating JWT tokens"
-    name: "jwt-secret" .}: Option[string]
+    name: "jwt-secret" .}: Option[InputFile]
 
   # Testing
   stopAtEpoch* {.
@@ -148,3 +152,6 @@ template loadJwtSecret*(
     config: LightClientConf,
     allowCreate: bool): Option[seq[byte]] =
   rng.loadJwtSecret(string(config.dataDir), config.jwtSecret, allowCreate)
+
+proc engineApiUrls*(config: LightClientConf): seq[EngineApiUrl] =
+  (config.elUrls & config.web3Urls).toFinalEngineApiUrls(config.jwtSecret)