feat: add keymanager to all validator processes (#502)

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 executors:
   ubuntu_vm:
     machine:
-      image: ubuntu-2204:2023.07.2
+      image: ubuntu-2204:current
 
 parameters:
   should-enable-check-latest-version-workflow:

.github/tests/blobber.yaml

@@ -2,7 +2,6 @@ participants:
   - el_client_type: geth
     el_client_image: ethpandaops/geth:master
     cl_client_type: lighthouse
-    cl_client_image: ethpandaops/lighthouse:sidecar-inclusion-proof-c6be31c
     blobber_enabled: true
     blobber_extra_params:
       - --proposal-action-frequency=1
@@ -11,7 +10,6 @@ participants:
   - el_client_type: geth
     el_client_image: ethpandaops/geth:master
     cl_client_type: lodestar
-    cl_client_image: ethpandaops/lodestar:blobs-inclproof-d5a5a47
     count: 1
 network_params:
   deneb_fork_epoch: 1

.github/tests/split-teku.yaml

@@ -1,8 +1,8 @@
 participants:
   - el_client_type: geth
     cl_client_type: teku
-    use_separate_validator_client: true
     validator_count: 0
+    use_separate_validator_client: true
   - el_client_type: nethermind
     cl_client_type: teku
     use_separate_validator_client: true

main.star

@@ -75,6 +75,14 @@ def run(plan, args={}):
         src=static_files.JWT_PATH_FILEPATH,
         name="jwt_file",
     )
+    keymanager_file = plan.upload_files(
+        src=static_files.KEYMANAGER_PATH_FILEPATH,
+        name="keymanager_file",
+    )
+    keymanager_p12_file = plan.upload_files(
+        src=static_files.KEYMANAGER_P12_PATH_FILEPATH,
+        name="keymanager_p12_file",
+    )
     plan.print("Read the prometheus, grafana templates")
 
     plan.print(
@@ -93,6 +101,8 @@ def run(plan, args={}):
         network_params,
         args_with_right_defaults.global_client_log_level,
         jwt_file,
+        keymanager_file,
+        keymanager_p12_file,
         persistent,
         xatu_sentry_params,
         global_tolerations,

src/cl/nimbus/nimbus_launcher.star

@@ -5,7 +5,7 @@ cl_client_context = import_module("../../cl/cl_client_context.star")
 cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star")
 node_metrics = import_module("../../node_metrics_info.star")
 constants = import_module("../../package_io/constants.star")
-
+validator_client_shared = import_module("../../validator_client/shared.star")
 #  ---------------------------------- Beacon client -------------------------------------
 # Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we
 #  have to to create it
@@ -15,6 +15,7 @@ BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
 BEACON_HTTP_PORT_ID = "http"
 BEACON_METRICS_PORT_ID = "metrics"
+VALIDATOR_HTTP_PORT_ID = "http-validator"
 
 # Port nums
 BEACON_DISCOVERY_PORT_NUM = 9000
@@ -135,6 +136,7 @@ def launch(
         plan,
         launcher.el_cl_genesis_data,
         launcher.jwt_file,
+        launcher.keymanager_file,
         launcher.network,
         image,
         beacon_service_name,
@@ -209,6 +211,7 @@ def get_beacon_config(
     plan,
     el_cl_genesis_data,
     jwt_file,
+    keymanager_file,
     network,
     image,
     service_name,
@@ -296,11 +299,13 @@ def get_beacon_config(
         + constants.CL_CLIENT_TYPE.nimbus
         + "-"
         + el_client_context.client_name,
+        "--keymanager",
+        "--keymanager-port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
+        "--keymanager-address=0.0.0.0",
+        "--keymanager-allow-origin=*",
+        "--keymanager-token-file=" + constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER,
     ]
 
-    if node_keystore_files != None and not use_separate_validator_client:
-        cmd.extend(validator_flags)
-
     if network not in constants.PUBLIC_NETWORKS:
         cmd.append(
             "--bootstrap-file="
@@ -325,10 +330,22 @@ def get_beacon_config(
         constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
         constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
     }
+    beacon_validator_used_ports = {}
+    beacon_validator_used_ports.update(BEACON_USED_PORTS)
     if node_keystore_files != None and not use_separate_validator_client:
+        validator_http_port_id_spec = shared_utils.new_port_spec(
+            validator_client_shared.VALIDATOR_HTTP_PORT_NUM,
+            shared_utils.TCP_PROTOCOL,
+            shared_utils.HTTP_APPLICATION_PROTOCOL,
+        )
+        beacon_validator_used_ports.update(
+            {VALIDATOR_HTTP_PORT_ID: validator_http_port_id_spec}
+        )
+        cmd.extend(validator_flags)
         files[
             VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS
         ] = node_keystore_files.files_artifact_uuid
+        files[constants.KEYMANAGER_MOUNT_PATH_ON_CLIENTS] = keymanager_file
 
     if persistent:
         files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
@@ -338,7 +355,7 @@ def get_beacon_config(
 
     return ServiceConfig(
         image=image,
-        ports=BEACON_USED_PORTS,
+        ports=beacon_validator_used_ports,
         cmd=cmd,
         files=files,
         private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER,
@@ -362,9 +379,10 @@ def get_beacon_config(
     )
 
 
-def new_nimbus_launcher(el_cl_genesis_data, jwt_file, network):
+def new_nimbus_launcher(el_cl_genesis_data, jwt_file, network, keymanager_file):
     return struct(
         el_cl_genesis_data=el_cl_genesis_data,
         jwt_file=jwt_file,
         network=network,
+        keymanager_file=keymanager_file,
     )

src/cl/teku/teku_launcher.star

@@ -4,9 +4,10 @@ cl_client_context = import_module("../../cl/cl_client_context.star")
 node_metrics = import_module("../../node_metrics_info.star")
 cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star")
 constants = import_module("../../package_io/constants.star")
+validator_client_shared = import_module("../../validator_client/shared.star")
+#  ---------------------------------- Beacon client -------------------------------------
 TEKU_BINARY_FILEPATH_IN_IMAGE = "/opt/teku/bin/teku"
 
-#  ---------------------------------- Beacon client -------------------------------------
 # The Docker container runs as the "teku" user so we can't write to root
 BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-beacon-data"
 
@@ -15,6 +16,7 @@ BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery"
 BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery"
 BEACON_HTTP_PORT_ID = "http"
 BEACON_METRICS_PORT_ID = "metrics"
+VALIDATOR_HTTP_PORT_ID = "http-validator"
 
 # Port nums
 BEACON_DISCOVERY_PORT_NUM = 9000
@@ -124,6 +126,8 @@ def launch(
         plan,
         launcher.el_cl_genesis_data,
         launcher.jwt_file,
+        launcher.keymanager_file,
+        launcher.keymanager_p12_file,
         launcher.network,
         image,
         beacon_service_name,
@@ -200,6 +204,8 @@ def get_beacon_config(
     plan,
     el_cl_genesis_data,
     jwt_file,
+    keymanager_file,
+    keymanager_p12_file,
     network,
     image,
     service_name,
@@ -290,11 +296,19 @@ def get_beacon_config(
         + constants.CL_CLIENT_TYPE.teku
         + "-"
         + el_client_context.client_name,
+        "--validator-api-enabled=true",
+        "--validator-api-host-allowlist=*",
+        "--validator-api-port={0}".format(
+            validator_client_shared.VALIDATOR_HTTP_PORT_NUM
+        ),
+        "--validator-api-interface=0.0.0.0",
+        "--validator-api-keystore-file="
+        + constants.KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER,
+        "--validator-api-keystore-password-file="
+        + constants.KEYMANAGER_MOUNT_PATH_ON_CONTAINER,
+        "--validator-api-docs-enabled=true",
     ]
 
-    if node_keystore_files != None and not use_separate_validator_client:
-        cmd.extend(validator_flags)
-
     if network not in constants.PUBLIC_NETWORKS:
         cmd.append(
             "--initial-state="
@@ -366,10 +380,23 @@ def get_beacon_config(
         constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
         constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,
     }
+    beacon_validator_used_ports = {}
+    beacon_validator_used_ports.update(BEACON_USED_PORTS)
     if node_keystore_files != None and not use_separate_validator_client:
+        validator_http_port_id_spec = shared_utils.new_port_spec(
+            validator_client_shared.VALIDATOR_HTTP_PORT_NUM,
+            shared_utils.TCP_PROTOCOL,
+            shared_utils.HTTP_APPLICATION_PROTOCOL,
+        )
+        beacon_validator_used_ports.update(
+            {VALIDATOR_HTTP_PORT_ID: validator_http_port_id_spec}
+        )
+        cmd.extend(validator_flags)
         files[
             VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER
         ] = node_keystore_files.files_artifact_uuid
+        files[constants.KEYMANAGER_MOUNT_PATH_ON_CLIENTS] = keymanager_file
+        files[constants.KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS] = keymanager_p12_file
 
     if persistent:
         files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory(
@@ -378,7 +405,7 @@ def get_beacon_config(
         )
     return ServiceConfig(
         image=image,
-        ports=BEACON_USED_PORTS,
+        ports=beacon_validator_used_ports,
         cmd=cmd,
         # entrypoint=ENTRYPOINT_ARGS,
         files=files,
@@ -403,7 +430,13 @@ def get_beacon_config(
     )
 
 
-def new_teku_launcher(el_cl_genesis_data, jwt_file, network):
+def new_teku_launcher(
+    el_cl_genesis_data, jwt_file, network, keymanager_file, keymanager_p12_file
+):
     return struct(
-        el_cl_genesis_data=el_cl_genesis_data, jwt_file=jwt_file, network=network
+        el_cl_genesis_data=el_cl_genesis_data,
+        jwt_file=jwt_file,
+        network=network,
+        keymanager_file=keymanager_file,
+        keymanager_p12_file=keymanager_p12_file,
     )

src/package_io/constants.star

@@ -55,6 +55,14 @@ GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER = (
 JWT_MOUNTPOINT_ON_CLIENTS = "/jwt"
 JWT_MOUNT_PATH_ON_CONTAINER = JWT_MOUNTPOINT_ON_CLIENTS + "/jwtsecret"
 
+KEYMANAGER_MOUNT_PATH_ON_CLIENTS = "/keymanager"
+KEYMANAGER_MOUNT_PATH_ON_CONTAINER = (
+    KEYMANAGER_MOUNT_PATH_ON_CLIENTS + "/keymanager.txt"
+)
+KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS = "/keymanager-p12"
+KEYMANAGER_P12_MOUNT_PATH_ON_CONTAINER = (
+    KEYMANAGER_P12_MOUNT_PATH_ON_CLIENTS + "/validator_keystore.p12"
+)
 
 GENESIS_FORK_VERSION = "0x10000038"
 BELLATRIX_FORK_VERSION = "0x30000038"

src/participant_network.star

@@ -68,6 +68,8 @@ def launch_participant_network(
     network_params,
     global_log_level,
     jwt_file,
+    keymanager_file,
+    keymanager_p12_file,
     persistent,
     xatu_sentry_params,
     global_tolerations,
@@ -523,7 +525,7 @@ def launch_participant_network(
         },
         constants.CL_CLIENT_TYPE.nimbus: {
             "launcher": nimbus.new_nimbus_launcher(
-                el_cl_data, jwt_file, network_params.network
+                el_cl_data, jwt_file, network_params.network, keymanager_file
             ),
             "launch_method": nimbus.launch,
         },
@@ -542,6 +544,8 @@ def launch_participant_network(
                 el_cl_data,
                 jwt_file,
                 network_params.network,
+                keymanager_file,
+                keymanager_p12_file,
             ),
             "launch_method": teku.launch,
         },
@@ -775,6 +779,8 @@ def launch_participant_network(
             launcher=validator_client.new_validator_client_launcher(
                 el_cl_genesis_data=el_cl_data
             ),
+            keymanager_file=keymanager_file,
+            keymanager_p12_file=keymanager_p12_file,
             service_name="vc-{0}-{1}-{2}".format(
                 index_str, validator_client_type, el_client_type
             ),
@@ -797,6 +803,8 @@ def launch_participant_network(
             participant_tolerations=participant.tolerations,
             global_tolerations=global_tolerations,
             node_selectors=node_selectors,
+            network=network_params.network,  # TODO: remove when deneb rebase is done
+            electra_fork_epoch=network_params.electra_fork_epoch,  # TODO: remove when deneb rebase is done
         )
         all_validator_client_contexts.append(validator_client_context)
 

src/static_files/static_files.star

@@ -68,5 +68,9 @@ CL_GENESIS_GENERATION_MNEMONICS_TEMPLATE_FILEPATH = (
 )
 
 JWT_PATH_FILEPATH = STATIC_FILES_DIRPATH + "/jwt/jwtsecret"
+KEYMANAGER_PATH_FILEPATH = STATIC_FILES_DIRPATH + "/keymanager/keymanager.txt"
+KEYMANAGER_P12_PATH_FILEPATH = (
+    STATIC_FILES_DIRPATH + "/keymanager/validator_keystore.p12"
+)
 
 SHADOWFORK_FILEPATH = "/network-configs/latest_block.json"

src/validator_client/lighthouse.star

@@ -32,6 +32,8 @@ def get_config(
     extra_labels,
     tolerations,
     node_selectors,
+    network,
+    electra_fork_epoch,
 ):
     log_level = input_parser.get_client_log_level_or_default(
         participant_log_level, global_log_level, VERBOSITY_LEVELS
@@ -60,6 +62,11 @@ def get_config(
         # "--enable-doppelganger-protection", // Disabled to not have to wait 2 epochs before validator can start
         # burn address - If unset, the validator will scream in its logs
         "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
+        "--http",
+        "--http-port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
+        "--http-address=0.0.0.0",
+        "--http-allow-origin=*",
+        "--unencrypted-http-transport",
         # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv
         "--metrics",
         "--metrics-address=0.0.0.0",
@@ -74,6 +81,9 @@ def get_config(
         + el_client_context.client_name,
     ]
 
+    if not (constants.NETWORK_NAME.verkle in network and electra_fork_epoch == None):
+        cmd.append("--produce-block-v3")
+
     if len(extra_params):
         cmd.extend([param for param in extra_params])
 

src/validator_client/lodestar.star

@@ -54,6 +54,11 @@ def get_config(
         "--keystoresDir=" + validator_keys_dirpath,
         "--secretsDir=" + validator_secrets_dirpath,
         "--suggestedFeeRecipient=" + constants.VALIDATING_REWARDS_ACCOUNT,
+        "--keymanager",
+        "--keymanager.authEnabled=true",
+        "--keymanager.port={0}".format(validator_client_shared.VALIDATOR_HTTP_PORT_NUM),
+        "--keymanager.address=0.0.0.0",
+        "--keymanager.cors=*",
         # vvvvvvvvvvvvvvvvvvv PROMETHEUS CONFIG vvvvvvvvvvvvvvvvvvvvv
         "--metrics",
         "--metrics.address=0.0.0.0",
@@ -65,6 +70,7 @@ def get_config(
         + cl_client_context.client_name
         + "-"
         + el_client_context.client_name,
+        "--useProduceBlockV3",
     ]
 
     if len(extra_params) > 0: