From 2283464b614b0ade4aa98fccd842e8e4b23e188a Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 9 Jan 2024 13:57:13 +0100 Subject: [PATCH] fix: guid fix for besu/teku/erigon/nimbus (#443) --- .github/tests/broken-docker-persistence.yaml | 7 ----- .github/tests/mix-persistence-k8s.yaml | 11 ++++---- .github/tests/mix-persistence.yaml | 13 ++++++---- .github/tests/split-nimbus.yaml | 6 +++++ README.md | 1 + src/cl/nimbus/nimbus_launcher.star | 13 ++++------ src/cl/teku/teku_launcher.star | 5 ++-- src/el/besu/besu_launcher.star | 3 ++- src/el/erigon/erigon_launcher.star | 27 ++++++++++---------- src/package_io/input_parser.star | 4 +-- 10 files changed, 46 insertions(+), 44 deletions(-) delete mode 100644 .github/tests/broken-docker-persistence.yaml diff --git a/.github/tests/broken-docker-persistence.yaml b/.github/tests/broken-docker-persistence.yaml deleted file mode 100644 index 96195aad7..000000000 --- a/.github/tests/broken-docker-persistence.yaml +++ /dev/null @@ -1,7 +0,0 @@ -participants: - - el_client_type: besu - cl_client_type: teku - - el_client_type: erigon - cl_client_type: teku -additional_services: [] -persistent: true diff --git a/.github/tests/mix-persistence-k8s.yaml b/.github/tests/mix-persistence-k8s.yaml index d0eb9cc0d..252af09b7 100644 --- a/.github/tests/mix-persistence-k8s.yaml +++ b/.github/tests/mix-persistence-k8s.yaml @@ -1,19 +1,18 @@ participants: - el_client_type: geth - cl_client_type: prysm + cl_client_type: teku + cl_split_mode_enabled: true - el_client_type: nethermind + cl_client_type: prysm + - el_client_type: erigon cl_client_type: nimbus + cl_client_image: ethpandaops/nimbus:unstable cl_split_mode_enabled: true - el_client_type: besu cl_client_type: lighthouse - el_client_type: reth cl_client_type: lodestar - el_client_type: ethereumjs - cl_client_type: teku - cl_split_mode_enabled: true - - el_client_type: erigon - cl_client_type: teku - - el_client_type: besu cl_client_type: nimbus additional_services: [] persistent: true diff --git a/.github/tests/mix-persistence.yaml b/.github/tests/mix-persistence.yaml index 1b01666a1..252af09b7 100644 --- a/.github/tests/mix-persistence.yaml +++ b/.github/tests/mix-persistence.yaml @@ -1,15 +1,18 @@ participants: - el_client_type: geth + cl_client_type: teku + cl_split_mode_enabled: true + - el_client_type: nethermind cl_client_type: prysm - - el_client_type: geth + - el_client_type: erigon cl_client_type: nimbus + cl_client_image: ethpandaops/nimbus:unstable cl_split_mode_enabled: true - - el_client_type: nethermind + - el_client_type: besu cl_client_type: lighthouse - el_client_type: reth cl_client_type: lodestar - el_client_type: ethereumjs - cl_client_type: lighthouse -additional_services: - - dora + cl_client_type: nimbus +additional_services: [] persistent: true diff --git a/.github/tests/split-nimbus.yaml b/.github/tests/split-nimbus.yaml index b7c14a4c0..d44805ff4 100644 --- a/.github/tests/split-nimbus.yaml +++ b/.github/tests/split-nimbus.yaml @@ -1,21 +1,27 @@ participants: - el_client_type: geth cl_client_type: nimbus + cl_client_image: ethpandaops/nimbus:unstable cl_split_mode_enabled: true validator_count: 0 - el_client_type: nethermind cl_client_type: nimbus cl_split_mode_enabled: true + cl_client_image: ethpandaops/nimbus:unstable - el_client_type: erigon cl_client_type: nimbus cl_split_mode_enabled: true + cl_client_image: ethpandaops/nimbus:unstable - el_client_type: besu cl_client_type: nimbus cl_split_mode_enabled: true + cl_client_image: ethpandaops/nimbus:unstable - el_client_type: reth cl_client_type: nimbus cl_split_mode_enabled: true + cl_client_image: ethpandaops/nimbus:unstable - el_client_type: ethereumjs cl_client_type: nimbus cl_split_mode_enabled: true + cl_client_image: ethpandaops/nimbus:unstable additional_services: [] diff --git a/README.md b/README.md index 09eae396a..c7e79d2c2 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ participants: # A list of optional extra params that will be passed to the CL to run separate Beacon and validator nodes # Only possible for nimbus or teku + # Please note that in order to get it to work with Nimbus, you have to use `ethpandaops/nimbus:unstable` as the image (default upstream image does not yet support this out of the box) # Defaults to false cl_split_mode_enabled: false diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index afc69792a..0c607b2a7 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -7,6 +7,9 @@ node_metrics = import_module("../../node_metrics_info.star") constants = import_module("../../package_io/constants.star") # ---------------------------------- Beacon client ------------------------------------- +# Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we +# have to to create it +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/nimbus/beacon-data" # Port IDs BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery" BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery" @@ -29,7 +32,7 @@ DEFAULT_BEACON_IMAGE_ENTRYPOINT = ["nimbus_beacon_node"] BEACON_METRICS_PATH = "/metrics" # ---------------------------------- Validator client ------------------------------------- -VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" +VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/nimbus/validator-keys" VALIDATOR_HTTP_PORT_ID = "http" VALIDATOR_METRICS_PORT_ID = "metrics" VALIDATOR_HTTP_PORT_NUM = 5042 @@ -49,13 +52,6 @@ DEFAULT_VALIDATOR_IMAGE_ENTRYPOINT = ["nimbus_validator_client"] VALIDATOR_METRICS_PATH = "/metrics" # ---------------------------------- Genesis Files ---------------------------------- -# Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we -# have to to create it -BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/nimbus/beacon-data" -# Nimbus wants the data dir to have these perms -CONSENSUS_DATA_DIR_PERMS_STR = "0700" - - # Nimbus needs write access to the validator keys/secrets directories, and b/c the module container runs as root # while the Nimbus container does not, we can't just point the Nimbus binary to the paths in the shared dir because # it won't be able to open them. To get around this, we copy the validator keys/secrets to a path inside the Nimbus @@ -431,6 +427,7 @@ def get_beacon_config( el_client_context.client_name, extra_labels, ), + user=User(uid=0, gid=0), ) diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index 71608ea33..ea68cb142 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -8,7 +8,7 @@ TEKU_BINARY_FILEPATH_IN_IMAGE = "/opt/teku/bin/teku" # ---------------------------------- Beacon client ------------------------------------- # The Docker container runs as the "teku" user so we can't write to root -BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-beacon-data" +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-beacon-data" # Port IDs BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery" @@ -31,7 +31,7 @@ BEACON_METRICS_PATH = "/metrics" # ---------------------------------- Validator client ------------------------------------- # These will get mounted as root and Teku needs directory write permissions, so we'll copy this # into the Teku user's home directory to get around it -VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-validator-data" +VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/teku/teku-validator-data" VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" @@ -438,6 +438,7 @@ def get_beacon_config( el_client_context.client_name, extra_labels, ), + user=User(uid=0, gid=0), ) diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star index e39a36374..b589436d9 100644 --- a/src/el/besu/besu_launcher.star +++ b/src/el/besu/besu_launcher.star @@ -5,7 +5,7 @@ el_admin_node_info = import_module("../../el/el_admin_node_info.star") node_metrics = import_module("../../node_metrics_info.star") constants = import_module("../../package_io/constants.star") # The dirpath of the execution data directory on the client container -EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/opt/besu/execution-data" +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/besu/execution-data" METRICS_PATH = "/metrics" @@ -254,6 +254,7 @@ def get_config( cl_client_name, extra_labels, ), + user=User(uid=0, gid=0), ) diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star index 3f47d1001..8645104f4 100644 --- a/src/el/erigon/erigon_launcher.star +++ b/src/el/erigon/erigon_launcher.star @@ -7,7 +7,7 @@ node_metrics = import_module("../../node_metrics_info.star") constants = import_module("../../package_io/constants.star") # The dirpath of the execution data directory on the client container -EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/home/erigon/execution-data" +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/erigon/execution-data" METRICS_PATH = "/metrics" @@ -193,8 +193,20 @@ def get_config( "--metrics", "--metrics.addr=0.0.0.0", "--metrics.port={0}".format(METRICS_PORT_NUM), + "--db.size.limit={0}MB".format(el_volume_size), ] + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, + } + + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + size=el_volume_size, + ) + if network == "kurtosis": if len(existing_el_clients) > 0: cmd.append( @@ -216,7 +228,6 @@ def get_config( ) ) elif network not in constants.PUBLIC_NETWORKS: - cmd.append("--db.size.limit=100GB") cmd.append( "--bootnodes=" + shared_utils.get_devnet_enodes( @@ -240,20 +251,9 @@ def get_config( command_arg_str = " && ".join(command_arg) else: cmd.append("--chain={0}".format(network)) - cmd.append("--db.size.limit=3TB") command_arg = cmd command_arg_str = " ".join(command_arg) - files = { - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file, - } - - if persistent: - files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( - persistent_key="data-{0}".format(service_name), - size=el_volume_size, - ) return ServiceConfig( image=image, ports=USED_PORTS, @@ -273,6 +273,7 @@ def get_config( cl_client_name, extra_labels, ), + user=User(uid=0, gid=0), ) diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 1d75e64f7..49fc429b7 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -6,7 +6,7 @@ genesis_constants = import_module( DEFAULT_EL_IMAGES = { "geth": "ethereum/client-go:latest", - "erigon": "ethpandaops/erigon:2.53.0", + "erigon": "ethpandaops/erigon:devel", "nethermind": "nethermind/nethermind:latest", "besu": "hyperledger/besu:latest", "reth": "ghcr.io/paradigmxyz/reth", @@ -16,7 +16,7 @@ DEFAULT_EL_IMAGES = { DEFAULT_CL_IMAGES = { "lighthouse": "sigp/lighthouse:latest", "teku": "consensys/teku:latest", - "nimbus": "ethpandaops/nimbus:unstable", + "nimbus": "statusim/nimbus-eth2:multiarch-latest", "prysm": "prysmaticlabs/prysm-beacon-chain:latest,prysmaticlabs/prysm-validator:latest", "lodestar": "chainsafe/lodestar:latest", }