diff --git a/.circleci/config.yml b/.circleci/config.yml index e883a3988..3c0057d13 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -109,6 +109,13 @@ jobs: - checkout - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-with-tools-mev.yaml)" + mix_persistence_k8s: + resource_class: xlarge + executor: ubuntu_vm + steps: + - checkout + - run: kurtosis run ${PWD} "$(cat ./.github/tests/mix-persistence-k8s.yaml)" + workflows: nightly_tests: when: << pipeline.parameters.should-enable-check-latest-version-workflow >> @@ -155,3 +162,10 @@ workflows: branches: ignore: - main + + - mix_persistence_k8s: + <<: *setup_kurtosis_k3s + filters: + branches: + ignore: + - main diff --git a/.github/tests/besu-persistence.yaml b/.github/tests/besu-persistence.yaml new file mode 100644 index 000000000..b84df9add --- /dev/null +++ b/.github/tests/besu-persistence.yaml @@ -0,0 +1,5 @@ +participants: + - el_client_type: besu + cl_client_type: lighthouse +additional_services: [] +persistent: true diff --git a/.github/tests/erigon-persistence.yaml b/.github/tests/erigon-persistence.yaml new file mode 100644 index 000000000..45dd0ba77 --- /dev/null +++ b/.github/tests/erigon-persistence.yaml @@ -0,0 +1,5 @@ +participants: + - el_client_type: erigon + cl_client_type: lighthouse +additional_services: [] +persistent: true diff --git a/.github/tests/mix-persistence-k8s.yaml b/.github/tests/mix-persistence-k8s.yaml new file mode 100644 index 000000000..d0eb9cc0d --- /dev/null +++ b/.github/tests/mix-persistence-k8s.yaml @@ -0,0 +1,19 @@ +participants: + - el_client_type: geth + cl_client_type: prysm + - el_client_type: nethermind + cl_client_type: nimbus + cl_split_mode_enabled: true + - el_client_type: besu + cl_client_type: lighthouse + - el_client_type: reth + cl_client_type: lodestar + - el_client_type: ethereumjs + cl_client_type: teku + cl_split_mode_enabled: true + - el_client_type: erigon + cl_client_type: teku + - el_client_type: besu + cl_client_type: nimbus +additional_services: [] +persistent: true diff --git a/.github/tests/mix-persistence.yaml b/.github/tests/mix-persistence.yaml new file mode 100644 index 000000000..1b01666a1 --- /dev/null +++ b/.github/tests/mix-persistence.yaml @@ -0,0 +1,15 @@ +participants: + - el_client_type: geth + cl_client_type: prysm + - el_client_type: geth + cl_client_type: nimbus + cl_split_mode_enabled: true + - el_client_type: nethermind + cl_client_type: lighthouse + - el_client_type: reth + cl_client_type: lodestar + - el_client_type: ethereumjs + cl_client_type: lighthouse +additional_services: + - dora +persistent: true diff --git a/.github/tests/teku-persistence.yaml b/.github/tests/teku-persistence.yaml new file mode 100644 index 000000000..dad2f9b4e --- /dev/null +++ b/.github/tests/teku-persistence.yaml @@ -0,0 +1,5 @@ +participants: + - el_client_type: geth + cl_client_type: teku +additional_services: [] +persistent: true diff --git a/.github/workflows/per-pr.yml b/.github/workflows/per-pr.yml index 89c6c5b9c..c7e5c7005 100644 --- a/.github/workflows/per-pr.yml +++ b/.github/workflows/per-pr.yml @@ -32,6 +32,7 @@ jobs: "./.github/tests/mev.yaml", "./.github/tests/mev-mock.yaml", "./.github/tests/mix-with-tools.yaml", + "./.github/tests/mix-persistence.yaml", "./network_params.yaml" ] runs-on: ubuntu-latest diff --git a/README.md b/README.md index cca15c1f0..d0218cdfd 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,7 @@ grafana_additional_dashboards: [] # Whether the environment should be persistent; this is WIP and is slowly being rolled out accross services # Note this requires Kurtosis greater than 0.85.49 to work +# Note Erigon, Besu, Teku persistence is not currently supported with docker. # Defaults to False persistent: False diff --git a/main.star b/main.star index 9ed7b8917..de75ccaf2 100644 --- a/main.star +++ b/main.star @@ -86,6 +86,7 @@ def run(plan, args={}): args_with_right_defaults.participants, network_params, args_with_right_defaults.global_client_log_level, + persistent, parallel_keystore_generation, ) diff --git a/src/cl/lighthouse/lighthouse_launcher.star b/src/cl/lighthouse/lighthouse_launcher.star index dd2e4eab1..a5f71167a 100644 --- a/src/cl/lighthouse/lighthouse_launcher.star +++ b/src/cl/lighthouse/lighthouse_launcher.star @@ -10,13 +10,11 @@ blobber_launcher = import_module("../../blobber/blobber_launcher.star") LIGHTHOUSE_BINARY_COMMAND = "lighthouse" -VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" - RUST_BACKTRACE_ENVVAR_NAME = "RUST_BACKTRACE" RUST_FULL_BACKTRACE_KEYWORD = "full" # ---------------------------------- Beacon client ------------------------------------- -CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/consensus-data" +BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER = "/data/lighthouse/beacon-data" # Port IDs BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery" @@ -36,6 +34,7 @@ BEACON_MIN_MEMORY = 256 BEACON_MAX_MEMORY = 1024 # ---------------------------------- Validator client ------------------------------------- +VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/data/lighthouse/validator-keys" VALIDATOR_HTTP_PORT_ID = "http" VALIDATOR_METRICS_PORT_ID = "metrics" VALIDATOR_HTTP_PORT_NUM = 5042 @@ -121,10 +120,11 @@ def launch( extra_validator_params, extra_beacon_labels, extra_validator_labels, + persistent, split_mode_enabled=False, ): - beacon_node_service_name = "{0}".format(service_name) - validator_node_service_name = "{0}-{1}".format( + beacon_service_name = "{0}".format(service_name) + validator_service_name = "{0}-{1}".format( service_name, VALIDATOR_SUFFIX_SERVICE_NAME ) @@ -141,6 +141,7 @@ def launch( beacon_config = get_beacon_config( launcher.el_cl_genesis_data, image, + beacon_service_name, bootnode_contexts, el_client_context, log_level, @@ -152,9 +153,10 @@ def launch( snooper_engine_context, extra_beacon_params, extra_beacon_labels, + persistent, ) - beacon_service = plan.add_service(beacon_node_service_name, beacon_config) + beacon_service = plan.add_service(beacon_service_name, beacon_config) beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID] beacon_http_url = "http://{0}:{1}".format( beacon_service.ip_address, beacon_http_port.number @@ -162,7 +164,7 @@ def launch( # Blobber config if blobber_enabled: - blobber_service_name = "{0}-{1}".format("blobber", beacon_node_service_name) + blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name) blobber_config = blobber_launcher.get_config( blobber_service_name, node_keystore_files, @@ -190,6 +192,7 @@ def launch( validator_config = get_validator_config( launcher.el_cl_genesis_data, image, + validator_service_name, log_level, beacon_http_url, el_client_context, @@ -198,14 +201,12 @@ def launch( v_max_cpu, v_min_mem, v_max_mem, - validator_node_service_name, extra_validator_params, extra_validator_labels, + persistent, ) - validator_service = plan.add_service( - validator_node_service_name, validator_config - ) + validator_service = plan.add_service(validator_service_name, validator_config) # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( @@ -218,7 +219,7 @@ def launch( }, ) response = plan.request( - recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name + recipe=beacon_node_identity_recipe, service_name=beacon_service_name ) beacon_node_enr = response["extract.enr"] beacon_multiaddr = response["extract.multiaddr"] @@ -229,7 +230,7 @@ def launch( beacon_service.ip_address, beacon_metrics_port.number ) beacon_node_metrics_info = node_metrics.new_node_metrics_info( - beacon_node_service_name, METRICS_PATH, beacon_metrics_url + beacon_service_name, METRICS_PATH, beacon_metrics_url ) nodes_metrics_info = [beacon_node_metrics_info] @@ -239,7 +240,7 @@ def launch( validator_service.ip_address, validator_metrics_port.number ) validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_node_service_name, METRICS_PATH, validator_metrics_url + validator_service_name, METRICS_PATH, validator_metrics_url ) nodes_metrics_info.append(validator_node_metrics_info) @@ -249,8 +250,8 @@ def launch( beacon_service.ip_address, BEACON_HTTP_PORT_NUM, nodes_metrics_info, - beacon_node_service_name, - validator_node_service_name, + beacon_service_name, + validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -264,6 +265,7 @@ def launch( def get_beacon_config( el_cl_genesis_data, image, + service_name, boot_cl_client_ctxs, el_client_context, log_level, @@ -275,6 +277,7 @@ def get_beacon_config( snooper_engine_context, extra_params, extra_labels, + persistent, ): # If snooper is enabled use the snooper engine context, otherwise use the execution client context if snooper_enabled: @@ -300,7 +303,7 @@ def get_beacon_config( LIGHTHOUSE_BINARY_COMMAND, "beacon_node", "--debug-level=" + log_level, - "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER, + "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER, "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER, # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv "--disable-enr-auto-update", @@ -358,14 +361,19 @@ def get_beacon_config( recipe = GetHttpRequestRecipe( endpoint="/eth/v1/node/identity", port_id=BEACON_HTTP_PORT_ID ) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid + } + if persistent: + files[BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) return ServiceConfig( image=image, ports=BEACON_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid - }, + files=files, env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD}, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( @@ -388,6 +396,7 @@ def get_beacon_config( def get_validator_config( el_cl_genesis_data, image, + service_name, log_level, beacon_client_http_url, el_client_context, @@ -396,9 +405,9 @@ def get_validator_config( v_max_cpu, v_min_mem, v_max_mem, - validator_node_service_name, extra_params, extra_labels, + persistent, ): validator_keys_dirpath = shared_utils.path_join( VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS, @@ -442,14 +451,16 @@ def get_validator_config( if len(extra_params): cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, + } + return ServiceConfig( image=image, ports=VALIDATOR_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, - }, + files=files, env_vars={RUST_BACKTRACE_ENVVAR_NAME: RUST_FULL_BACKTRACE_KEYWORD}, min_cpu=v_min_cpu, max_cpu=v_max_cpu, diff --git a/src/cl/lodestar/lodestar_launcher.star b/src/cl/lodestar/lodestar_launcher.star index 79d9d4a0d..b9484bf7b 100644 --- a/src/cl/lodestar/lodestar_launcher.star +++ b/src/cl/lodestar/lodestar_launcher.star @@ -7,7 +7,7 @@ blobber_launcher = import_module("../../blobber/blobber_launcher.star") constants = import_module("../../package_io/constants.star") # ---------------------------------- Beacon client ------------------------------------- -CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/consensus-data" +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/lodestar/beacon-data" # Port IDs TCP_DISCOVERY_PORT_ID = "tcp-discovery" UDP_DISCOVERY_PORT_ID = "udp-discovery" @@ -28,6 +28,7 @@ BEACON_MAX_MEMORY = 1024 # ---------------------------------- Validator client ------------------------------------- VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" +VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/lodestar/validator-data" # The min/max CPU/memory that the validator node can use VALIDATOR_MIN_CPU = 50 VALIDATOR_MAX_CPU = 300 @@ -97,10 +98,11 @@ def launch( extra_validator_params, extra_beacon_labels, extra_validator_labels, + persistent, split_mode_enabled=False, ): - beacon_node_service_name = "{0}".format(service_name) - validator_node_service_name = "{0}-{1}".format( + beacon_service_name = "{0}".format(service_name) + validator_service_name = "{0}-{1}".format( service_name, VALIDATOR_SUFFIX_SERVICE_NAME ) log_level = input_parser.get_client_log_level_or_default( @@ -116,6 +118,7 @@ def launch( beacon_config = get_beacon_config( launcher.el_cl_genesis_data, image, + beacon_service_name, bootnode_contexts, el_client_context, log_level, @@ -127,9 +130,10 @@ def launch( snooper_engine_context, extra_beacon_params, extra_beacon_labels, + persistent, ) - beacon_service = plan.add_service(beacon_node_service_name, beacon_config) + beacon_service = plan.add_service(beacon_service_name, beacon_config) beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID] @@ -139,7 +143,7 @@ def launch( # Blobber config if blobber_enabled: - blobber_service_name = "{0}-{1}".format("blobber", beacon_node_service_name) + blobber_service_name = "{0}-{1}".format("blobber", beacon_service_name) blobber_config = blobber_launcher.get_config( blobber_service_name, node_keystore_files, @@ -163,9 +167,9 @@ def launch( v_min_mem = int(v_min_mem) if int(v_min_mem) > 0 else VALIDATOR_MIN_MEMORY v_max_mem = int(v_max_mem) if int(v_max_mem) > 0 else VALIDATOR_MAX_MEMORY validator_config = get_validator_config( - validator_node_service_name, launcher.el_cl_genesis_data, image, + validator_service_name, log_level, beacon_http_url, el_client_context, @@ -174,12 +178,12 @@ def launch( v_max_cpu, v_min_mem, v_max_mem, - validator_node_service_name, extra_validator_params, extra_validator_labels, + persistent, ) - plan.add_service(validator_node_service_name, validator_config) + plan.add_service(validator_service_name, validator_config) # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module @@ -193,7 +197,7 @@ def launch( }, ) response = plan.request( - recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name + recipe=beacon_node_identity_recipe, service_name=beacon_service_name ) beacon_node_enr = response["extract.enr"] beacon_multiaddr = response["extract.multiaddr"] @@ -215,8 +219,8 @@ def launch( beacon_service.ip_address, HTTP_PORT_NUM, nodes_metrics_info, - beacon_node_service_name, - validator_node_service_name, + beacon_service_name, + validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -230,6 +234,7 @@ def launch( def get_beacon_config( el_cl_genesis_data, image, + service_name, bootnode_contexts, el_client_context, log_level, @@ -241,6 +246,7 @@ def get_beacon_config( snooper_engine_context, extra_params, extra_labels, + persistent, ): el_client_rpc_url_str = "http://{0}:{1}".format( el_client_context.ip_addr, @@ -264,7 +270,7 @@ def get_beacon_config( "--logLevel=" + log_level, "--port={0}".format(DISCOVERY_PORT_NUM), "--discoveryPort={0}".format(DISCOVERY_PORT_NUM), - "--dataDir=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, + "--dataDir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--paramsFile=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", @@ -306,14 +312,19 @@ def get_beacon_config( if len(extra_params) > 0: # this is a repeated, we convert it into Starlark cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid + } + if persistent: + files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name) + ) return ServiceConfig( image=image, ports=BEACON_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID @@ -333,9 +344,9 @@ def get_beacon_config( def get_validator_config( - service_name, el_cl_genesis_data, image, + service_name, log_level, beacon_client_http_url, el_client_context, @@ -344,12 +355,12 @@ def get_validator_config( v_max_cpu, v_min_mem, v_max_mem, - validator_node_service_name, extra_params, extra_labels, + persistent, ): root_dirpath = shared_utils.path_join( - CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, service_name ) validator_keys_dirpath = shared_utils.path_join( @@ -365,7 +376,7 @@ def get_validator_config( cmd = [ "validator", "--logLevel=" + log_level, - "--dataDir=" + root_dirpath, + # "--dataDir=" + root_dirpath, "--paramsFile=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", @@ -388,14 +399,16 @@ def get_validator_config( # this is a repeated, we convert it into Starlark cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + } + return ServiceConfig( image=image, ports=VALIDATOR_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=v_min_cpu, max_cpu=v_max_cpu, diff --git a/src/cl/nimbus/nimbus_launcher.star b/src/cl/nimbus/nimbus_launcher.star index 0e347c220..91672d9cb 100644 --- a/src/cl/nimbus/nimbus_launcher.star +++ b/src/cl/nimbus/nimbus_launcher.star @@ -51,7 +51,7 @@ VALIDATOR_METRICS_PATH = "/metrics" # Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we # have to to create it -CONSENSUS_DATA_DIRPATH_IN_SERVICE_CONTAINER = "$HOME/consensus-data" +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/beacon-data" # Nimbus wants the data dir to have these perms CONSENSUS_DATA_DIR_PERMS_STR = "0700" @@ -137,10 +137,11 @@ def launch( extra_validator_params, extra_beacon_labels, extra_validator_labels, + persistent, split_mode_enabled, ): - beacon_node_service_name = "{0}".format(service_name) - validator_node_service_name = "{0}-{1}".format( + beacon_service_name = "{0}".format(service_name) + validator_service_name = "{0}-{1}".format( service_name, VALIDATOR_SUFFIX_SERVICE_NAME ) @@ -156,6 +157,7 @@ def launch( beacon_config = get_beacon_config( launcher.el_cl_genesis_data, image, + beacon_service_name, bootnode_contexts, el_client_context, log_level, @@ -164,15 +166,15 @@ def launch( bn_max_cpu, bn_min_mem, bn_max_mem, - beacon_node_service_name, snooper_enabled, snooper_engine_context, extra_beacon_params, extra_beacon_labels, split_mode_enabled, + persistent, ) - beacon_service = plan.add_service(beacon_node_service_name, beacon_config) + beacon_service = plan.add_service(beacon_service_name, beacon_config) beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID] beacon_metrics_port = beacon_service.ports[BEACON_METRICS_PORT_ID] beacon_http_url = "http://{0}:{1}".format( @@ -214,6 +216,7 @@ def launch( validator_config = get_validator_config( launcher.el_cl_genesis_data, image, + validator_service_name, log_level, beacon_http_url, el_client_context, @@ -222,14 +225,12 @@ def launch( v_max_cpu, v_min_mem, v_max_mem, - validator_node_service_name, extra_validator_params, extra_validator_labels, + persistent, ) - validator_service = plan.add_service( - validator_node_service_name, validator_config - ) + validator_service = plan.add_service(validator_service_name, validator_config) if validator_service: validator_metrics_port = validator_service.ports[VALIDATOR_METRICS_PORT_ID] @@ -237,7 +238,7 @@ def launch( validator_service.ip_address, validator_metrics_port.number ) validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_node_service_name, VALIDATOR_METRICS_PATH, validator_metrics_url + validator_service_name, VALIDATOR_METRICS_PATH, validator_metrics_url ) nodes_metrics_info.append(validator_node_metrics_info) @@ -247,8 +248,8 @@ def launch( beacon_service.ip_address, BEACON_HTTP_PORT_NUM, nodes_metrics_info, - beacon_node_service_name, - validator_node_service_name, + beacon_service_name, + validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -262,6 +263,7 @@ def launch( def get_beacon_config( el_cl_genesis_data, image, + service_name, bootnode_contexts, el_client_context, log_level, @@ -270,12 +272,12 @@ def get_beacon_config( bn_max_cpu, bn_min_mem, bn_max_mem, - beacon_node_service_name, snooper_enabled, snooper_engine_context, extra_params, extra_labels, split_mode_enabled, + persistent, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -306,7 +308,7 @@ def get_beacon_config( "--udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), "--tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM), "--network=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER, - "--data-dir=" + CONSENSUS_DATA_DIRPATH_IN_SERVICE_CONTAINER, + "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--web3-url=" + EXECUTION_ENGINE_ENDPOINT, "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER, "--enr-auto-update=false", @@ -364,6 +366,11 @@ def get_beacon_config( VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS ] = node_keystore_files.files_artifact_uuid + if persistent: + files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name) + ) + return ServiceConfig( image=image, ports=BEACON_USED_PORTS, @@ -390,6 +397,7 @@ def get_beacon_config( def get_validator_config( el_cl_genesis_data, image, + service_name, log_level, beacon_http_url, el_client_context, @@ -398,9 +406,9 @@ def get_validator_config( v_max_cpu, v_min_mem, v_max_mem, - validator_node_service_name, extra_params, extra_labels, + persistent, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -432,14 +440,16 @@ def get_validator_config( if len(extra_params) > 0: cmd.extend([param for param in extra_params if param != "--split=true"]) + files = { + VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, + } + return ServiceConfig( image=image, ports=VALIDATOR_USED_PORTS, cmd=cmd, entrypoint=DEFAULT_VALIDATOR_IMAGE_ENTRYPOINT, - files={ - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=v_min_cpu, max_cpu=v_max_cpu, diff --git a/src/cl/prysm/prysm_launcher.star b/src/cl/prysm/prysm_launcher.star index 90a9c0d6b..e9b0266e0 100644 --- a/src/cl/prysm/prysm_launcher.star +++ b/src/cl/prysm/prysm_launcher.star @@ -8,7 +8,7 @@ IMAGE_SEPARATOR_DELIMITER = "," EXPECTED_NUM_IMAGES = 2 # ---------------------------------- Beacon client ------------------------------------- -CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/consensus-data" +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/prysm/beacon-data/" # Port IDs TCP_DISCOVERY_PORT_ID = "tcp-discovery" @@ -31,6 +31,7 @@ BEACON_MIN_MEMORY = 256 BEACON_MAX_MEMORY = 1024 # ---------------------------------- Validator client ------------------------------------- +VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/prysm/validator-data/" VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER = "/prysm-password" @@ -109,6 +110,7 @@ def launch( extra_validator_params, extra_beacon_labels, extra_validator_labels, + persistent, split_mode_enabled=False, ): split_images = images.split(IMAGE_SEPARATOR_DELIMITER) @@ -126,8 +128,8 @@ def launch( if validator_image.strip() == "": fail("An empty validator image was provided") - beacon_node_service_name = "{0}".format(service_name) - validator_node_service_name = "{0}-{1}".format( + beacon_service_name = "{0}".format(service_name) + validator_service_name = "{0}-{1}".format( service_name, VALIDATOR_SUFFIX_SERVICE_NAME ) log_level = input_parser.get_client_log_level_or_default( @@ -142,6 +144,7 @@ def launch( beacon_config = get_beacon_config( launcher.el_cl_genesis_data, beacon_image, + beacon_service_name, bootnode_contexts, el_client_context, log_level, @@ -153,9 +156,10 @@ def launch( snooper_engine_context, extra_beacon_params, extra_beacon_labels, + persistent, ) - beacon_service = plan.add_service(beacon_node_service_name, beacon_config) + beacon_service = plan.add_service(beacon_service_name, beacon_config) beacon_http_port = beacon_service.ports[BEACON_HTTP_PORT_ID] @@ -172,7 +176,7 @@ def launch( validator_config = get_validator_config( launcher.el_cl_genesis_data, validator_image, - validator_node_service_name, + validator_service_name, log_level, beacon_rpc_endpoint, beacon_http_endpoint, @@ -186,11 +190,10 @@ def launch( extra_validator_labels, launcher.prysm_password_relative_filepath, launcher.prysm_password_artifact_uuid, + persistent, ) - validator_service = plan.add_service( - validator_node_service_name, validator_config - ) + validator_service = plan.add_service(validator_service_name, validator_config) # TODO(old) add validator availability using the validator API: https://ethereum.github.io/beacon-APIs/?urls.primaryName=v1#/ValidatorRequiredApi | from eth2-merge-kurtosis-module beacon_node_identity_recipe = GetHttpRequestRecipe( @@ -203,7 +206,7 @@ def launch( }, ) response = plan.request( - recipe=beacon_node_identity_recipe, service_name=beacon_node_service_name + recipe=beacon_node_identity_recipe, service_name=beacon_service_name ) beacon_node_enr = response["extract.enr"] beacon_multiaddr = response["extract.multiaddr"] @@ -214,7 +217,7 @@ def launch( beacon_service.ip_address, beacon_metrics_port.number ) beacon_node_metrics_info = node_metrics.new_node_metrics_info( - beacon_node_service_name, METRICS_PATH, beacon_metrics_url + beacon_service_name, METRICS_PATH, beacon_metrics_url ) nodes_metrics_info = [beacon_node_metrics_info] @@ -224,7 +227,7 @@ def launch( validator_service.ip_address, validator_metrics_port.number ) validator_node_metrics_info = node_metrics.new_node_metrics_info( - validator_node_service_name, METRICS_PATH, validator_metrics_url + validator_service_name, METRICS_PATH, validator_metrics_url ) nodes_metrics_info.append(validator_node_metrics_info) @@ -234,8 +237,8 @@ def launch( beacon_service.ip_address, HTTP_PORT_NUM, nodes_metrics_info, - beacon_node_service_name, - validator_node_service_name, + beacon_service_name, + validator_service_name, beacon_multiaddr, beacon_peer_id, snooper_enabled, @@ -249,6 +252,7 @@ def launch( def get_beacon_config( el_cl_genesis_data, beacon_image, + service_name, bootnode_contexts, el_client_context, log_level, @@ -260,6 +264,7 @@ def get_beacon_config( snooper_engine_context, extra_params, extra_labels, + persistent, ): # If snooper is enabled use the snooper engine context, otherwise use the execution client context if snooper_enabled: @@ -275,7 +280,7 @@ def get_beacon_config( cmd = [ "--accept-terms-of-use=true", # it's mandatory in order to run the node - "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, + "--datadir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--chain-config-file=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", @@ -316,13 +321,20 @@ def get_beacon_config( # we do the for loop as otherwise its a proto repeated array cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid + } + + if persistent: + files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name) + ) + return ServiceConfig( image=beacon_image, ports=BEACON_NODE_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, ready_conditions=cl_node_ready_conditions.get_ready_conditions( BEACON_HTTP_PORT_ID @@ -358,6 +370,7 @@ def get_validator_config( extra_labels, prysm_password_relative_filepath, prysm_password_artifact_uuid, + persistent, ): validator_keys_dirpath = shared_utils.path_join( VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER, @@ -377,7 +390,7 @@ def get_validator_config( "--beacon-rpc-provider=" + beacon_rpc_endpoint, "--wallet-dir=" + validator_keys_dirpath, "--wallet-password-file=" + validator_secrets_dirpath, - "--datadir=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, + # "--datadir=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--monitoring-port={0}".format(VALIDATOR_MONITORING_PORT_NUM), "--verbosity=" + log_level, "--suggested-fee-recipient=" + constants.VALIDATING_REWARDS_ACCOUNT, @@ -396,16 +409,21 @@ def get_validator_config( if len(extra_params) > 0: # we do the for loop as otherwise its a proto repeated array cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, + PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: prysm_password_artifact_uuid, + } + if persistent: + files[VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name) + ) return ServiceConfig( image=validator_image, ports=VALIDATOR_NODE_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: node_keystore_files.files_artifact_uuid, - PRYSM_PASSWORD_MOUNT_DIRPATH_ON_SERVICE_CONTAINER: prysm_password_artifact_uuid, - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=v_min_cpu, max_cpu=v_max_cpu, diff --git a/src/cl/teku/teku_launcher.star b/src/cl/teku/teku_launcher.star index e5c3bddd8..6b2b42a26 100644 --- a/src/cl/teku/teku_launcher.star +++ b/src/cl/teku/teku_launcher.star @@ -7,14 +7,10 @@ cl_node_ready_conditions = import_module("../../cl/cl_node_ready_conditions.star constants = import_module("../../package_io/constants.star") TEKU_BINARY_FILEPATH_IN_IMAGE = "/opt/teku/bin/teku" +# ---------------------------------- Beacon client ------------------------------------- # The Docker container runs as the "teku" user so we can't write to root -CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/consensus-data" +BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-beacon-data" -# These will get mounted as root and Teku needs directory write permissions, so we'll copy this -# into the Teku user's home directory to get around it -VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" - -# ---------------------------------- Beacon client ------------------------------------- # Port IDs BEACON_TCP_DISCOVERY_PORT_ID = "tcp-discovery" BEACON_UDP_DISCOVERY_PORT_ID = "udp-discovery" @@ -34,6 +30,12 @@ BEACON_MAX_MEMORY = 2048 BEACON_METRICS_PATH = "/metrics" # ---------------------------------- Validator client ------------------------------------- +# These will get mounted as root and Teku needs directory write permissions, so we'll copy this +# into the Teku user's home directory to get around it +VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/opt/teku/teku-validator-data" + +VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER = "/validator-keys" + VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS = "/validator-keys" VALIDATOR_HTTP_PORT_ID = "http" VALIDATOR_METRICS_PORT_ID = "metrics" @@ -122,6 +124,7 @@ def launch( extra_validator_params, extra_beacon_labels, extra_validator_labels, + persistent, split_mode_enabled, ): beacon_service_name = "{0}".format(service_name) @@ -144,6 +147,7 @@ def launch( config = get_beacon_config( launcher.el_cl_genesis_data, image, + beacon_service_name, bootnode_context, el_client_context, log_level, @@ -154,10 +158,10 @@ def launch( bn_max_mem, snooper_enabled, snooper_engine_context, - beacon_service_name, extra_beacon_params, extra_beacon_labels, split_mode_enabled, + persistent, ) beacon_service = plan.add_service(service_name, config) @@ -204,6 +208,7 @@ def launch( validator_config = get_validator_config( launcher.el_cl_genesis_data, image, + validator_service_name, log_level, beacon_http_url, el_client_context, @@ -215,6 +220,7 @@ def launch( validator_service_name, extra_validator_params, extra_validator_labels, + persistent, ) validator_service = plan.add_service(validator_service_name, validator_config) @@ -250,6 +256,7 @@ def launch( def get_beacon_config( el_cl_genesis_data, image, + service_name, bootnode_contexts, el_client_context, log_level, @@ -260,10 +267,10 @@ def get_beacon_config( bn_max_mem, snooper_enabled, snooper_engine_context, - service_name, extra_params, extra_labels, split_mode_enabled, + persistent, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -296,7 +303,7 @@ def get_beacon_config( "--initial-state=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/genesis.ssz", - "--data-path=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, + "--data-path=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--data-storage-mode={0}".format( "ARCHIVE" if constants.ARCHIVE_MODE else "PRUNE" ), @@ -366,7 +373,11 @@ def get_beacon_config( files[ VALIDATOR_KEYS_DIRPATH_ON_SERVICE_CONTAINER ] = node_keystore_files.files_artifact_uuid - cmd_str = " ".join(cmd) + + if persistent: + files[BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name) + ) return ServiceConfig( image=image, ports=BEACON_USED_PORTS, @@ -394,6 +405,7 @@ def get_beacon_config( def get_validator_config( el_cl_genesis_data, image, + service_name, log_level, beacon_http_url, el_client_context, @@ -405,6 +417,7 @@ def get_validator_config( validator_service_name, extra_params, extra_labels, + persistent, ): validator_keys_dirpath = "" validator_secrets_dirpath = "" @@ -424,8 +437,8 @@ def get_validator_config( "--network=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml", - "--data-path=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, - "--data-validator-path=" + CONSENSUS_DATA_DIRPATH_ON_SERVICE_CONTAINER, + # "--data-path=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, + # "--data-validator-path=" + VALIDATOR_DATA_DIRPATH_ON_SERVICE_CONTAINER, "--beacon-node-api-endpoint=" + beacon_http_url, "--validator-keys={0}:{1}".format( validator_keys_dirpath, @@ -447,14 +460,16 @@ def get_validator_config( if len(extra_params) > 0: cmd.extend([param for param in extra_params if param != "--split=true"]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, + } + return ServiceConfig( image=image, ports=VALIDATOR_USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - VALIDATOR_KEYS_MOUNTPOINT_ON_CLIENTS: node_keystore_files.files_artifact_uuid, - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=v_min_cpu, max_cpu=v_max_cpu, diff --git a/src/el/besu/besu_launcher.star b/src/el/besu/besu_launcher.star index 7d8b08d3b..2e8d9c515 100644 --- a/src/el/besu/besu_launcher.star +++ b/src/el/besu/besu_launcher.star @@ -75,6 +75,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, BESU_LOG_LEVELS @@ -91,6 +92,7 @@ def launch( launcher.network_id, launcher.el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -101,6 +103,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ) service = plan.add_service(service_name, config) @@ -129,6 +132,7 @@ def get_config( network_id, el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -139,6 +143,7 @@ def get_config( extra_params, extra_env_vars, extra_labels, + persistent, ): cmd = [ "besu", @@ -190,13 +195,20 @@ def get_config( cmd_str = " ".join(cmd) extra_env_vars = extra_env_vars | JAVA_OPTS + + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + } + + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) return ServiceConfig( image=image, ports=USED_PORTS, cmd=[cmd_str], - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - }, + files=files, env_vars=extra_env_vars, entrypoint=ENTRYPOINT_ARGS, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, diff --git a/src/el/erigon/erigon_launcher.star b/src/el/erigon/erigon_launcher.star index 128f2f4bd..1892be186 100644 --- a/src/el/erigon/erigon_launcher.star +++ b/src/el/erigon/erigon_launcher.star @@ -76,6 +76,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, ERIGON_LOG_LEVELS @@ -92,6 +93,7 @@ def launch( launcher.network_id, launcher.el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -102,6 +104,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ) service = plan.add_service(service_name, config) @@ -132,6 +135,7 @@ def get_config( network_id, el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, verbosity_level, @@ -142,6 +146,7 @@ def get_config( extra_params, extra_env_vars, extra_labels, + persistent, ): network_id = network_id @@ -201,14 +206,19 @@ def get_config( command_arg = [init_datadir_cmd_str, " ".join(cmd)] command_arg_str = " && ".join(command_arg) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + } + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) return ServiceConfig( image=image, ports=USED_PORTS, cmd=[command_arg_str], - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - }, + files=files, entrypoint=ENTRYPOINT_ARGS, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, diff --git a/src/el/ethereumjs/ethereumjs_launcher.star b/src/el/ethereumjs/ethereumjs_launcher.star index 9edd4d5d2..f61a05b0d 100644 --- a/src/el/ethereumjs/ethereumjs_launcher.star +++ b/src/el/ethereumjs/ethereumjs_launcher.star @@ -31,7 +31,7 @@ METRICS_PORT_ID = "metrics" METRICS_PATH = "/metrics" # The dirpath of the execution data directory on the client container -EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data" +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/ethereumjs/execution-data" PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -80,6 +80,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -95,6 +96,7 @@ def launch( config = get_config( launcher.el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -105,6 +107,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ) service = plan.add_service(service_name, config) @@ -131,6 +134,7 @@ def launch( def get_config( el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, verbosity_level, @@ -141,6 +145,7 @@ def get_config( extra_params, extra_env_vars, extra_labels, + persistent, ): cmd = [ "--gethGenesis=" @@ -182,13 +187,19 @@ def get_config( # this is a repeated, we convert it into Starlark cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + } + + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) return ServiceConfig( image=image, ports=USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - }, + files=files, entrypoint=ENTRYPOINT_ARGS, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, diff --git a/src/el/geth/geth_launcher.star b/src/el/geth/geth_launcher.star index caee6a9d3..56a63762e 100644 --- a/src/el/geth/geth_launcher.star +++ b/src/el/geth/geth_launcher.star @@ -36,7 +36,7 @@ NUM_MINING_THREADS = 1 METRICS_PATH = "/debug/metrics/prometheus" # The dirpath of the execution data directory on the client container -EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data" +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/geth/execution-data" PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -86,6 +86,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -101,6 +102,7 @@ def launch( launcher.network_id, launcher.el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -114,6 +116,7 @@ def launch( launcher.capella_fork_epoch, launcher.electra_fork_epoch, launcher.final_genesis_timestamp, + persistent, ) service = plan.add_service(service_name, config) @@ -144,6 +147,7 @@ def get_config( network_id, el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, verbosity_level, @@ -157,6 +161,7 @@ def get_config( capella_fork_epoch, electra_fork_epoch, final_genesis_timestamp, + persistent, ): # TODO: Remove this once electra fork has path based storage scheme implemented if electra_fork_epoch != None: @@ -263,13 +268,18 @@ def get_config( ] command_str = " && ".join(subcommand_strs) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid + } + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) return ServiceConfig( image=image, ports=USED_PORTS, cmd=[command_str], - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid - }, + files=files, entrypoint=ENTRYPOINT_ARGS, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, diff --git a/src/el/nethermind/nethermind_launcher.star b/src/el/nethermind/nethermind_launcher.star index cb0e125a6..993812a1b 100644 --- a/src/el/nethermind/nethermind_launcher.star +++ b/src/el/nethermind/nethermind_launcher.star @@ -7,7 +7,7 @@ node_metrics = import_module("../../node_metrics_info.star") constants = import_module("../../package_io/constants.star") # The dirpath of the execution data directory on the client container -EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data" +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/nethermind/execution-data" METRICS_PATH = "/metrics" @@ -74,6 +74,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, NETHERMIND_LOG_LEVELS @@ -89,6 +90,7 @@ def launch( config = get_config( launcher.el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -99,6 +101,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ) service = plan.add_service(service_name, config) @@ -127,6 +130,7 @@ def launch( def get_config( el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -137,6 +141,7 @@ def get_config( extra_params, extra_env_vars, extra_labels, + persistent, ): cmd = [ "--log=" + log_level, @@ -177,13 +182,20 @@ def get_config( # this is a repeated, we convert it into Starlark cmd.extend([param for param in extra_params]) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + } + + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) + return ServiceConfig( image=image, ports=USED_PORTS, cmd=cmd, - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - }, + files=files, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, max_cpu=el_max_cpu, diff --git a/src/el/reth/reth_launcher.star b/src/el/reth/reth_launcher.star index 896ce4130..01986c56e 100644 --- a/src/el/reth/reth_launcher.star +++ b/src/el/reth/reth_launcher.star @@ -30,7 +30,7 @@ METRICS_PORT_ID = "metrics" METRICS_PATH = "/metrics" # The dirpath of the execution data directory on the client container -EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/execution-data" +EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER = "/data/reth/execution-data" PRIVATE_IP_ADDRESS_PLACEHOLDER = "KURTOSIS_IP_ADDR_PLACEHOLDER" @@ -78,6 +78,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ): log_level = input_parser.get_client_log_level_or_default( participant_log_level, global_log_level, VERBOSITY_LEVELS @@ -93,6 +94,7 @@ def launch( config = get_config( launcher.el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, log_level, @@ -103,6 +105,7 @@ def launch( extra_params, extra_env_vars, extra_labels, + persistent, ) service = plan.add_service(service_name, config) @@ -130,6 +133,7 @@ def launch( def get_config( el_cl_genesis_data, image, + service_name, existing_el_clients, cl_client_name, verbosity_level, @@ -140,6 +144,7 @@ def get_config( extra_params, extra_env_vars, extra_labels, + persistent, ): init_datadir_cmd_str = "reth init --datadir={0} --chain={1}".format( EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER, @@ -194,13 +199,19 @@ def get_config( ] command_str = " && ".join(subcommand_strs) + files = { + constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, + } + + if persistent: + files[EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER] = Directory( + persistent_key="data-{0}".format(service_name), + ) return ServiceConfig( image=image, ports=USED_PORTS, cmd=[command_str], - files={ - constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid, - }, + files=files, entrypoint=ENTRYPOINT_ARGS, private_ip_address_placeholder=PRIVATE_IP_ADDRESS_PLACEHOLDER, min_cpu=el_min_cpu, diff --git a/src/participant_network.star b/src/participant_network.star index 571a5fb16..2f318aec1 100644 --- a/src/participant_network.star +++ b/src/participant_network.star @@ -56,6 +56,7 @@ def launch_participant_network( participants, network_params, global_log_level, + persistent, parallel_keystore_generation=False, ): num_participants = len(participants) @@ -225,6 +226,7 @@ def launch_participant_network( participant.el_extra_params, participant.el_extra_env_vars, participant.el_extra_labels, + persistent, ) # Add participant el additional prometheus metrics @@ -344,6 +346,7 @@ def launch_participant_network( participant.validator_extra_params, participant.beacon_extra_labels, participant.validator_extra_labels, + persistent, participant.cl_split_mode_enabled, ) else: @@ -374,6 +377,7 @@ def launch_participant_network( participant.validator_extra_params, participant.beacon_extra_labels, participant.validator_extra_labels, + persistent, participant.cl_split_mode_enabled, )