feat: add support for testnets (#437)

.github/tests/dencun-devnet-12.yaml

@@ -0,0 +1,28 @@
+participants:
+  - el_client_type: geth
+    el_client_image: ethpandaops/geth:master
+    cl_client_type: teku
+    cl_client_image: ethpandaops/teku:master
+  - el_client_type: nethermind
+    el_client_image: ethpandaops/nethermind:master
+    cl_client_type: prysm
+    cl_client_image: ethpandaops/prysm:develop,ethpandaops/prysm-validator:develop
+  - el_client_type: erigon
+    el_client_image: ethpandaops/erigon:devel
+    cl_client_type: nimbus
+    cl_client_image: ethpandaops/nimbus:unstable
+  - el_client_type: besu
+    el_client_image: ethpandaops/besu:main
+    cl_client_type: lighthouse
+    cl_client_image: ethpandaops/lighthouse:unstable
+  - el_client_type: reth
+    el_client_image: ethpandaops/reth:main
+    cl_client_type: lodestar
+    cl_client_image: ethpandaops/lodestar:unstable
+  - el_client_type: ethereumjs
+    el_client_image: ethpandaops/ethereumjs:master
+    cl_client_type: teku
+    cl_client_image: ethpandaops/teku:master
+network_params:
+  network: "dencun-devnet-12"
+additional_services: []

.github/tests/sepolia-mix.yaml

@@ -0,0 +1,16 @@
+participants:
+  - el_client_type: geth
+    cl_client_type: teku
+  - el_client_type: nethermind
+    cl_client_type: prysm
+  - el_client_type: erigon
+    cl_client_type: lighthouse
+  - el_client_type: besu
+    cl_client_type: lighthouse
+  - el_client_type: reth
+    cl_client_type: lodestar
+  - el_client_type: ethereumjs
+    cl_client_type: nimbus
+network_params:
+  network: "sepolia"
+additional_services: []

.github/tests/verkle-gen-devnet-2.yaml

@@ -0,0 +1,13 @@
+participants:
+  - el_client_type: geth
+    el_client_image: ethpandaops/geth:gballet-kaustinen-with-shapella-fc8f4b9
+    cl_client_type: lighthouse
+    cl_client_image: ethpandaops/lighthouse:verkle-trees-capella-2ffb8a9
+    count: 2
+  - el_client_type: geth
+    el_client_image: ethpandaops/geth:gballet-kaustinen-with-shapella-fc8f4b9
+    cl_client_type: lodestar
+    cl_client_image: ethpandaops/lodestar:g11tech-verge-815364b
+network_params:
+  network: verkle-gen-devnet-2
+

.github/workflows/nightly.yml

@@ -13,7 +13,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - id: set-matrix
-        run: echo "matrix=$(ls ./.github/tests/*.yaml | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+        # List all yaml files in the .github/tests directory, except for the k8s.yaml file
+        run: echo "matrix=$(ls ./.github/tests/*.yaml | grep -vE 'k8s.yaml$' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
   run_with_args:
     needs: list-yamls
     strategy:
@@ -47,5 +48,5 @@ jobs:
           DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
         with:
           args: "The nightly test for ${{matrix.file_name}} on ethereum-package has failed find it here ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-      
+
 

README.md

@@ -4,6 +4,8 @@
 
 This is a [Kurtosis][kurtosis-repo] package that will spin up a private Ethereum testnet over Docker or Kubernetes with multi-client support, Flashbot's `mev-boost` infrastructure for PBS-related testing/validation, and other useful network tools (transaction spammer, monitoring tools, etc). Kurtosis packages are entirely reproducible and composable, so this will work the same way over Docker or Kubernetes, in the cloud or locally on your machine.
 
+You now have the ability to spin up a private Ethereum testnet or public devnet/testnet (e.g. Goerli, Holesky, Sepolia, dencun-devnet-12, verkle-gen-devnet-2 etc) with a single command. This package is designed to be used for testing, validation, and development of Ethereum clients, and is not intended for production use. For more details check network_params.network in the [configuration section](./README.md#configuration).
+
 Specifically, this [package][package-reference] will:
 
 1. Generate Execution Layer (EL) & Consensus Layer (CL) genesis information using [the Ethereum genesis generator](https://github.com/ethpandaops/ethereum-genesis-generator).
@@ -272,6 +274,12 @@ network_params:
   deneb_fork_epoch: 500
   electra_fork_epoch: null
 
+  # Network name, used to enable syncing of alternative networks
+  # Defaults to "kurtosis"
+  # You can sync any public network by setting this to the network name (e.g. "mainnet", "goerli", "sepolia", "holesky")
+  # You can sync any devnet by setting this to the network name (e.g. "dencun-devnet-12", "verkle-gen-devnet-2")
+  network: "kurtosis"
+
 # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz
 tx_spammer_params:
   # A list of optional extra params that will be passed to the TX Spammer container for modifying its behaviour

src/cl/lighthouse/lighthouse_launcher.star

@@ -309,7 +309,6 @@ def get_beacon_config(
         "beacon_node",
         "--debug-level=" + log_level,
         "--datadir=" + BEACON_DATA_DIRPATH_ON_BEACON_SERVICE_CONTAINER,
-        "--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER,
         # vvvvvvvvvvvvvvvvvvv REMOVE THESE WHEN CONNECTING TO EXTERNAL NET vvvvvvvvvvvvvvvvvvvvv
         "--disable-enr-auto-update",
         "--enr-address=" + PRIVATE_IP_ADDRESS_PLACEHOLDER,
@@ -342,6 +341,12 @@ def get_beacon_config(
         # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
     ]
 
+    if network not in constants.PUBLIC_NETWORKS:
+        cmd.append("--testnet-dir=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER)
+    else:
+        cmd.append("--network=" + network)
+        cmd.append("--checkpoint-sync-url=" + constants.CHECKPOINT_SYNC_URL[network])
+
     if network == "kurtosis":
         if boot_cl_client_ctxs != None:
             cmd.append(

src/cl/lodestar/lodestar_launcher.star

@@ -277,12 +277,6 @@ def get_beacon_config(
         "--port={0}".format(DISCOVERY_PORT_NUM),
         "--discoveryPort={0}".format(DISCOVERY_PORT_NUM),
         "--dataDir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
-        "--paramsFile="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/config.yaml",
-        "--genesisStateFile="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/genesis.ssz",
         "--eth1.depositContractDeployBlock=0",
         "--network.connectToDiscv5Bootnodes=true",
         "--discv5=true",
@@ -306,21 +300,39 @@ def get_beacon_config(
         "--metrics.port={0}".format(METRICS_PORT_NUM),
         # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
     ]
-    if network == "kurtosis":
-        if bootnode_contexts != None:
+
+    if network not in constants.PUBLIC_NETWORKS:
+        cmd.append(
+            "--paramsFile="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/config.yaml"
+        )
+        cmd.append(
+            "--genesisStateFile="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/genesis.ssz"
+        )
+        if network == "kurtosis":  # kurtosis
+            if bootnode_contexts != None:
+                cmd.append(
+                    "--bootnodes="
+                    + ",".join(
+                        [
+                            ctx.enr
+                            for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]
+                        ]
+                    )
+                )
+        else:  # devnet
             cmd.append(
                 "--bootnodes="
-                + ",".join(
-                    [ctx.enr for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]]
+                + shared_utils.get_devnet_enrs_list(
+                    plan, el_cl_genesis_data.files_artifact_uuid
                 )
             )
-    elif network not in constants.PUBLIC_NETWORKS:
-        cmd.append(
-            "--bootnodes="
-            + shared_utils.get_devnet_enrs_list(
-                plan, el_cl_genesis_data.files_artifact_uuid
-            )
-        )
+    else:
+        cmd.append("--network=" + network)
+        cmd.append("--checkpointSyncUrl=" + constants.CHECKPOINT_SYNC_URL[network])
 
     if len(extra_params) > 0:
         # this is a repeated<proto type>, we convert it into Starlark

src/cl/nimbus/nimbus_launcher.star

@@ -51,7 +51,7 @@ VALIDATOR_METRICS_PATH = "/metrics"
 
 # Nimbus requires that its data directory already exists (because it expects you to bind-mount it), so we
 #  have to to create it
-BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/beacon-data"
+BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER = "/data/nimbus/beacon-data"
 # Nimbus wants the data dir to have these perms
 CONSENSUS_DATA_DIR_PERMS_STR = "0700"
 
@@ -313,7 +313,11 @@ def get_beacon_config(
         "--log-level=" + log_level,
         "--udp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
         "--tcp-port={0}".format(BEACON_DISCOVERY_PORT_NUM),
-        "--network=" + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER,
+        "--network={0}".format(
+            network
+            if network in constants.PUBLIC_NETWORKS
+            else constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+        ),
         "--data-dir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--web3-url=" + EXECUTION_ENGINE_ENDPOINT,
         "--nat=extip:" + PRIVATE_IP_ADDRESS_PLACEHOLDER,

src/cl/prysm/prysm_launcher.star

@@ -287,12 +287,6 @@ def get_beacon_config(
     cmd = [
         "--accept-terms-of-use=true",  # it's mandatory in order to run the node
         "--datadir=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
-        "--chain-config-file="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/config.yaml",
-        "--genesis-state="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/genesis.ssz",
         "--execution-endpoint=" + EXECUTION_ENGINE_ENDPOINT,
         "--rpc-host=0.0.0.0",
         "--rpc-port={0}".format(RPC_PORT_NUM),
@@ -316,18 +310,34 @@ def get_beacon_config(
         "--monitoring-port={0}".format(BEACON_MONITORING_PORT_NUM)
         # ^^^^^^^^^^^^^^^^^^^ METRICS CONFIG ^^^^^^^^^^^^^^^^^^^^^
     ]
-    if network == "kurtosis":
-        if bootnode_contexts != None:
-            for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]:
-                cmd.append("--peer=" + ctx.multiaddr)
-                cmd.append("--bootstrap-node=" + ctx.enr)
-            cmd.append("--p2p-static-id=true")
-    elif network not in constants.PUBLIC_NETWORKS:
+    if network not in constants.PUBLIC_NETWORKS:
+        cmd.append("--p2p-static-id=true")
         cmd.append(
-            "--bootstrap-node="
-            + shared_utils.get_devnet_enr(plan, el_cl_genesis_data.files_artifact_uuid)
+            "--chain-config-file="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/config.yaml"
         )
-        cmd.append("--p2p-static-id=true")
+        cmd.append(
+            "--genesis-state="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/genesis.ssz",
+        )
+        if network == "kurtosis":  # Kurtosis
+            if bootnode_contexts != None:
+                for ctx in bootnode_contexts[: constants.MAX_ENR_ENTRIES]:
+                    cmd.append("--peer=" + ctx.multiaddr)
+                    cmd.append("--bootstrap-node=" + ctx.enr)
+        else:  # Devnet
+            cmd.append(
+                "--bootstrap-node="
+                + shared_utils.get_devnet_enr(
+                    plan, el_cl_genesis_data.files_artifact_uuid
+                )
+            )
+    else:  # Public network
+        cmd.append("--{}".format(network))
+        cmd.append("--genesis-beacon-api-url=" + constants.CHECKPOINT_SYNC_URL[network])
+        cmd.append("--checkpoint-sync-url=" + constants.CHECKPOINT_SYNC_URL[network])
 
     if len(extra_params) > 0:
         # we do the for loop as otherwise its a proto repeated array

src/cl/teku/teku_launcher.star

@@ -302,12 +302,11 @@ def get_beacon_config(
     cmd = [
         "--logging=" + log_level,
         "--log-destination=CONSOLE",
-        "--network="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/config.yaml",
-        "--initial-state="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/genesis.ssz",
+        "--network={0}".format(
+            network
+            if network in constants.PUBLIC_NETWORKS
+            else constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER + "/config.yaml"
+        ),
         "--data-path=" + BEACON_DATA_DIRPATH_ON_SERVICE_CONTAINER,
         "--data-storage-mode={0}".format(
             "ARCHIVE" if constants.ARCHIVE_MODE else "PRUNE"
@@ -349,6 +348,15 @@ def get_beacon_config(
         + el_client_context.client_name,
     ]
 
+    if network not in constants.PUBLIC_NETWORKS:
+        cmd.append(
+            "--initial-state="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/genesis.ssz"
+        )
+    else:
+        cmd.append("--checkpoint-sync-url=" + constants.CHECKPOINT_SYNC_URL[network])
+
     if node_keystore_files != None and not split_mode_enabled:
         cmd.extend(validator_flags)
     if network == "kurtosis":

src/el/besu/besu_launcher.star

@@ -152,9 +152,6 @@ def get_config(
         "besu",
         "--logging=" + log_level,
         "--data-path=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
-        "--genesis-file="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/besu.json",
         "--host-allowlist=*",
         "--rpc-http-enabled=true",
         "--rpc-http-host=0.0.0.0",
@@ -178,6 +175,15 @@ def get_config(
         "--metrics-host=0.0.0.0",
         "--metrics-port={0}".format(METRICS_PORT_NUM),
     ]
+    if network not in constants.PUBLIC_NETWORKS:
+        cmd.append(
+            "--genesis-file="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/besu.json"
+        )
+    else:
+        cmd.append("--network=" + network)
+
     if network == "kurtosis":
         if len(existing_el_clients) > 0:
             cmd.append(

src/el/erigon/erigon_launcher.star

@@ -178,7 +178,6 @@ def get_config(
         "--metrics",
         "--metrics.addr=0.0.0.0",
         "--metrics.port={0}".format(METRICS_PORT_NUM),
-        "--db.size.limit=100GB",
     ]
 
     if network == "kurtosis":
@@ -202,6 +201,7 @@ def get_config(
                 )
             )
     elif network not in constants.PUBLIC_NETWORKS:
+        cmd.append("--db.size.limit=100GB")
         cmd.append(
             "--bootnodes="
             + shared_utils.get_devnet_enodes(
@@ -219,9 +219,16 @@ def get_config(
         # this is a repeated<proto type>, we convert it into Starlark
         cmd.extend([param for param in extra_params])
 
-    command_arg = [init_datadir_cmd_str, " ".join(cmd)]
+    if network not in constants.PUBLIC_NETWORKS:
+        command_arg = [init_datadir_cmd_str, " ".join(cmd)]
+
+        command_arg_str = " && ".join(command_arg)
+    else:
+        cmd.append("--chain={0}".format(network))
+        cmd.append("--db.size.limit=3TB")
+        command_arg = cmd
+        command_arg_str = " ".join(command_arg)
 
-    command_arg_str = " && ".join(command_arg)
     files = {
         constants.GENESIS_DATA_MOUNTPOINT_ON_CLIENTS: el_cl_genesis_data.files_artifact_uuid,
         constants.JWT_MOUNTPOINT_ON_CLIENTS: jwt_file,

src/el/ethereumjs/ethereumjs_launcher.star

@@ -153,9 +153,6 @@ def get_config(
     persistent,
 ):
     cmd = [
-        "--gethGenesis="
-        + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
-        + "/genesis.json",
         "--dataDir=" + EXECUTION_DATA_DIRPATH_ON_CLIENT_CONTAINER,
         "--port={0}".format(DISCOVERY_PORT_NUM),
         "--rpc",
@@ -176,6 +173,16 @@ def get_config(
         "--isSingleNode=true",
         "--logLevel={0}".format(verbosity_level),
     ]
+
+    if network not in constants.PUBLIC_NETWORKS:
+        cmd.append(
+            "--gethGenesis="
+            + constants.GENESIS_CONFIG_MOUNT_PATH_ON_CONTAINER
+            + "/genesis.json",
+        )
+    else:
+        cmd.append("--network=" + network)
+
     if network == "kurtosis":
         if len(existing_el_clients) > 0:
             cmd.append(