feat: Support participants_matrix (ethereum#620)

This PR adds a new config argument called `participants_matrix`, which
allows the user to easily spin up a matrix of EL/CL combos. The `el` and
`cl` keys within this argument can be fully fledged normal
`participants`, so they support the exact same arguments.

For example:

# Example 1
```
participants_matrix:
  el:
  - el_type: geth
  - el_type: besu
  - el_type: reth
  cl:
  - cl_type: nimbus
 ```
 
 
 This config would create the following participants:
 - `nimbus-geth`
 - `nimbus-besu`
 - `nimbus-reth`

# Example 2
```
participants_matrix:
  el:
  - el_type: geth
  - el_type: besu
  - el_type: reth
  cl:
  - cl_type: nimbus
participants:
 - el_type: nethermind
   cl_type: lighthouse
 ```
 This config would create the following participants:
 - `nimbus-geth`
 - `nimbus-besu`
 - `nimbus-reth`
 - `lighthouse-nethermind`
 
 
# Example 3
```
participants_matrix:
  el:
  - el_type: geth
  - el_type: besu
  - el_type: reth
  cl:
  - cl_type: nimbus
     count: 5
 ```
 This config would create the following participants:
 - `5x nimbus-geth`
 - `5x nimbus-besu`
 - `5x nimbus-reth`

---------

Co-authored-by: Barnabas Busa <busa.barnabas@gmail.com>

Author: samcm

.github/tests/apache.yaml

@@ -1,2 +1,4 @@
+participants:
+  - count: 1
 additional_services:
   - apache

.github/tests/mev-mock.yaml

@@ -1,3 +1,6 @@
+participants:
+  - el_type: geth
+    cl_type: lighthouse
 network_params:
   seconds_per_slot: 3
 additional_services: []

.github/tests/mev.yaml

@@ -1,3 +1,6 @@
+participants:
+  - el_type: geth
+    cl_type: lighthouse
 mev_type: flashbots
 additional_services:
   - tx_spammer

.github/tests/participants-matrix.yaml

@@ -0,0 +1,9 @@
+participants_matrix:
+  el:
+    - el_type: besu
+  cl:
+    - cl_type: prysm
+  vc:
+    - vc_type: nimbus
+participants:
+  - count: 1

.github/tests/peerdas-fork.yaml

@@ -1,17 +1,28 @@
 participants:
   - el_type: geth
+    el_image: ethpandaops/geth:master
     cl_type: prysm
-    cl_image: ethpandaops/prysm-beacon-chain:peerDASE2E
+    cl_image: ethpandaops/prysm-beacon-chain:peerDAS
     cl_max_mem: 2048
   - el_type: geth
+    el_image: ethpandaops/geth:master
     cl_type: lighthouse
-    cl_extra_args: [
+    cl_extra_params: [
         --subscribe-all-data-column-subnets,
     ]
     cl_image: ethpandaops/lighthouse:das
   - el_type: geth
     cl_type: lighthouse
     cl_image: ethpandaops/lighthouse:das
+  - el_type: geth
+    el_image: ethpandaops/geth:master
+    cl_type: teku
+    cl_image: ethpandaops/teku:das
+  - el_type: geth
+    el_image: ethpandaops/geth:master
+    cl_type: nimbus
+    cl_image: ethpandaops/nimbus-eth2:wip-peerdas
+    validator_count: 1
 network_params:
   eip7594_fork_epoch: 0
   eip7594_fork_version: "0x50000038"

.github/workflows/per-pr.yml

@@ -22,7 +22,7 @@ jobs:
           kurtosis analytics disable
 
       - name: Run Starlark
-        run: kurtosis run ${{ github.workspace }}
+        run: kurtosis run ${{ github.workspace }} --args-file network_params.yaml
 
   run_with_args:
     strategy:

README.md

@@ -456,6 +456,20 @@ participants:
     # Defaults null and then set to default global keymanager_enabled (false)
     keymanager_enabled: null
 
+# Participants matrix creates a participant for each combination of EL, CL and VC clients
+# Each EL/CL/VC item can provide the same parameters as a standard participant
+participants_matrix: {}
+  # el:
+  #   - el_type: geth
+  #   - el_type: besu
+  # cl:
+  #   - cl_type: prysm
+  #   - cl_type: lighthouse
+  # vc:
+  #   - vc_type: prysm
+  #   - vc_type: lighthouse
+
+
 # Default configuration parameters for the network
 network_params:
   # Network name, used to enable syncing of alternative networks

src/package_io/input_parser.star

@@ -371,6 +371,42 @@ def parse_network_params(plan, input_args):
     result = default_input_args()
     if input_args.get("network_params", {}).get("preset") == "minimal":
         result["network_params"] = default_minimal_network_params()
+
+    # Ensure we handle matrix participants before standard participants are handled.
+    if "participants_matrix" in input_args:
+        participants_matrix = []
+        participants = []
+
+        el_matrix = []
+        if "el" in input_args["participants_matrix"]:
+            el_matrix = input_args["participants_matrix"]["el"]
+        cl_matrix = []
+        if "cl" in input_args["participants_matrix"]:
+            cl_matrix = input_args["participants_matrix"]["cl"]
+        vc_matrix = []
+        if "vc" in input_args["participants_matrix"]:
+            vc_matrix = input_args["participants_matrix"]["vc"]
+
+        participants = []
+
+        for el in el_matrix:
+            for cl in cl_matrix:
+                participant = {k: v for k, v in el.items()}
+                for k, v in cl.items():
+                    participant[k] = v
+
+                participants.append(participant)
+
+        for index, participant in enumerate(participants):
+            for vc in vc_matrix:
+                for k, v in vc.items():
+                    participants[index][k] = v
+
+        if "participants" in input_args:
+            input_args["participants"].extend(participants)
+        else:
+            input_args["participants"] = participants
+
     for attr in input_args:
         value = input_args[attr]
         # if its insterted we use the value inserted
@@ -664,9 +700,11 @@ def get_client_node_selectors(participant_node_selectors, global_node_selectors)
 
 def default_input_args():
     network_params = default_network_params()
-    participants = [default_participant()]
+    participants = []
+    participants_matrix = []
     return {
         "participants": participants,
+        "participants_matrix": participants_matrix,
         "network_params": network_params,
         "wait_for_finalization": False,
         "global_log_level": "info",

src/participant_network.star

@@ -310,7 +310,7 @@ def launch_participant_network(
             )
         all_snooper_beacon_contexts.append(snooper_beacon_context)
         full_name = (
-            "{0}-{1}-{2}".format(index_str, el_type, cl_type) + "-{0}".format(vc_type)
+            "{0}-{1}-{2}-{3}".format(index_str, el_type, cl_type, vc_type)
             if participant.cl_type != participant.vc_type
             else "{0}-{1}-{2}".format(index_str, el_type, cl_type)
         )
@@ -319,7 +319,7 @@ def launch_participant_network(
             plan=plan,
             launcher=vc.new_vc_launcher(el_cl_genesis_data=el_cl_data),
             keymanager_file=keymanager_file,
-            service_name="vc-{0}-{1}-{2}".format(index_str, vc_type, el_type),
+            service_name="vc-{0}".format(full_name),
             vc_type=vc_type,
             image=participant.vc_image,
             participant_log_level=participant.vc_log_level,