feat: add assertoor to additional toolings (#419)

Added assertoor testnet testing tool.

There are currently 6 tests included:
* Check for chain stability (finalizing, high voting consensus, no
reorgs)
* Check if all clients propose a block
* Validator Lifecycle Test
Test for the whole validator lifecycle, includes Deposits, BLSChanges,
Exits & Slashings
This Test takes up to 2 days to run through due to deposit delays &
queues.
* Normal Transaction Test
Test if normal EOA transactions (type 0/2) are processed by all client
pairs (inclusion & submission)
* Blob Transaction Test
Test if blob transactions (type 3) are processed by all client pairs
(inclusion & submission)
* All-Opcodes Transaction Test
  Generates & submits a transaction that triggers every EVM opcode once.
  Checks if the transaction succeeds and no client forks off

---------

Co-authored-by: Barnabas Busa <busa.barnabas@gmail.com>

.github/tests/assertoor.yaml

@@ -0,0 +1,9 @@
+participants:
+  - el_client_type: geth
+    cl_client_type: lighthouse
+    count: 1
+  - el_client_type: geth
+    cl_client_type: lodestar
+    count: 1
+additional_services:
+  - assertoor

.github/workflows/per-pr.yml

@@ -33,6 +33,7 @@ jobs:
             "./.github/tests/mev-mock.yaml",
             "./.github/tests/mix-with-tools.yaml",
             "./.github/tests/mix-persistence.yaml",
+            "./.github/tests/assertoor.yaml",
             "./network_params.yaml"
           ]
     runs-on: ubuntu-latest

README.md

@@ -314,6 +314,62 @@ goomy_blob_params:
   # A list of optional params that will be passed to the blob-spammer comamnd for modifying its behaviour
   goomy_blob_args: []
 
+# Configuration place for the assertoor testing tool - https:#github.com/ethpandaops/assertoor
+assertoor_params:
+  # Check chain stability
+  # This check monitors the chain and succeeds if:
+  # - all clients are synced
+  # - chain is finalizing for min. 2 epochs
+  # - >= 98% correct target votes
+  # - >= 80% correct head votes
+  # - no reorgs with distance > 2 blocks
+  # - no more than 2 reorgs per epoch
+  run_stability_check: true
+
+  # Check block propöosals
+  # This check monitors the chain and succeeds if:
+  # - all client pairs have proposed a block
+  run_block_proposal_check: true
+
+  # Run normal transaction test
+  # This test generates random EOA transactions and checks inclusion with/from all client pairs
+  # This test checks for:
+  # - block proposals with transactions from all client pairs
+  # - transaction inclusion when submitting via each client pair
+  # test is done twice, first with legacy (type 0) transactions, then with dynfee (type 2) transactions
+  run_transaction_test: false
+
+  # Run blob transaction test
+  # This test generates blob transactions and checks inclusion with/from all client pairs
+  # This test checks for:
+  # - block proposals with blobs from all client pairs
+  # - blob inclusion when submitting via each client pair
+  run_blob_transaction_test: false
+
+  # Run all-opcodes transaction test
+  # This test generates a transaction that triggers all EVM OPCODES once
+  # This test checks for:
+  # - all-opcodes transaction success
+  run_opcodes_transaction_test: false
+
+  # Run validator lifecycle test (~48h to complete)
+  # This test requires exactly 500 active validator keys.
+  # The test will cause a temporary chain unfinality when running.
+  # This test checks:
+  # - Deposit inclusion with/from all client pairs
+  # - BLS Change inclusion with/from all client pairs
+  # - Voluntary Exit inclusion with/from all client pairs
+  # - Attester Slashing inclusion with/from all client pairs
+  # - Proposer Slashing inclusion with/from all client pairs
+  # all checks are done during finality & unfinality
+  run_lifecycle_test: false
+
+  # Run additional tests from external test definitions
+  # eg:
+  #   - https://raw.githubusercontent.com/ethpandaops/assertoor/master/example/tests/block-proposal-check.yaml
+  tests: []
+
+
 # By default includes
 # - A transaction spammer & blob spammer is launched to fake transactions sent to the network
 # - Forkmon for EL will be launched
@@ -322,6 +378,7 @@ goomy_blob_params:
 # - A light beacon chain explorer will be launched
 # - Default: ["tx_spammer", "blob_spammer", "el_forkmon", "beacon_metrics_gazer", "dora"," "prometheus_grafana"]
 additional_services:
+  - assertoor
   - broadcaster
   - tx_spammer
   - blob_spammer
@@ -607,6 +664,7 @@ Here's a table of where the keys are used
 | 5             | eip4788_deployment  | ✅                |                 | As contract deployer       |
 | 6             | mev_flood           | ✅                |                 | As the contract owner      |
 | 7             | mev_flood           | ✅                |                 | As the user_key            |
+| 8             | assertoor           | ✅                | ✅              | As the funding for tests   |
 | 11            | mev_custom_flood    | ✅                |                 | As the sender of balance   |
 
 ## Developing On This Package

main.star

@@ -38,6 +38,7 @@ eip4788_deployment = import_module(
     "./src/eip4788_deployment/eip4788_deployment_launcher.star"
 )
 broadcaster = import_module("./src/broadcaster/broadcaster.star")
+assertoor = import_module("./src/assertoor/assertoor_launcher.star")
 
 GRAFANA_USER = "admin"
 GRAFANA_PASSWORD = "admin"
@@ -371,6 +372,20 @@ def run(plan, args={}):
         elif additional_service == "prometheus_grafana":
             # Allow prometheus to be launched last so is able to collect metrics from other services
             launch_prometheus_grafana = True
+        elif additional_service == "assertoor":
+            plan.print("Launching assertoor")
+            assertoor_config_template = read_file(
+                static_files.ASSERTOOR_CONFIG_TEMPLATE_FILEPATH
+            )
+            assertoor_params = args_with_right_defaults.assertoor_params
+            assertoor.launch_assertoor(
+                plan,
+                assertoor_config_template,
+                all_participants,
+                args_with_right_defaults.participants,
+                assertoor_params,
+            )
+            plan.print("Successfully launched assertoor")
         elif additional_service == "custom_flood":
             mev_custom_flood.spam_in_background(
                 plan,

src/assertoor/assertoor_launcher.star

@@ -0,0 +1,144 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+static_files = import_module("../static_files/static_files.star")
+constants = import_module("../package_io/constants.star")
+SERVICE_NAME = "assertoor"
+
+HTTP_PORT_ID = "http"
+HTTP_PORT_NUMBER = 8080
+
+ASSERTOOR_CONFIG_FILENAME = "assertoor-config.yaml"
+
+ASSERTOOR_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/config"
+ASSERTOOR_TESTS_MOUNT_DIRPATH_ON_SERVICE = "/tests"
+
+VALIDATOR_RANGES_MOUNT_DIRPATH_ON_SERVICE = "/validator-ranges"
+VALIDATOR_RANGES_ARTIFACT_NAME = "validator-ranges"
+
+# The min/max CPU/memory that assertoor can use
+MIN_CPU = 100
+MAX_CPU = 1000
+MIN_MEMORY = 128
+MAX_MEMORY = 2048
+
+USED_PORTS = {
+    HTTP_PORT_ID: shared_utils.new_port_spec(
+        HTTP_PORT_NUMBER,
+        shared_utils.TCP_PROTOCOL,
+        shared_utils.HTTP_APPLICATION_PROTOCOL,
+    )
+}
+
+
+def launch_assertoor(
+    plan,
+    config_template,
+    participant_contexts,
+    participant_configs,
+    assertoor_params,
+):
+    all_client_info = []
+    validator_client_info = []
+
+    for index, participant in enumerate(participant_contexts):
+        participant_config = participant_configs[index]
+        cl_client = participant.cl_client_context
+        el_client = participant.el_client_context
+
+        all_client_info.append(
+            new_client_info(
+                cl_client.ip_addr,
+                cl_client.http_port_num,
+                el_client.ip_addr,
+                el_client.rpc_port_num,
+                cl_client.beacon_service_name,
+            )
+        )
+
+        if participant_config.validator_count != 0:
+            validator_client_info.append(
+                new_client_info(
+                    cl_client.ip_addr,
+                    cl_client.http_port_num,
+                    el_client.ip_addr,
+                    el_client.rpc_port_num,
+                    cl_client.beacon_service_name,
+                )
+            )
+
+    template_data = new_config_template_data(
+        HTTP_PORT_NUMBER, all_client_info, validator_client_info, assertoor_params
+    )
+
+    template_and_data = shared_utils.new_template_and_data(
+        config_template, template_data
+    )
+    template_and_data_by_rel_dest_filepath = {}
+    template_and_data_by_rel_dest_filepath[
+        ASSERTOOR_CONFIG_FILENAME
+    ] = template_and_data
+
+    config_files_artifact_name = plan.render_templates(
+        template_and_data_by_rel_dest_filepath, "assertoor-config"
+    )
+
+    tests_config_artifacts_name = plan.upload_files(
+        static_files.ASSERTOOR_TESTS_CONFIG_DIRPATH, name="assertoor-tests"
+    )
+
+    config = get_config(
+        config_files_artifact_name,
+        tests_config_artifacts_name,
+    )
+
+    plan.add_service(SERVICE_NAME, config)
+
+
+def get_config(config_files_artifact_name, tests_config_artifacts_name):
+    config_file_path = shared_utils.path_join(
+        ASSERTOOR_CONFIG_MOUNT_DIRPATH_ON_SERVICE,
+        ASSERTOOR_CONFIG_FILENAME,
+    )
+
+    IMAGE_NAME = "ethpandaops/assertoor:master"
+
+    return ServiceConfig(
+        image=IMAGE_NAME,
+        ports=USED_PORTS,
+        files={
+            ASSERTOOR_CONFIG_MOUNT_DIRPATH_ON_SERVICE: config_files_artifact_name,
+            ASSERTOOR_TESTS_MOUNT_DIRPATH_ON_SERVICE: tests_config_artifacts_name,
+            VALIDATOR_RANGES_MOUNT_DIRPATH_ON_SERVICE: VALIDATOR_RANGES_ARTIFACT_NAME,
+        },
+        cmd=["--config", config_file_path],
+        min_cpu=MIN_CPU,
+        max_cpu=MAX_CPU,
+        min_memory=MIN_MEMORY,
+        max_memory=MAX_MEMORY,
+    )
+
+
+def new_config_template_data(
+    listen_port_num, client_info, validator_client_info, assertoor_params
+):
+    return {
+        "ListenPortNum": listen_port_num,
+        "ClientInfo": client_info,
+        "ValidatorClientInfo": validator_client_info,
+        "RunStabilityCheck": assertoor_params.run_stability_check,
+        "RunBlockProposalCheck": assertoor_params.run_block_proposal_check,
+        "RunLifecycleTest": assertoor_params.run_lifecycle_test,
+        "RunTransactionTest": assertoor_params.run_transaction_test,
+        "RunBlobTransactionTest": assertoor_params.run_blob_transaction_test,
+        "RunOpcodesTransactionTest": assertoor_params.run_opcodes_transaction_test,
+        "AdditionalTests": assertoor_params.tests,
+    }
+
+
+def new_client_info(cl_ip_addr, cl_port_num, el_ip_addr, el_port_num, service_name):
+    return {
+        "CLIPAddr": cl_ip_addr,
+        "CLPortNum": cl_port_num,
+        "ELIPAddr": el_ip_addr,
+        "ELPortNum": el_port_num,
+        "Name": service_name,
+    }

src/package_io/input_parser.star

@@ -52,6 +52,7 @@ ATTR_TO_BE_SKIPPED_AT_ROOT = (
     "network_params",
     "participants",
     "mev_params",
+    "assertoor_params",
     "goomy_blob_params",
     "tx_spammer_params",
     "custom_flood_params",
@@ -72,6 +73,8 @@ def input_parser(plan, input_args):
     result["tx_spammer_params"] = get_default_tx_spammer_params()
     result["custom_flood_params"] = get_default_custom_flood_params()
     result["disable_peer_scoring"] = False
+    result["goomy_blob_params"] = get_default_goomy_blob_params()
+    result["assertoor_params"] = get_default_assertoor_params()
     result["persistent"] = False
 
     for attr in input_args:
@@ -92,6 +95,14 @@ def input_parser(plan, input_args):
             for sub_attr in input_args["custom_flood_params"]:
                 sub_value = input_args["custom_flood_params"][sub_attr]
                 result["custom_flood_params"][sub_attr] = sub_value
+        elif attr == "goomy_blob_params":
+            for sub_attr in input_args["goomy_blob_params"]:
+                sub_value = input_args["goomy_blob_params"][sub_attr]
+                result["goomy_blob_params"][sub_attr] = sub_value
+        elif attr == "assertoor_params":
+            for sub_attr in input_args["assertoor_params"]:
+                sub_value = input_args["assertoor_params"][sub_attr]
+                result["assertoor_params"][sub_attr] = sub_value
 
     if result.get("disable_peer_scoring"):
         result = enrich_disable_peer_scoring(result)
@@ -116,7 +127,6 @@ def input_parser(plan, input_args):
             )
         )
 
-    result["goomy_blob_params"] = get_default_goomy_blob_params()
     return struct(
         participants=[
             struct(
@@ -213,6 +223,21 @@ def input_parser(plan, input_args):
         goomy_blob_params=struct(
             goomy_blob_args=result["goomy_blob_params"]["goomy_blob_args"],
         ),
+        assertoor_params=struct(
+            run_stability_check=result["assertoor_params"]["run_stability_check"],
+            run_block_proposal_check=result["assertoor_params"][
+                "run_block_proposal_check"
+            ],
+            run_lifecycle_test=result["assertoor_params"]["run_lifecycle_test"],
+            run_transaction_test=result["assertoor_params"]["run_transaction_test"],
+            run_blob_transaction_test=result["assertoor_params"][
+                "run_blob_transaction_test"
+            ],
+            run_opcodes_transaction_test=result["assertoor_params"][
+                "run_opcodes_transaction_test"
+            ],
+            tests=result["assertoor_params"]["tests"],
+        ),
         custom_flood_params=struct(
             interval_between_transactions=result["custom_flood_params"][
                 "interval_between_transactions"
@@ -514,6 +539,18 @@ def get_default_goomy_blob_params():
     return {"goomy_blob_args": []}
 
 
+def get_default_assertoor_params():
+    return {
+        "run_stability_check": True,
+        "run_block_proposal_check": True,
+        "run_lifecycle_test": False,
+        "run_transaction_test": False,
+        "run_blob_transaction_test": False,
+        "run_opcodes_transaction_test": False,
+        "tests": [],
+    }
+
+
 def get_default_custom_flood_params():
     # this is a simple script that increases the balance of the coinbase address at a cadence
     return {"interval_between_transactions": 1}

src/static_files/static_files.star

@@ -22,6 +22,15 @@ FULL_BEACONCHAIN_CONFIG_TEMPLATE_FILEPATH = (
     STATIC_FILES_DIRPATH + "/full-beaconchain-config/config.yaml.tmpl"
 )
 
+# assertoor config
+ASSERTOOR_CONFIG_DIRPATH = "/assertoor-config"
+ASSERTOOR_CONFIG_TEMPLATE_FILEPATH = (
+    STATIC_FILES_DIRPATH + ASSERTOOR_CONFIG_DIRPATH + "/config.yaml.tmpl"
+)
+ASSERTOOR_TESTS_CONFIG_DIRPATH = (
+    STATIC_FILES_DIRPATH + ASSERTOOR_CONFIG_DIRPATH + "/tests"
+)
+
 # Grafana config
 GRAFANA_CONFIG_DIRPATH = "/grafana-config"
 GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH = (