feat: add blutgang rpc load balancer (#569)

.github/tests/mix-with-tools-mev.yaml

@@ -23,6 +23,7 @@ additional_services:
   - blobscan
   - blockscout
   - dugtrio
+  - blutgang
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 mev_type: full

.github/tests/mix-with-tools-minimal.yaml

@@ -33,6 +33,7 @@ additional_services:
   - blobscan
   - blockscout
   - dugtrio
+  - blutgang
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 keymanager_enabled: true

.github/tests/mix-with-tools.yaml

@@ -25,6 +25,7 @@ additional_services:
   - blobscan
   - blockscout
   - dugtrio
+  - blutgang
 ethereum_metrics_exporter_enabled: true
 snooper_enabled: true
 keymanager_enabled: true

README.md

@@ -543,6 +543,7 @@ additional_services:
   - prometheus_grafana
   - blobscan
   - dugtrio
+  - blutgang
 
 # Configuration place for transaction spammer - https:#github.com/MariusVanDerWijden/tx-fuzz
 tx_spammer_params:

main.star

@@ -22,6 +22,7 @@ beacon_metrics_gazer = import_module(
 )
 dora = import_module("./src/dora/dora_launcher.star")
 dugtrio = import_module("./src/dugtrio/dugtrio_launcher.star")
+blutgang = import_module("./src/blutgang/blutgang_launcher.star")
 blobscan = import_module("./src/blobscan/blobscan_launcher.star")
 full_beaconchain_explorer = import_module(
     "./src/full_beaconchain/full_beaconchain_launcher.star"
@@ -397,6 +398,20 @@ def run(plan, args={}):
                 global_node_selectors,
             )
             plan.print("Successfully launched dugtrio")
+        elif additional_service == "blutgang":
+            plan.print("Launching blutgang")
+            blutgang_config_template = read_file(
+                static_files.BLUTGANG_CONFIG_TEMPLATE_FILEPATH
+            )
+            blutgang.launch_blutgang(
+                plan,
+                blutgang_config_template,
+                all_participants,
+                args_with_right_defaults.participants,
+                network_params,
+                global_node_selectors,
+            )
+            plan.print("Successfully launched blutgang")
         elif additional_service == "blobscan":
             plan.print("Launching blobscan")
             blobscan.launch_blobscan(

src/blutgang/blutgang_launcher.star

@@ -0,0 +1,131 @@
+shared_utils = import_module("../shared_utils/shared_utils.star")
+constants = import_module("../package_io/constants.star")
+SERVICE_NAME = "blutgang"
+
+HTTP_PORT_ID = "http"
+HTTP_PORT_NUMBER = 3000
+
+ADMIN_PORT_ID = "admin"
+ADMIN_PORT_NUMBER = 5715
+
+BLUTGANG_CONFIG_FILENAME = "config.toml"
+
+BLUTGANG_CONFIG_MOUNT_DIRPATH_ON_SERVICE = "/config"
+
+IMAGE_NAME = "makemake1337/blutgang:latest"
+# IMAGE_NAME = "busybox:latest"
+
+# The min/max CPU/memory that blutgang can use
+MIN_CPU = 100
+MAX_CPU = 1000
+MIN_MEMORY = 128
+MAX_MEMORY = 2048
+
+USED_PORTS = {
+    HTTP_PORT_ID: shared_utils.new_port_spec(
+        HTTP_PORT_NUMBER,
+        shared_utils.TCP_PROTOCOL,
+        shared_utils.HTTP_APPLICATION_PROTOCOL,
+    ),
+    ADMIN_PORT_ID: shared_utils.new_port_spec(
+        ADMIN_PORT_NUMBER,
+        shared_utils.TCP_PROTOCOL,
+        shared_utils.HTTP_APPLICATION_PROTOCOL,
+    ),
+}
+
+
+def launch_blutgang(
+    plan,
+    config_template,
+    participant_contexts,
+    participant_configs,
+    network_params,
+    global_node_selectors,
+):
+    all_el_client_info = []
+    for index, participant in enumerate(participant_contexts):
+        full_name, _, el_client, _ = shared_utils.get_client_names(
+            participant, index, participant_contexts, participant_configs
+        )
+        all_el_client_info.append(
+            new_el_client_info(
+                el_client.ip_addr,
+                el_client.rpc_port_num,
+                el_client.ws_port_num,
+                full_name,
+            )
+        )
+
+    template_data = new_config_template_data(
+        network_params.network, HTTP_PORT_NUMBER, all_el_client_info
+    )
+
+    template_and_data = shared_utils.new_template_and_data(
+        config_template, template_data
+    )
+    template_and_data_by_rel_dest_filepath = {}
+    template_and_data_by_rel_dest_filepath[BLUTGANG_CONFIG_FILENAME] = template_and_data
+
+    config_files_artifact_name = plan.render_templates(
+        template_and_data_by_rel_dest_filepath, "blutgang-config"
+    )
+
+    config = get_config(
+        config_files_artifact_name,
+        network_params,
+        global_node_selectors,
+    )
+
+    plan.add_service(SERVICE_NAME, config)
+
+
+def get_config(
+    config_files_artifact_name,
+    network_params,
+    node_selectors,
+):
+    config_file_path = shared_utils.path_join(
+        BLUTGANG_CONFIG_MOUNT_DIRPATH_ON_SERVICE,
+        BLUTGANG_CONFIG_FILENAME,
+    )
+
+    return ServiceConfig(
+        image=IMAGE_NAME,
+        ports=USED_PORTS,
+        files={
+            BLUTGANG_CONFIG_MOUNT_DIRPATH_ON_SERVICE: config_files_artifact_name,
+        },
+        cmd=["/app/blutgang", "-c", config_file_path],
+        min_cpu=MIN_CPU,
+        max_cpu=MAX_CPU,
+        min_memory=MIN_MEMORY,
+        max_memory=MAX_MEMORY,
+        node_selectors=node_selectors,
+        ready_conditions=ReadyCondition(
+            recipe=GetHttpRequestRecipe(
+                port_id="admin",
+                endpoint="/ready",
+            ),
+            field="code",
+            assertion="==",
+            target_value=200,
+        ),
+    )
+
+
+def new_config_template_data(network, listen_port_num, el_client_info):
+    return {
+        "Network": network,
+        "ListenPortNum": listen_port_num,
+        "ELClientInfo": el_client_info,
+    }
+
+
+def new_el_client_info(ip_addr, rpc_port_num, ws_port_num, full_name):
+    return {
+        "IP_Addr": ip_addr,
+        "RPC_PortNum": rpc_port_num,
+        "WS_PortNum": ws_port_num,
+        "FullName": full_name,
+    }

src/static_files/static_files.star

@@ -20,6 +20,9 @@ DORA_CONFIG_TEMPLATE_FILEPATH = STATIC_FILES_DIRPATH + "/dora-config/config.yaml
 DUGTRIO_CONFIG_TEMPLATE_FILEPATH = (
     STATIC_FILES_DIRPATH + "/dugtrio-config/config.yaml.tmpl"
 )
+BLUTGANG_CONFIG_TEMPLATE_FILEPATH = (
+    STATIC_FILES_DIRPATH + "/blutgang-config/config.toml.tmpl"
+)
 
 FULL_BEACONCHAIN_CONFIG_TEMPLATE_FILEPATH = (
     STATIC_FILES_DIRPATH + "/full-beaconchain-config/config.yaml.tmpl"

static_files/blutgang-config/config.toml.tmpl

@@ -0,0 +1,74 @@
+# To use the config file, use the -c/--config option pointing to the path of a config file
+
+# Config for blutgang goes here
+[blutgang]
+# Clear the cache DB on startup
+do_clear = false
+# Where to bind blutgang to
+address = "0.0.0.0:3000"
+# Moving average length for the latency
+ma_length = 100
+# Sort RPCs by latency on startup. Recommended to leave on.
+sort_on_startup = true
+# Enable health checking
+health_check = true
+# Enable content type header checking. Set this to `true` if you want
+# Blutgang to be JSON-RPC compliant.
+header_check = true
+# Acceptable time to wait for a response in ms
+ttl = 30
+# How many times to retry a request before giving up
+max_retries = 32
+# Block time in ms, used as a sanity check when not receiving subscriptions
+expected_block_time = 13000
+# Time between health checks in ms
+health_check_ttl = 400
+# Supress the health check running info messages
+supress_rpc_check = false
+
+# Note: the admin namespace contains volatile functions and
+# should not be exposed publicly.
+[admin]
+# Enable the admin namespace
+enabled = true
+# Address for the admin RPC
+address = "0.0.0.0:5715"
+# Only allow read-only methods
+# Recommended `true` unless you 100% need write methods
+readonly = true
+# Enable the use of JWT for auth
+# Should be on if exposing to the internet
+jwt = false
+# jwt token
+key = ""
+
+# Sled config
+# Sled is the database we use for our cache, for more info check their docs
+[sled]
+# Path to db
+db_path = "./blutgang-cache"
+# sled mode. Can be HighThroughput/LowSpace
+mode = "HighThroughput"
+# Cache size in bytes.
+cache_capacity = 1000000000
+# Use zstd compression. Reduces size 60-70%,
+# and increases CPU and latency by around 10% for db writes and 2% for reads.
+# If storage constrained, it's fine to have it be on.
+compression = false
+# Print DB profile when dropped. Doesn't do anything for now.
+print_profile = false
+# Frequency of flushes in ms
+flush_every_ms = 240
+
+# Add separate RPCs as TOML tables
+# DO NOT name an rpc `blutgang`, `admin`, or `sled`
+
+
+# RPC Node Endpoints
+{{ range $elClient := .ELClientInfo }}
+[{{ $elClient.FullName }}]
+url = "http://{{ $elClient.IP_Addr }}:{{ $elClient.RPC_PortNum }}"
+ws_url = "ws://{{ $elClient.IP_Addr }}:{{ $elClient.WS_PortNum }}"
+max_consecutive = 150
+max_per_second = 200
+{{- end }}