Problem: testground infra is not easy to setup

flake.nix

@@ -42,6 +42,10 @@
  apps = {
  cronosd = mkApp packages.cronosd;
  cronosd-testnet = mkApp packages.cronosd-testnet;
+ stateless-testcase = {
+ type = "app";
+ program = "${pkgs.testground-testcase}/bin/stateless-testcase";
+ };
  };
  defaultPackage = packages.cronosd;
  defaultApp = apps.cronosd;

testground/README.md

@@ -1,3 +1,5 @@
+# Testground
+
 [Testground documentation](https://docs.testground.ai/)
 
 ## Build Image
@@ -67,3 +69,28 @@ mounts:
  writable: true
 ```
 
+
+
+# Stateless Mode
+
+To simplify cluster setup, we are introducing a stateless mode.
+
+## Generate Data Files Locally
+
+You need to have a `cronosd` in `PATH`.
+
+```bash
+$ nix run github:crypto-org-chain/cronos#stateless-testcase gen /tmp/data/out 3 7
+```
+
+## Run In Local Docker
+
+```bash
+$ jsonnet -S testground/benchmark/compositions/docker-compose.jsonnet | docker-compose -f /dev/stdin up
+```
+
+It'll mount the data files to all the containers.
+
+## Run In Cluster
+
+TODO

testground/benchmark/benchmark/main.py

@@ -1,17 +1,14 @@
 import os
 import subprocess
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 import web3
 
 from .cli import ChainCommand
 from .context import Context
-from .peer import bootstrap
-from .sendtx import fund_test_accounts, sendtx
-from .utils import export_eth_account, wait_for_block, wait_for_port
-
-CRONOSD_PATH = "/bin/cronosd"
+from .peer import CONTAINER_CRONOSD_PATH, bootstrap
+from .sendtx import generate_load
+from .utils import wait_for_block, wait_for_port
 
 
 def influxdb_url():
@@ -21,15 +18,15 @@ def influxdb_url():
 def entrypoint(ctx: Context):
  ctx.init_common()
 
- cli = ChainCommand(CRONOSD_PATH)
+ cli = ChainCommand(CONTAINER_CRONOSD_PATH)
 
  # build the genesis file collectively, and setup the network topology
  bootstrap(ctx, cli)
 
  # start the node
  logfile = Path(ctx.params.test_outputs_path) / "node.log"
  proc = subprocess.Popen(
- [CRONOSD_PATH, "start"],
+ [CONTAINER_CRONOSD_PATH, "start"],
  stdout=open(logfile, "ab", buffering=0),
  )
 
@@ -39,21 +36,7 @@ def entrypoint(ctx: Context):
 
  test_finish_entry = f"finish-test-{ctx.params.test_group_id}"
  if not ctx.is_validator:
- w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
- assert w3.eth.chain_id == 777
- genesis_account = export_eth_account(cli, "account")
- accounts = fund_test_accounts(w3, genesis_account, ctx.params.num_accounts)
- with ThreadPoolExecutor(max_workers=ctx.params.num_accounts) as executor:
- futs = (
- executor.submit(sendtx, w3, acct, ctx.params.num_txs)
- for acct in accounts
- )
- for fut in as_completed(futs):
- try:
- fut.result()
- except Exception as e:
- print("test task failed", e)
-
+ generate_load(cli, ctx.params.num_accounts, ctx.params.num_txs)
  print("finish test", ctx.group_seq)
  ctx.sync.signal_and_wait(
  test_finish_entry, ctx.params.test_group_instance_count

testground/benchmark/benchmark/peer.py

@@ -3,86 +3,129 @@
 from pathlib import Path
 from typing import List
 
+from .cli import ChainCommand
 from .context import Context
 from .network import get_data_ip
 from .topology import connect_all
 from .types import GenesisAccount, PeerPacket
 from .utils import patch_json, patch_toml
 
+VAL_ACCOUNT = "validator"
 VAL_INITIAL_AMOUNT = "100000000000000000000basecro"
 VAL_STAKED_AMOUNT = "10000000000000000000basecro"
-ACC_INITIAL_AMOUNT = "100000000000000000000basecro"
+ACC_INITIAL_AMOUNT = "100000000000000000000000basecro"
 MEMPOOL_SIZE = 50000
+DEFAULT_DENOM = "basecro"
+VALIDATOR_GROUP = "validators"
+FULLNODE_GROUP = "fullnodes"
+CONTAINER_CRONOSD_PATH = "/bin/cronosd"
 
 
 def bootstrap(ctx: Context, cli) -> PeerPacket:
- ip = get_data_ip(ctx.params)
- cli(
- "init",
- f"node{ctx.global_seq}",
- chain_id=ctx.params.chain_id,
- default_denom="basecro",
+ home = Path.home() / ".cronos"
+ peer = init_node(
+ cli,
+ home,
+ get_data_ip(ctx.params),
+ ctx.params.chain_id,
+ ctx.params.test_group_id,
+ ctx.group_seq,
  )
 
- cli("keys", "add", "validator", keyring_backend="test")
- cli("keys", "add", "account", keyring_backend="test")
- validator_addr = cli(
- "keys", "show", "validator", "--address", keyring_backend="test"
+ data = ctx.sync.publish_subscribe_simple(
+ "peers", peer.dict(), ctx.params.test_instance_count
+ )
+ peers: List[PeerPacket] = [PeerPacket.model_validate(item) for item in data]
+
+ if ctx.is_fullnode_leader:
+ # prepare genesis file and publish
+ genesis = gen_genesis(cli, home, peers)
+ ctx.sync.publish("genesis", genesis)
+ else:
+ genesis = ctx.sync.subscribe_simple("genesis", 1)[0]
+ (home / "config" / "genesis.json").write_text(json.dumps(genesis))
+ cli("genesis", "validate", home=home)
+
+ p2p_peers = connect_all(peer, peers)
+ patch_configs(home, ctx.params.test_group_id, p2p_peers)
+ return peer
+
+
+def init_node(
+ cli: ChainCommand,
+ home: Path,
+ ip: str,
+ chain_id: str,
+ group: str,
+ group_seq: int,
+) -> PeerPacket:
+ default_kwargs = {
+ "home": home,
+ "chain_id": chain_id,
+ "keyring_backend": "test",
+ }
+ cli(
+ "init",
+ f"{group}-{group_seq}",
+ default_denom=DEFAULT_DENOM,
+ **default_kwargs,
  )
- account_addr = cli("keys", "show", "account", "--address", keyring_backend="test")
+ cli("keys", "add", VAL_ACCOUNT, **default_kwargs)
+ cli("keys", "add", "account", **default_kwargs)
+ validator_addr = cli("keys", "show", VAL_ACCOUNT, "--address", **default_kwargs)
+ account_addr = cli("keys", "show", "account", "--address", **default_kwargs)
  accounts = [
  GenesisAccount(address=validator_addr, balance=VAL_INITIAL_AMOUNT),
  GenesisAccount(address=account_addr, balance=ACC_INITIAL_AMOUNT),
  ]
 
- node_id = cli("comet", "show-node-id")
+ node_id = cli("comet", "show-node-id", **default_kwargs)
  peer_id = f"{node_id}@{ip}:26656"
- current = PeerPacket(
+ peer = PeerPacket(
  ip=str(ip),
  node_id=node_id,
  peer_id=peer_id,
  accounts=accounts,
  )
 
- if ctx.is_validator:
- current.gentx = gentx(cli, ctx.params.chain_id)
-
- data = ctx.sync.publish_subscribe_simple(
- "peers", current.dict(), ctx.params.test_instance_count
+ if group == VALIDATOR_GROUP:
+ peer.gentx = gentx(cli, **default_kwargs)
+
+ return peer
+
+
+def gen_genesis(cli: ChainCommand, leader_home: Path, peers: List[PeerPacket]):
+ for peer in peers:
+ for account in peer.accounts:
+ cli(
+ "genesis",
+ "add-genesis-account",
+ account.address,
+ account.balance,
+ home=leader_home,
+ )
+ collect_gen_tx(cli, peers, home=leader_home)
+ cli("genesis", "validate", home=leader_home)
+ return patch_json(
+ leader_home / "config" / "genesis.json",
+ {
+ "consensus.params.block.max_gas": "81500000",
+ "app_state.evm.params.evm_denom": "basecro",
+ "app_state.feemarket.params.no_base_fee": True,
+ },
  )
- peers: List[PeerPacket] = [PeerPacket.model_validate(item) for item in data]
 
- config_path = Path.home() / ".cronos" / "config"
- if ctx.is_fullnode_leader:
- # prepare genesis file and publish
- for peer in peers:
- for account in peer.accounts:
- cli("genesis", "add-genesis-account", account.address, account.balance)
- collect_gen_tx(cli, peers)
- cli("genesis", "validate")
- genesis = patch_json(
- config_path / "genesis.json",
- {
- "consensus.params.block.max_gas": "81500000",
- "app_state.evm.params.evm_denom": "basecro",
- "app_state.feemarket.params.no_base_fee": True,
- },
- )
- ctx.sync.publish("genesis", genesis)
- else:
- genesis = ctx.sync.subscribe_simple("genesis", 1)[0]
- genesis_file = config_path / "genesis.json"
- genesis_file.write_text(json.dumps(genesis))
- cli("genesis", "validate")
 
+def patch_configs(home: Path, group: str, peers: str):
  # update persistent_peers and other configs in config.toml
  config_patch = {
- "p2p.persistent_peers": connect_all(current, peers),
+ "p2p.persistent_peers": peers,
+ "p2p.addr_book_strict": False,
  "mempool.recheck": "false",
  "mempool.size": MEMPOOL_SIZE,
  "consensus.timeout_commit": "2s",
  }
- if ctx.is_validator:
+ if group == VALIDATOR_GROUP:
  config_patch["tx_index.indexer"] = "null"
 
  app_patch = {
@@ -92,35 +135,33 @@ def bootstrap(ctx: Context, cli) -> PeerPacket:
  "mempool.max-txs": MEMPOOL_SIZE,
  }
 
- patch_toml(config_path / "config.toml", config_patch)
- patch_toml(config_path / "app.toml", app_patch)
+ patch_toml(home / "config" / "config.toml", config_patch)
+ patch_toml(home / "config" / "app.toml", app_patch)
 
- return current
 
-
-def gentx(cli, chain_id):
+def gentx(cli, **kwargs):
  cli(
  "genesis",
  "add-genesis-account",
- "validator",
+ VAL_ACCOUNT,
  VAL_INITIAL_AMOUNT,
- keyring_backend="test",
+ **kwargs,
  )
- output = Path("gentx.json")
- cli(
- "genesis",
- "gentx",
- "validator",
- VAL_STAKED_AMOUNT,
- min_self_delegation=1,
- chain_id=chain_id,
- output_document=output,
- keyring_backend="test",
- )
- return json.loads(output.read_text())
+ with tempfile.TemporaryDirectory() as tmp:
+  output = Path(tmp) / "gentx.json"
+ cli(
+  "genesis",
+  "gentx",
+  VAL_ACCOUNT,
+  VAL_STAKED_AMOUNT,
+  min_self_delegation=1,
+  output_document=output,
+  **kwargs,
+  )
+  return json.loads(output.read_text())
 
 
-def collect_gen_tx(cli, peers):
+def collect_gen_tx(cli, peers, **kwargs):
  """
  save gentxs to file and call collect-gentxs
  leader node prepare genesis file and broadcast to other nodes
@@ -130,4 +171,4 @@ def collect_gen_tx(cli, peers):
  for i, peer in enumerate(peers):
  if peer.gentx is not None:
  (tmpdir / f"gentx-{i}.json").write_text(json.dumps(peer.gentx))
- cli("genesis", "collect-gentxs", gentx_dir=str(tmpdir))
+ cli("genesis", "collect-gentxs", gentx_dir=str(tmpdir), **kwargs)

testground/benchmark/benchmark/sendtx.py

@@ -1,9 +1,10 @@
 import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import web3
 from eth_account import Account
 
-from .utils import send_transaction
+from .utils import export_eth_account, send_transaction
 
 TEST_AMOUNT = 1000000000000000000
 GAS_PRICE = 1000000000
@@ -54,3 +55,17 @@ def sendtx(w3: web3.Web3, acct: Account, tx_amount: int):
 
  if nonce % 100 == 0:
  print(f"{acct.address} sent {nonce} transactions")
+
+
+def generate_load(cli, num_accounts, num_txs, **kwargs):
+ w3 = web3.Web3(web3.providers.HTTPProvider("http://localhost:8545"))
+ assert w3.eth.chain_id == 777
+ genesis_account = export_eth_account(cli, "account", **kwargs)
+ accounts = fund_test_accounts(w3, genesis_account, num_accounts)
+ with ThreadPoolExecutor(max_workers=num_accounts) as executor:
+ futs = (executor.submit(sendtx, w3, acct, num_txs) for acct in accounts)
+ for fut in as_completed(futs):
+ try:
+ fut.result()
+ except Exception as e:
+ print("test task failed", e)