R4R: Multi-seed parallel simulation

Makefile

@@ -159,9 +159,9 @@ test_sim_gaia_fast:
 	@echo "Running quick Gaia simulation. This may take several minutes..."
 	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=400 -SimulationBlockSize=200 -SimulationCommit=true -v -timeout 24h
 
-test_sim_gaia_slow:
-	@echo "Running full Gaia simulation. This may take a while!"
-	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationVerbose=true -SimulationCommit=true -v -timeout 24h
+test_sim_gaia_full:
+	@echo "Running full multi-seed Gaia simulation. This may take awhile!"
+	@sh scripts/multisim.sh
 
 SIM_NUM_BLOCKS ?= 210
 SIM_BLOCK_SIZE ?= 200

PENDING.md

@@ -90,6 +90,7 @@ FEATURES
 
 * Gaia
   * [cli] #2170 added ability to show the node's address via `gaiad tendermint show-address`
+  * [simulation] #2313 Reworked `make test_sim_gaia_slow` to `make test_sim_gaia_full`, now simulates from multiple starting seeds in parallel
   * [cli] [\#1921] (https://github.com/cosmos/cosmos-sdk/issues/1921)
     * New configuration file `gaiad.toml` is now created to host Gaia-specific configuration.
     * New --minimum_fees/minimum_fees flag/config option to set a minimum fee.

scripts/multisim.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+seeds=(1 2 4 7 9 20 32 123 4728 37827 981928 87821 891823782 989182 89182391)
+
+echo "Running multi-seed simulation with seeds: ${seeds[@]}"
+echo "Edit scripts/multisim.sh to add new seeds. Keeping parameters in the file makes failures easy to reproduce."
+echo "This script will kill all sub-simulations on SIGINT/SIGTERM/EXIT (i.e. Ctrl-C)."
+
+trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
+
+tmpdir=$(mktemp -d)
+echo "Using temporary log directory: $tmpdir"
+
+sim() {
+  seed=$1
+	echo "Running full Gaia simulation with seed $seed. This may take awhile!"
+  file="$tmpdir/gaia-simulation-seed-$seed-date-$(date -Iseconds -u).stdout"
+  echo "Writing stdout to $file..."
+	go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 \
+    -SimulationVerbose=true -SimulationCommit=true -SimulationSeed=$seed -v -timeout 24h > $file
+}
+
+i=0
+pids=()
+for seed in ${seeds[@]}; do
+  sim $seed &
+  pids[${i}]=$!
+  i=$(($i+1))
+  sleep 0.1 # start in order, nicer logs
+done
+
+echo "Simulation processes spawned, waiting for completion..."
+
+code=0
+
+i=0
+for pid in ${pids[*]}; do
+  wait $pid
+  last=$?
+  if [ $last -ne 0 ]; then
+    seed=${seeds[${i}]}
+    echo "Simulation with seed $seed failed!"
+    code=1
+  fi
+  i=$(($i+1))
+done
+
+exit $code