Matching simulation comparison tool

.gitignore

@@ -25,6 +25,7 @@ test_eventsV2.log
 test_eventsV2_xdc
 test_eventsV2_xdc.log
 matching-simulator-output/
+simulation_comparison.csv
 
 # Executables produced by cadence repo
 /cadence

codecov.yml

@@ -70,3 +70,4 @@ ignore:
   - "testflags/**"
   - "tools/common/schema/test/**"
   - "tools/linter/**"
+  - "tools/matchingsimulationcomparison/**"

host/matching_simulation_test.go

@@ -24,12 +24,15 @@
 /*
 To run locally:
 
-1. Change the matchingconfig in host/testdata/matching_simulation.yaml as you wish
+1. Pick a scenario from the existing config files host/testdata/matching_simulation_.*.yaml or add a new one
 
-2. Run `./scripts/run_matching_simulator.sh`
+2. Run the scenario
+`./scripts/run_matching_simulator.sh default`
 
-Full test logs can be found at test.log file. Event json logs can be found at matching-simulator-output.json.
+Full test logs can be found at test.log file. Event json logs can be found at matching-simulator-output folder.
 See the run_matching_simulator.sh script for more details about how to parse events.
+
+If you want to run all the scenarios and compare them refer to tools/matchingsimulationcomparison/README.md
 */
 package host
 

host/testdata/matching_simulation_default.yaml

@@ -14,8 +14,8 @@ matchingconfig:
     forwardermaxoutstandingtasks: 1
     forwardermaxratepersecond: 10
     forwardermaxchildrenpernode: 20
-    localpollwaittime: 0ms
-    localtaskwaittime: 0ms
+    localpollwaittime: 10ms
+    localtaskwaittime: 10ms
     tasks:
       - numtaskgenerators: 2
         taskspersecond: 80

host/testdata/matching_simulation_more_read_partitions.yaml

@@ -14,8 +14,8 @@ matchingconfig:
     forwardermaxoutstandingtasks: 1
     forwardermaxratepersecond: 10
     forwardermaxchildrenpernode: 20
-    localpollwaittime: 0ms
-    localtaskwaittime: 0ms
+    localpollwaittime: 10ms
+    localtaskwaittime: 10ms
     tasks:
       - numtaskgenerators: 2
         taskspersecond: 80

host/testdata/matching_simulation_no_forwarding.yaml

@@ -14,8 +14,8 @@ matchingconfig:
     forwardermaxoutstandingtasks: 0
     forwardermaxratepersecond: 10
     forwardermaxchildrenpernode: 20
-    localpollwaittime: 0ms
-    localtaskwaittime: 0ms
+    localpollwaittime: 10ms
+    localtaskwaittime: 10ms
     tasks:
       - numtaskgenerators: 2
         taskspersecond: 80
@@ -25,4 +25,4 @@ matchingconfig:
         numpollers: 8
         polltimeout: 60s
 workerconfig:
-  enableasyncwfconsumer: false
+  enableasyncwfconsumer: false

host/testdata/matching_simulation_no_wait_time_for_addtask.yaml

@@ -0,0 +1,28 @@
+enablearchival: false
+clusterno: 1
+messagingclientconfig:
+  usemock: true
+historyconfig:
+  numhistoryshards: 4
+  numhistoryhosts: 1
+matchingconfig:
+  nummatchinghosts: 4
+  simulationconfig:
+    tasklistwritepartitions: 4
+    tasklistreadpartitions: 4
+    forwardermaxoutstandingpolls: 1
+    forwardermaxoutstandingtasks: 1
+    forwardermaxratepersecond: 10
+    forwardermaxchildrenpernode: 20
+    localpollwaittime: 10ms
+    localtaskwaittime: 0ms
+    tasks:
+      - numtaskgenerators: 2
+        taskspersecond: 80
+        maxtasktogenerate: 3000
+    pollers:
+      - taskprocesstime: 1ms
+        numpollers: 8
+        polltimeout: 60s
+workerconfig:
+  enableasyncwfconsumer: false

host/testdata/matching_simulation_no_wait_time_for_polltask.yaml

@@ -0,0 +1,28 @@
+enablearchival: false
+clusterno: 1
+messagingclientconfig:
+  usemock: true
+historyconfig:
+  numhistoryshards: 4
+  numhistoryhosts: 1
+matchingconfig:
+  nummatchinghosts: 4
+  simulationconfig:
+    tasklistwritepartitions: 4
+    tasklistreadpartitions: 4
+    forwardermaxoutstandingpolls: 1
+    forwardermaxoutstandingtasks: 1
+    forwardermaxratepersecond: 10
+    forwardermaxchildrenpernode: 20
+    localpollwaittime: 0ms
+    localtaskwaittime: 10ms
+    tasks:
+      - numtaskgenerators: 2
+        taskspersecond: 80
+        maxtasktogenerate: 3000
+    pollers:
+      - taskprocesstime: 1ms
+        numpollers: 8
+        polltimeout: 60s
+workerconfig:
+  enableasyncwfconsumer: false

host/testdata/matching_simulation_no_wait_time_for_polltask_and_addtask.yaml

@@ -0,0 +1,28 @@
+enablearchival: false
+clusterno: 1
+messagingclientconfig:
+  usemock: true
+historyconfig:
+  numhistoryshards: 4
+  numhistoryhosts: 1
+matchingconfig:
+  nummatchinghosts: 4
+  simulationconfig:
+    tasklistwritepartitions: 4
+    tasklistreadpartitions: 4
+    forwardermaxoutstandingpolls: 1
+    forwardermaxoutstandingtasks: 1
+    forwardermaxratepersecond: 10
+    forwardermaxchildrenpernode: 20
+    localpollwaittime: 0ms
+    localtaskwaittime: 0ms
+    tasks:
+      - numtaskgenerators: 2
+        taskspersecond: 80
+        maxtasktogenerate: 3000
+    pollers:
+      - taskprocesstime: 1ms
+        numpollers: 8
+        polltimeout: 60s
+workerconfig:
+  enableasyncwfconsumer: false

host/testdata/matching_simulation_throughput.yaml

@@ -14,8 +14,8 @@ matchingconfig:
     forwardermaxoutstandingtasks: 1
     forwardermaxratepersecond: 10
     forwardermaxchildrenpernode: 20
-    localpollwaittime: 0ms
-    localtaskwaittime: 0ms
+    localpollwaittime: 10ms
+    localtaskwaittime: 10ms
     tasks:
       - numtaskgenerators: 2
         taskspersecond: 80

host/testdata/matching_simulation_zonal_isolation.yaml

@@ -14,8 +14,8 @@ matchingconfig:
     forwardermaxoutstandingtasks: 1
     forwardermaxratepersecond: 10
     forwardermaxchildrenpernode: 20
-    localpollwaittime: 0ms
-    localtaskwaittime: 0ms
+    localpollwaittime: 10ms
+    localtaskwaittime: 10ms
     tasks:
       - numtaskgenerators: 2
         taskspersecond: 80

scripts/run_matching_simulator.sh

@@ -7,21 +7,22 @@ set -eo pipefail
 
 testCase="${1:-default}"
 testCfg="testdata/matching_simulation_$testCase.yaml"
-testName="test-$testCase-$(date '+%Y-%m-%d-%H-%M-%S')"
+now="$(date '+%Y-%m-%d-%H-%M-%S')"
+timestamp="${2:-$now}"
+testName="test-$testCase-$timestamp"
 resultFolder="matching-simulator-output"
 mkdir -p "$resultFolder"
 eventLogsFile="$resultFolder/events.json"
 testSummaryFile="$resultFolder/$testName-summary.txt"
 
-
 echo "Building test image"
 docker-compose -f docker/buildkite/docker-compose-local-matching-simulation.yml \
   build matching-simulator
 
 echo "Running the test $testCase"
 docker-compose \
   -f docker/buildkite/docker-compose-local-matching-simulation.yml \
-  run -e MATCHING_SIMULATION_CONFIG=$testCfg --rm matching-simulator \
+  run -e MATCHING_SIMULATION_CONFIG=$testCfg --rm --remove-orphans matching-simulator \
   | grep -a --line-buffered "Matching New Event" \
   | sed "s/Matching New Event: //" \
   | jq . > "$eventLogsFile"
@@ -187,5 +188,7 @@ cat "$eventLogsFile" \
   | jq -c '.' | sort -n | uniq -c | sed -e 's/^/     /' | tee -a $testSummaryFile
 
 
+echo "End of summary" | tee -a $testSummaryFile
+
 printf "\nResults are saved in $testSummaryFile\n"
 printf "For further analysis, please check $eventLogsFile via jq queries\n"

tools/matchingsimulationcomparison/README.md

@@ -0,0 +1,17 @@
+
+This tool runs a set of matching simulation tests, extracts stats from their output and generates a csv to compare them easily.
+
+Note: The parsing logic might break in the future if the `run_matching_simulator.sh` starts spitting different shaped lines. Alternative is to load all the event logs into a sqlite table and then run queries on top instead of parsing outputs of jq in this tool.
+
+
+Run all the scenarios and compare:
+```
+go run tools/matchingsimulationcomparison/*.go --out simulation_comparison.csv
+```
+
+If you have already run some scenarios before and made changes in the output/comparison then run in Compare mode
+```
+go run tools/matchingsimulationcomparison/*.go --out simulation_comparison.csv \
+    --ts 2024-09-12-18-16-44 \
+    --mode Compare
+```