Timeline of loadgen stats (#38)

Scripts to generate a CSV file containing a small subset of the loadgen stats across multiple revisions.
Increase default loadgen load

Dockerfile

@@ -153,13 +153,16 @@ ENV IS_DOCKER=true
 ENV SDK_SRC=/src
 ENV OUTPUT_DIR=/out
 ENV SDK_REVISION=
+ENV SDK_REPO=
 ENV SDK_BUILD=0
 ENV NVM_RC_VERSION=
 
 WORKDIR /app
 COPY --chown=$USER_UID:$USER_GID . .
 
-RUN mkdir -p $SDK_SRC $OUTPUT_DIR && chown $USER_UID:$USER_GID $SDK_SRC $OUTPUT_DIR /app
+RUN mkdir -p $SDK_SRC $OUTPUT_DIR /home/node/.cache/yarn /go/pkg/mod && \
+	chown -R $USER_UID:$USER_GID $SDK_SRC $OUTPUT_DIR /home/node/.cache /go && \
+	chown $USER_UID:$USER_GID /app
 
 USER $USER_UID
 

runner/lib/main.js

@@ -42,8 +42,8 @@ const pipeline = promisify(pipelineCallback);
 const finished = promisify(finishedCallback);
 
 const defaultLoadgenConfig = {
-  vault: { interval: 120 },
-  amm: { wait: 60, interval: 120 },
+  vault: { interval: 12, limit: 10 },
+  amm: { wait: 6, interval: 12, limit: 10 },
 };
 const defaultMonitorIntervalMinutes = 5;
 const defaultStageDurationMinutes = 6 * 60;

runner/lib/stats/run.js

@@ -190,6 +190,12 @@ export const makeRunStats = (data) => {
     savedData.endedAt &&
     rounder(savedData.endedAt - savedData.startedAt);
 
+  const getChainBootstrapStartedAt = () =>
+    stages[0] && stages[0].chainStartedAt;
+  const getChainBootstrapEndedAt = () => stages[0] && stages[0].chainReadyAt;
+  const getChainBootstrapDuration = () =>
+    stages[0] && stages[0].chainInitDuration;
+
   const getWalletDeployDuration = () =>
     savedData.walletDeployStartedAt &&
     savedData.walletDeployEndedAt &&
@@ -217,6 +223,9 @@ export const makeRunStats = (data) => {
         stages: () => stages,
         stageCount: getStageCount,
         duration: getDuration,
+        chainBootstrapStartedAt: getChainBootstrapStartedAt,
+        chainBootstrapEndedAt: getChainBootstrapEndedAt,
+        chainBootstrapDuration: getChainBootstrapDuration,
         walletDeployDuration: getWalletDeployDuration,
         loadgenDeployDuration: getLoadgenDeployDuration,
       }),

runner/lib/stats/types.d.ts

@@ -171,6 +171,9 @@ export interface RunStats extends RunStatsInitData {
   readonly startedAt: TimeValueS | undefined;
   readonly endedAt: TimeValueS | undefined;
   readonly duration: number | undefined;
+  readonly chainBootstrapStartedAt: TimeValueS | undefined;
+  readonly chainBootstrapEndedAt: TimeValueS | undefined;
+  readonly chainBootstrapDuration: number | undefined;
   readonly walletDeployStartedAt: TimeValueS | undefined;
   readonly walletDeployEndedAt: TimeValueS | undefined;
   readonly walletDeployDuration: number | undefined;

scripts/README.md

@@ -0,0 +1,35 @@
+This file contains scripts to automate running the loadgen on multiple revisions and generate stats.
+
+## Daily and manual perf scripts
+
+The `run-daily-perf.sh` and `run-manual.sh` both execute loadgen cycles on a "stream" of revisions, and save the resulting data in a new folder created in the current directory and named after the revision. In the former's case, the stream of revision is the latest HEAD of the github repository. In the latter's case, the stream of revision is a file provided as first argument.
+
+Requires a docker image named `loadgen-runner`. Extra arguments are passed to the container.
+
+### Daily perf service
+
+`loadgen-daily-perf.service` is an example of systemd service file to automatically run the daily-perf loadgen testing.
+
+### Manual perf example
+
+```sh
+cat << EOF | /path/to/testnet-load-generator/scripts/run-manual.sh - \
+  --no-stage.save-storage \
+  --stage.duration=30 \
+  --monitor-interval=1 \
+  --stage.loadgen.vault.interval=60 \
+  --stage.loadgen.amm.interval=60 \
+  --stage.loadgen.amm.wait=30
+549c301
+eba2fe2
+79a450f
+EOF
+```
+
+## Stats
+
+Currently gathering stats from the loadgen is done manually. To generate a CSV file with some summarized stats, use the following command, e.g. from the directory where manual loadgen results are stored:
+
+```sh
+tail -q -n 1 manual-*/perf.jsonl | /path/to/testnet-load-generator/scripts/perf_to_stats_csv.jq > stats.csv
+```

scripts/common-queue.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+set -m
+
+DOCKER_ID=
+SLEEP_PID=
+
+stop_sleep() { [ -z "$SLEEP_PID" ] && return; kill -TERM $SLEEP_PID; exit 0; }
+stop_container() { [ -z "${DOCKER_ID}" ] && return; docker kill --signal=SIGTERM ${DOCKER_ID}; }
+
+start() {
+  TEST_TYPE=$1
+  NEXT_REVISION_CMD=$2
+  shift 2
+
+  running=1
+  trap '' HUP
+  trap 'running=0; stop_sleep; stop_container' INT TERM
+
+  while [ $running -eq 1 ]
+  do
+    while true
+    do
+      PREV_SDK_REVISION=${SDK_REVISION}
+      SDK_REVISION=$(${NEXT_REVISION_CMD})
+      [ -z "$SDK_REVISION" ] && exit 1
+      OUTPUT_DIR="${TEST_TYPE}-${SDK_REVISION}"
+      [ ! -d "${OUTPUT_DIR}" ] && break
+      [ "x$SDK_REVISION" != "x$PREV_SDK_REVISION" ] && continue
+      sleep 60 &
+      SLEEP_PID=$!
+      wait $SLEEP_PID
+      SLEEP_PID=
+    done
+    echo "processing ${SDK_REVISION}"
+    mkdir "${OUTPUT_DIR}"
+    DOCKER_ID=$(docker create \
+      -v loadgen-go-pkg-mod:/go/pkg/mod \
+      -v loadgen-yarn-cache:/home/node/.cache/yarn \
+      -v "$(pwd)/${OUTPUT_DIR}:/out" \
+      -e SDK_REVISION=${SDK_REVISION} \
+      -e SDK_REPO=${SDK_REPO} \
+      --name "${OUTPUT_DIR}" \
+      loadgen-runner \
+      --test-data.test-type=${TEST_TYPE} "$@" \
+    ) || exit $?
+    docker start ${DOCKER_ID}
+    docker wait ${DOCKER_ID} >"${OUTPUT_DIR}/exit_code" &
+    DOCKER_WAIT_PID=$!
+    while kill -0 $DOCKER_WAIT_PID 2>/dev/null; do wait $DOCKER_WAIT_PID; done
+    docker logs ${DOCKER_ID} >"${OUTPUT_DIR}/docker.log" 2>&1
+    [ -d "/var/lib/docker" ] && sudo -n cat /var/lib/docker/containers/${DOCKER_ID}/${DOCKER_ID}-json.log >"${OUTPUT_DIR}/docker.json.log"
+    docker rm ${DOCKER_ID}
+    DOCKER_ID=
+  done
+}

scripts/perf_to_stats_csv.jq

@@ -0,0 +1,34 @@
+#!/usr/bin/env -S jq -Mf -s -r
+
+[ 
+  .[] |
+  select(has("stats")) |
+  .stats |
+  {
+    sdkRevision: (.metadata.testData.sdkRevision // "" | tostring), 
+    sdkCommitTime: (.metadata.testData.sdkCommitTime | strftime("%Y-%m-%d %H:%M:%S")? // ""),
+    success: (.duration != null),
+    chainBootstrapDuration: (.chainBootstrapDuration // .stages["0"].chainInitDuration),
+    walletDeployDuration: .walletDeployDuration,
+    loadgenDeployDuration: .loadgenDeployDuration,
+    cycleSuccessRate: .cyclesSummary.cycleSuccessRate,
+    cycleAvgDuration: .cyclesSummary.avgDuration,
+  } + (.stages | with_entries(
+    .value.stageConfig as $stageConfig |
+    ("stage" + .key + "_") as $stagePrefix |
+    if ($stageConfig["chainOnly"] != true and $stageConfig["durationConfig"] != 0) then 
+      { key: ($stagePrefix + "cycleSuccessRate"), value: .value.cyclesSummaries.all.cycleSuccessRate },
+      { key: ($stagePrefix + "cycleAvgDuration"), value: .value.cyclesSummaries.all.avgDuration },
+      { key: ($stagePrefix + "avgSwingsetBlockTime"), value: .value.blocksSummaries.onlyLive.avgSwingsetTime },
+      { key: ($stagePrefix + "avgSwingsetPercentage"), value: .value.blocksSummaries.onlyLive.avgSwingsetPercentage },
+      empty
+    else empty end
+  ))
+] |
+sort_by(.sdkCommitTime) |
+(
+  map(keys_unsorted) | add |
+  map({key: ., value: true}) | from_entries | keys_unsorted
+) as $cols |
+$cols, map(. as $row | $cols | map($row[.]))[] |
+@csv

scripts/run-daily-perf.sh

@@ -1,41 +1,17 @@
 #!/bin/sh
 set -x
 
-# Runs a full 24h loadgen on the latest HEAD and save the output in the current directory
+# Runs a loadgen cycle on the current HEAD, until stopped. Polls for a new revision every minute.
+# The arguments are passed to the loadgen-runner, e.g. to control the duration which by default is 24h
+# Output is saved in folders under the current directory, named `daily-perf-{revision}`
 # Requires a docker image named `loadgen-runner`
 
 SDK_REPO="${SDK_REPO:-https://github.com/Agoric/agoric-sdk.git}"
-running=1
-DOCKER_ID=
-SLEEP_PID=
 
-stop_sleep() { [ -z "$SLEEP_PID" ] && return; kill -TERM $SLEEP_PID; exit 0; }
-stop_container() { [ -z "${DOCKER_ID}" ] && return; docker kill --signal=SIGTERM ${DOCKER_ID}; }
+next_revision() {
+  git ls-remote ${SDK_REPO} HEAD | awk '{ print substr($1,1,12) }'
+}
 
-trap '' HUP
-trap 'running=0; stop_sleep; stop_container' INT TERM
+. $(dirname "$(readlink -f -- "$0")")/common-queue.sh
 
-while [ $running -eq 1 ]
-do
-  while true
-  do
-    SDK_REVISION=$(git ls-remote ${SDK_REPO} HEAD | awk '{ print substr($1,1,12) }')
-    OUTPUT_DIR="daily-perf-${SDK_REVISION}"
-    [ ! -d "${OUTPUT_DIR}" ] && break
-    sleep 60 &
-    SLEEP_PID=$!
-    wait $SLEEP_PID
-    SLEEP_PID=
-  done
-  echo "processing ${SDK_REVISION}"
-  mkdir "${OUTPUT_DIR}"
-  DOCKER_ID=$(docker create -v "$(pwd)/${OUTPUT_DIR}:/out" -e SDK_REVISION=${SDK_REVISION} --name "${OUTPUT_DIR}" loadgen-runner --test-data.test-type=daily-perf) || exit $?
-  docker start ${DOCKER_ID}
-  docker wait ${DOCKER_ID} >"${OUTPUT_DIR}/exit_code" &
-  DOCKER_WAIT_PID=$!
-  while kill -0 $DOCKER_WAIT_PID 2>/dev/null; do wait $DOCKER_WAIT_PID; done
-  docker logs ${DOCKER_ID} >"${OUTPUT_DIR}/docker.log" 2>&1
-  [ -d "/var/lib/docker" ] && sudo -n cat /var/lib/docker/containers/${DOCKER_ID}/${DOCKER_ID}-json.log >"${OUTPUT_DIR}/docker.json.log"
-  docker rm ${DOCKER_ID}
-  DOCKER_ID=
-done
+start "daily-perf" next_revision "$@"

scripts/run-manual.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+set -x
+
+# Runs a loadgen cycle for each revision from the list in the input file passed as first argument.
+# The rest of the arguments are passed to the loadgen-runner, e.g. to control the duration
+# Output is saved in folders under the current directory, named `manual-{revision}`
+# Requires a docker image named `loadgen-runner`
+
+next_revision() {
+  read -r REPLY <&3
+  echo $REPLY
+}
+
+. $(dirname "$(readlink -f -- "$0")")/common-queue.sh
+
+INPUT=$1
+shift
+
+if [ "x$INPUT" != "x-" ]; then
+  start "manual" next_revision "$@" 3<$INPUT
+else
+  start "manual" next_revision "$@" 3<&0
+fi