Loadgen runner

.dockerignore

@@ -0,0 +1,9 @@
+node_modules
+defaults.js
+installationConstants.js
+_agstate/yarn-links/
+.idea/
+ui/.cache/
+ui/dist/
+/Dockerfile
+/results

.github/workflows/lint-test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [12.x, 14.x]
+        node-version: [14.x]
     steps:
     - name: Checkout dapp
       uses: actions/checkout@v2

.gitignore

@@ -4,4 +4,6 @@ installationConstants.js
 _agstate/yarn-links/
 .idea/
 ui/.cache/
-ui/dist/
+ui/dist/
+/results/*
+!/results/.keep

Dockerfile

@@ -0,0 +1,161 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/typescript-node/.devcontainer/base.Dockerfile
+
+# [Choice] Node.js version: 16, 14, 12
+ARG VARIANT="14-buster"
+FROM mcr.microsoft.com/vscode/devcontainers/typescript-node:0-${VARIANT} as base
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment if you want to install an additional version of node using nvm
+# ARG EXTRA_NODE_VERSION=10
+# RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}"
+
+# [Optional] Uncomment if you want to install more global node packages
+# RUN su node -c "npm install -g <your-package-list -here>"
+
+##############################
+# From https://github.com/docker-library/golang/blob/master/1.16/buster/Dockerfile
+
+ENV PATH /usr/local/go/bin:$PATH
+
+ENV GOLANG_VERSION 1.16.5
+
+RUN set -eux; \
+	\
+	dpkgArch="$(dpkg --print-architecture)"; \
+	url=; \
+	case "${dpkgArch##*-}" in \
+		'amd64') \
+			url='https://dl.google.com/go/go1.16.5.linux-amd64.tar.gz'; \
+			sha256='b12c23023b68de22f74c0524f10b753e7b08b1504cb7e417eccebdd3fae49061'; \
+			;; \
+		'armel') \
+			export GOARCH='arm' GOARM='5' GOOS='linux'; \
+			;; \
+		'armhf') \
+			url='https://dl.google.com/go/go1.16.5.linux-armv6l.tar.gz'; \
+			sha256='93cacacfbe87e3106b5bf5821de106f0f0a43c8bd1029826d44445c15df795a5'; \
+			;; \
+		'arm64') \
+			url='https://dl.google.com/go/go1.16.5.linux-arm64.tar.gz'; \
+			sha256='d5446b46ef6f36fdffa852f73dfbbe78c1ddf010b99fa4964944b9ae8b4d6799'; \
+			;; \
+		'i386') \
+			url='https://dl.google.com/go/go1.16.5.linux-386.tar.gz'; \
+			sha256='a37c6b71d0b673fe8dfeb2a8b3de78824f05d680ad32b7ac6b58c573fa6695de'; \
+			;; \
+		'mips64el') \
+			export GOARCH='mips64le' GOOS='linux'; \
+			;; \
+		'ppc64el') \
+			url='https://dl.google.com/go/go1.16.5.linux-ppc64le.tar.gz'; \
+			sha256='fad2da6c86ede8448d2d0e66e1776e2f0ae9169714eade29b9ffbbdede7fc6cc'; \
+			;; \
+		's390x') \
+			url='https://dl.google.com/go/go1.16.5.linux-s390x.tar.gz'; \
+			sha256='21085f6a3568fae639edf383cce78bcb00d8f415e5e3d7feb04b6124e8e9efc1'; \
+			;; \
+		*) echo >&2 "error: unsupported architecture '$dpkgArch' (likely packaging update needed)"; exit 1 ;; \
+	esac; \
+	build=; \
+	if [ -z "$url" ]; then \
+# https://github.com/golang/go/issues/38536#issuecomment-616897960
+		build=1; \
+		url='https://dl.google.com/go/go1.16.5.src.tar.gz'; \
+		sha256='7bfa7e5908c7cc9e75da5ddf3066d7cbcf3fd9fa51945851325eebc17f50ba80'; \
+		echo >&2; \
+		echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; \
+		echo >&2; \
+	fi; \
+	\
+	wget -O go.tgz.asc "$url.asc" --progress=dot:giga; \
+	wget -O go.tgz "$url" --progress=dot:giga; \
+	echo "$sha256 *go.tgz" | sha256sum --strict --check -; \
+	\
+# https://github.com/golang/go/issues/14739#issuecomment-324767697
+	export GNUPGHOME="$(mktemp -d)"; \
+# https://www.google.com/linuxrepositories/
+#	gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \
+  curl 'https://dl.google.com/linux/linux_signing_key.pub' | gpg --batch --import; \
+	gpg --batch --verify go.tgz.asc go.tgz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME" go.tgz.asc; \
+	\
+	tar -C /usr/local -xzf go.tgz; \
+	rm go.tgz; \
+	\
+	if [ -n "$build" ]; then \
+		savedAptMark="$(apt-mark showmanual)"; \
+		apt-get update; \
+		apt-get install -y --no-install-recommends golang-go; \
+		\
+		( \
+			cd /usr/local/go/src; \
+# set GOROOT_BOOTSTRAP + GOHOST* such that we can build Go successfully
+			export GOROOT_BOOTSTRAP="$(go env GOROOT)" GOHOSTOS="$GOOS" GOHOSTARCH="$GOARCH"; \
+			./make.bash; \
+		); \
+		\
+		apt-mark auto '.*' > /dev/null; \
+		apt-mark manual $savedAptMark > /dev/null; \
+		apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+		rm -rf /var/lib/apt/lists/*; \
+		\
+# pre-compile the standard library, just like the official binary release tarballs do
+		go install std; \
+# go install: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64
+#		go install -race std; \
+		\
+# remove a few intermediate / bootstrapping files the official binary release tarballs do not contain
+		rm -rf \
+			/usr/local/go/pkg/*/cmd \
+			/usr/local/go/pkg/bootstrap \
+			/usr/local/go/pkg/obj \
+			/usr/local/go/pkg/tool/*/api \
+			/usr/local/go/pkg/tool/*/go_bootstrap \
+			/usr/local/go/src/cmd/dist/dist \
+		; \
+	fi; \
+	\
+	go version
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+#WORKDIR $GOPATH
+
+##############################
+# From https://github.com/microsoft/vscode-dev-containers/blob/v0.163.1/containers/go/.devcontainer/base.Dockerfile
+
+# Install Go tools
+ENV GO111MODULE=auto
+COPY library-scripts/go-debian.sh /tmp/library-scripts/
+RUN bash /tmp/library-scripts/go-debian.sh "none" "/usr/local/go" "${GOPATH}" "node" "false" \
+    && apt-get clean -y && rm -rf /tmp/library-scripts
+
+# Add Tini
+ENV TINI_VERSION v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
+RUN chmod +x /tini
+
+##############################
+FROM base
+
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+ENV IS_DOCKER=true
+ENV SDK_SRC=/src
+ENV OUTPUT_DIR=/out
+ENV SDK_REVISION=
+
+WORKDIR /app
+COPY --chown=$USER_UID:$USER_GID . .
+
+RUN mkdir -p $SDK_SRC $OUTPUT_DIR && chown $USER_UID:$USER_GID $SDK_SRC $OUTPUT_DIR /app
+
+USER $USER_UID:$USER_GID
+
+ENTRYPOINT ["/tini", "--", "/app/start.sh"]

Makefile

@@ -30,6 +30,11 @@ run-chain:
 run-client:
 	$(MAKE) -C $(CSDIR) scenario2-run-client
 
+build-docker:
+	docker build -t loadgen-runner .
+
+daily-perf:
+	cd results && ../scripts/run-daily-perf.sh
 
 run-loadgen:
 	yarn loadgen

README.md

@@ -1,5 +1,117 @@
 # Load Generator
 
+## Runner
+
+The loadgen runner automates running any number of load generation cycles on a local or testnet chain, monitoring the local chain node and vats processes. Depending on the use case, it can be ran using a local Agoric SDK repo, or checkout and setup any given revision, with multiple layers of helpers to automate the execution:
+
+- `loadgen-runner` executable: core tool automating loadgen cycles against an installed agoric SDK (available on PATH)
+- `start.sh` script: helper to automate checking out any agoric-sdk revision, compile and install it in a temporary location, and run the load generator with it. Can also be used an existing agoric-sdk repo.
+- docker image: A Linux Debian environment setup with all dependencies to allow compiling the agoric-sdk. The entrypoint executes the start script, and has mount points for output directory and optionally an SDK repo.
+- `run-daily-perf.sh` script: service entrypoint to continuously execute a `loadgen-runner` docker image against the latest revision with the default options.
+
+### `loadgen-runner` executable
+
+At the core, the loadgen-runner can be configured to run multiple stages of load generation, either on a local solo chain, or against an existing chain, automatically spawning a ag-solo client and deploying the loadgen tool. It captures the slog file of the local chain node, the state storage of the chain at the end of each stage, and process and disk usage information.
+
+#### Command
+
+Assuming the agoric-sdk and cosmic-swingset are built and installed, and the agoric cli is available in `PATH`.
+
+```sh
+mkdir -p $HOME/loadgen-output
+./runner/bin/loadgen-runner --output-dir=$HOME/loadgen-output
+```
+
+#### Options
+
+The runner uses `yargs-parser` to parse the string command line arguments, and handles dynamically creating a complex `argv` object from them. It automatically converts unary arguments into boolean (with support for `no-` negation prefix), number conversion, nested objects with dot (`.`) notation, and does kebab to camel case conversion.
+
+Currently the following options are available:
+
+- `--output-dir`: the directory location where to put the results from the loadgen cycles (`perf.jsonl`, chain node slogs, chain node storage). Defaults to `results/run-{posixtime}` in the working directory.
+- `--profile`: (experimental) the chain target, either `local` (default), `testnet` or `stage`.
+- `--no-monitor`: allows disabling running a chain monitor node (for non `local` profiles).
+- `--monitor-interval`: a number in minutes for the interval at which to capture process stats for the chain.
+- `--no-reset`: a boolean option to control whether the local chain state directory should be checked out clean before starting.
+- `--stages`: the total number of stages to run. Default to 6
+- `--stage.loadgen.*`: the object to use as default loadgen config for the stages. Created from multiple arguments and passed as-is to the loadgen tool.
+- `--stage.duration`: the time in minutes to use as default duration for each loadgen stage (non chain-only, see below). Defaults to 360 minutes (6 hours).
+- `--stage.n.*`: Override config for a given stage 0 <= n < `--stages`
+- `--stage.n.loadgen.*`: the object to use as loadgen config for the given stage. If specified and `chain-only` is missing Created from multiple arguments and passed as-is to the loadgen tool.
+- `--stage.n.
+- `--stage.n.chain-only`: boolean flag specifying if the stage should only run the chain node and not start a client or loadgen. Defaults to `true` for first and last stage. Defaults to `false` for other stages, or if `--stage.n.loadgen.*` is specified.
+- `--stage.n.save-storage`: boolean indicating if the storage of the chain node should be saved at the end of the stage. Defaults to `true` for non stage-only stages (where the loadgen runs), as well as for stage 0 (to capture local bootstrap).
+- `--stage.n.duration`: the time in minutes for the stage duration. Defaults to the shared duration above for non chain-only stages, or 0 (immediate stop after start) otherwise. Use a negative value to run until interrupted.
+
+### `start.sh` script
+
+The start script automates checking out and setting up any revision of the Agoric SDK before launching the loadgen-runner. It does so without interfering with an existing sdk installation by default, but can also be pointed to run the setup steps on an existing checked out repository.
+
+All command line arguments are passed through to `loadgen-runner`.
+
+#### Environment
+
+- `OUTPUT_DIR`: directory containing output artifacts. Creates temporary folder derived from revision if not set (`/tmp/agoric-sdk-out-{SDK_REVISION}`)
+- `SDK_SRC`: directory containing `agoric-sdk` repo. Creates temporary folder if not set (`/tmp/agoric-sdk-src-{SDK_REVISION}`)
+- `SDK_REVISION`: The agoric-sdk git revision to checkout for the test, if no existing repo found. Remote head if not set
+
+#### Examples
+
+```sh
+SDK_REVISION=fa7ff5e55e OUTPUT_DIR=$HOME/loadgen-output ./start.sh
+```
+
+```sh
+SDK_SRC=../agoric-sdk ./start.sh --stage.duration=10
+```
+
+### Docker image
+
+The Docker image provides a Linux Debian environment setup with all dependencies to allow compiling the agoric-sdk. The entrypoint executes the start script, and has mount points for output directory and optionally an SDK repo.
+
+#### Mount points
+
+- `/out`: directory containing output artifacts
+- `/src`: directory containing `agoric-sdk` repo. Automatically checked out if not a git repo (empty)
+
+#### Environment
+
+- `SDK_REVISION`: The agoric-sdk git revision to checkout for the test
+
+#### Examples
+
+First build the image:
+
+```sh
+docker build . -t loadgen-runner
+```
+
+To perform a loadgen cycle on a given revision:
+
+```sh
+OUTPUT_DIR=$HOME/loadgen-output
+mkdir -p $OUTPUT_DIR
+docker run --rm -v $OUTPUT_DIR:/out -e SDK_REVISION=fa7ff5e55e loadgen-runner --no-reset
+```
+
+To use an existing agoric-sdk copy
+
+```sh
+OUTPUT_DIR=$HOME/loadgen-output
+mkdir -p $OUTPUT_DIR
+docker run --rm -v $OUTPUT_DIR:/out -v ../agoric-sdk:/src loadgen-runner  --no-reset --stage.duration=10
+```
+
+### `run-daily-perf.sh` script
+
+The script is used as a service entrypoint to continuously execute a `loadgen-runner` docker image against the latest SDK revision with the default options. It creates output folders in the current working directory based on the latest revision. The script waits for a new revision to be available if results already exist.
+
+## Manual
+
+The loadgen is implemented as a dapp deploy script which runs forever, and opens an HTTP server on a local port to receive config updates.
+
+### Example
+
 In one terminal:
 
 ```sh
@@ -42,12 +154,15 @@ curl -X PUT --data '{"faucet":{"interval":60}}' http://127.0.0.1:3352/config
 ```
 
 To disable all generators:
+
 ```sh
 curl -X PUT --data '{}' http://127.0.0.1:3352/config
 ```
 
+### Loadgen types
+
 The load generators defined so far:
 
-* `faucet`: initialize by creating a `dapp-fungible-faucet` -style mint, then each cycle requests an invitation and completes it, adding 1000 Tokens to Bob's Purse. Takes 4 round-trips to complete.
-* `amm`: initialize by selling half our BLD to get RUN, then record the balances. Each cycle sells 1% of the recorded BLD to get RUN, then sells 1% of the recorded RUN to get BLD. Because of fees, the total available will drop slowly over time.
-* `vault`: initialize by recording our BLD balance and the BLD/RUN price. Each cycle deposits 1% of the recorded BLD balance and borrows half its value in RUN, then pays back the loan and recovers the BLD (less fees).
+- `faucet`: initialize by creating a `dapp-fungible-faucet` -style mint, then each cycle requests an invitation and completes it, adding 1000 Tokens to Bob's Purse. Takes 4 round-trips to complete.
+- `amm`: initialize by selling half our BLD to get RUN, then record the balances. Each cycle sells 1% of the recorded BLD to get RUN, then sells 1% of the recorded RUN to get BLD. Because of fees, the total available will drop slowly over time.
+- `vault`: initialize by recording our BLD balance and the BLD/RUN price. Each cycle deposits 1% of the recorded BLD balance and borrows half its value in RUN, then pays back the loan and recovers the BLD (less fees).

_agstate/agoric-servers/.gitignore

@@ -78,7 +78,7 @@ integration-test/transform-tests/output
 
 solo
 dev
-testnet
+testnet-*
 chain
 ve*
-local*
+local-*