chore: e2e on spots [ci rebuild] (#3068)

We spent 21538 compute minutes in e2e jobs in the last 2 days.


![image](https://github.com/AztecProtocol/aztec-packages/assets/5764343/465d2a45-7572-4163-9380-011f8bc66eb5)

Note that even in the reduced cost running on smallest docker instance,
CCI makes up 2/3rds of the cost! Their machines are *over* 10x the cost
of spot instance.

Not a *huge* deal today, but if we intend on continuing to scale our
collection of e2e tests it could be problematic.
For reference, we've double our CCI costs every month for the last 3
months from ~$800 to now ~$5000.

(Separate issue, but still hopeful we should eventually be able to run
an e2e test in 10-20 seconds rather than minutes).

Ignoring spot stuff and price, this PR makes an important step towards
the correct build-system abstraction, i.e. build-system is responsible
for:
* Building docker containers.
* Running containers, or compose files.
* Handles the conditionality (rebuilding only what needs to be rebuilt).
* Handles pulling of dependent containers to do builds or runs of
compose.
* Handles launching builds/runs on spots if wanting bigger cheap
machines or lower cost.
* Uploading processing benchmarks.

Specifically this PR introduces `cond_run_compose` and
`cond_spot_run_compose` scripts for launching built containers from
compose scripts.

We should *not* be calling custom project scripts from `config.yml`. We
should run a container (or compose file), and the running of that
container should run whatever script needs to be run. This takes us a
step closer to the next phase, which code-generating config.yml from
build-manifest.yml. **Please try to fit within this pattern and let's
discuss if it doesn't work for some reason**.

The code generating is more than just a convenience, but reflects that
we have an abstraction that can be defined as a simple set of rules
about how to build/run/deploy. It will make it much easier for engineers
not familiar with build-system to add projects and for
"just-the-right-thing" to happen. If we manage to get to that point, it
may become relatively painless to shift the entire CI over to github
actions, if so inclined.

build-system/scripts/build

@@ -105,7 +105,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do
   fetch_image $PARENT_IMAGE_URI
   # Tag it to look like an official release as that's what we use in Dockerfiles.
   TAG=$ECR_DEPLOY_URL/$PARENT_REPO
-  retry docker tag $PARENT_IMAGE_URI $TAG
+  docker tag $PARENT_IMAGE_URI $TAG
 done
 
 COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false)

build-system/scripts/cond_run_compose

@@ -0,0 +1,35 @@
+#!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+COMPOSE_FILE=$2
+shift 2
+
+CONTENT_HASH=$(calculate_content_hash $REPOSITORY)
+BASE_TAG=cache-$CONTENT_HASH
+SUCCESS_TAG=$BASE_TAG-$JOB_NAME
+
+echo "Content hash: $CONTENT_HASH"
+
+if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then
+  # Login to pull our ecr images with docker.
+  ecr_login
+
+  # For each dependency and self, pull in the latest image and give it correct tag.
+  for REPO in $(query_manifest runDependencies $REPOSITORY $JOB_NAME) $REPOSITORY; do
+    echo "Pulling $REPO..."
+    REPO_IMAGE_URI=$(calculate_image_uri $REPO)
+    retry docker pull $REPO_IMAGE_URI
+    docker tag $REPO_IMAGE_URI aztecprotocol/$REPO
+  done
+
+  cd $(query_manifest projectDir $REPOSITORY)
+
+  export $@
+  docker-compose -f $COMPOSE_FILE run $REPOSITORY
+
+  upload_logs_to_s3 log
+
+  retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG
+fi

build-system/scripts/cond_spot_run_compose

@@ -0,0 +1,10 @@
+#!/bin/bash
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+REPOSITORY=$1
+CPUS=$2
+shift 2
+
+export TAG_POSTFIX=$JOB_NAME
+cond_spot_run_script $REPOSITORY $CPUS cond_run_compose $REPOSITORY $@

build-system/scripts/query_manifest

@@ -75,6 +75,12 @@ case "$CMD" in
   dependencies)
     yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST
     ;;
+  runDependencies)
+    echo $REPO
+    RUN_NAME=$3
+    yq -r ".\"$REPO\".runDependencies // [] | .[]" $MANIFEST
+    yq -r ".\"$REPO\".run.\"$RUN_NAME\".dependencies // [] | .[]" $MANIFEST
+    ;;
   allDependencies)
     # Get dependencies for a given repo. Inclusive of repo itself.
     # If a string, attempt to execute <projectDir>/<string> if exists to get dependencies, else error.
@@ -94,8 +100,6 @@ case "$CMD" in
       ALL_DEPS+=("$1")
     }
     add_deps $REPO
-    # Remove ourself as a dependency.
-    # unset ALL_DEPS[-1]
     for DEP in "${ALL_DEPS[@]}"; do
       echo $DEP
     done

build-system/scripts/remote_run_script

@@ -24,6 +24,8 @@ ssh -A -F $SSH_CONFIG_PATH $IP "
   export COMMIT_TAG=$COMMIT_TAG
   export JOB_NAME=$JOB_NAME
   export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL
+  export BRANCH=${BRANCH:-}
+  export PULL_REQUEST=${PULL_REQUEST:-}
   export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
   export ECR_DEPLOY_URL=$ECR_DEPLOY_URL
   export ECR_URL=$ECR_URL

build-system/scripts/remote_runner

@@ -22,6 +22,6 @@ echo "Git checkout completed."
 
 BASH_ENV=/tmp/bash_env
 echo "Calling setup env..."
-source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL"
+source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" "$BRANCH" "$PULL_REQUEST"
 echo "Calling $@..."
 $@

build-system/scripts/setup_env

@@ -13,7 +13,7 @@ COMMIT_TAG=${2##*aztec-packages-}
 JOB_NAME=$3
 GIT_REPOSITORY_URL=${4:-}
 BRANCH=${5:-}
-COMMIT_MESSAGE=${6:-}
+PULL_REQUEST=${6:-}
 
 BASH_ENV=${BASH_ENV:-}
 BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
@@ -87,6 +87,7 @@ echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV
 echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV
 echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV
 echo export BRANCH=$BRANCH >> $BASH_ENV
+echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV
 
 # Only run the following if we're the result of a commit (i.e. not being run manually).
 if [ -n "$COMMIT_HASH" ]; then

yarn-project/end-to-end/scripts/upload_logs_to_s3.sh → build-system/scripts/upload_logs_to_s3

@@ -8,8 +8,8 @@
 [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
 set -eu
 
+LOG_FOLDER=$1
 BUCKET_NAME="aztec-ci-artifacts"
-LOG_FOLDER="${LOG_FOLDER:-log}"
 COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
 
 if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then
@@ -18,10 +18,10 @@ if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then
 fi
 
 # Duplicated in scripts/ci/assemble_e2e_benchmark.sh
-if [ "${CIRCLE_BRANCH:-}" = "master" ]; then
+if [ "${BRANCH:-}" = "master" ]; then
   TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/"
-elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then
-  TARGET_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}"
+elif [ -n "${PULL_REQUEST:-}" ]; then
+  TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}"
 fi
 
 if [ -n "${TARGET_FOLDER:-}" ]; then

build_manifest.yml

@@ -166,6 +166,8 @@ end-to-end:
   projectDir: yarn-project/end-to-end
   dependencies:
     - yarn-project
+  runDependencies:
+    - aztec-sandbox
 
 aztec-node:
   buildDir: yarn-project

cspell.json

@@ -27,6 +27,8 @@
     "camelcase",
     "cbind",
     "cbinds",
+    "cimg",
+    "cpus",
     "chainsafe",
     "cheatcode",
     "cheatcodes",
@@ -37,6 +39,7 @@
     "comlink",
     "composability",
     "concat",
+    "cond",
     "customizability",
     "danlee",
     "Daos",
@@ -73,6 +76,7 @@
     "Kademlia",
     "keccak",
     "keypairs",
+    "keyscan",
     "leveldb",
     "leveldown",
     "leveljs",
@@ -132,6 +136,7 @@
     "sload",
     "snakecase",
     "solhint",
+    "stdlib",
     "struct",
     "structs",
     "subrepo",
@@ -178,7 +183,5 @@
     "*.snap",
     "package.json"
   ],
-  "flagWords": [
-    "anonymous"
-  ]
+  "flagWords": ["anonymous"]
 }

yarn-project/end-to-end/scripts/docker-compose-browser.yml

@@ -30,11 +30,14 @@ services:
   end-to-end:
     image: aztecprotocol/end-to-end:latest
     environment:
-      BENCHMARK: true
+      BENCHMARK: 'true'
       DEBUG: ${DEBUG:-'aztec:*'}
       ETHEREUM_HOST: http://fork:8545
       CHAIN_ID: 31337
       PXE_URL: http://sandbox:8080
     entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts']
     volumes:
       - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - sandbox
+      - fork

yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml

@@ -15,7 +15,7 @@ services:
   end-to-end:
     image: aztecprotocol/end-to-end:latest
     environment:
-      BENCHMARK: true
+      BENCHMARK: 'true'
       DEBUG: ${DEBUG:-'aztec:*'}
       ETHEREUM_HOST: http://fork:8545
       CHAIN_ID: 31337
@@ -28,3 +28,5 @@ services:
     command: ${TEST:-./src/e2e_deploy_contract.test.ts}
     volumes:
       - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - fork

yarn-project/end-to-end/scripts/docker-compose.yml

@@ -30,7 +30,7 @@ services:
   end-to-end:
     image: aztecprotocol/end-to-end:latest
     environment:
-      BENCHMARK: true
+      BENCHMARK: 'true'
       LOG_LEVL: 'debug'
       DEBUG: ${DEBUG:-'aztec:*'}
       ETHEREUM_HOST: http://fork:8545
@@ -39,3 +39,6 @@ services:
     command: ${TEST:-./src/e2e_deploy_contract.test.ts}
     volumes:
       - ../log:/usr/src/yarn-project/end-to-end/log:rw
+    depends_on:
+      - sandbox
+      - fork