From 15db6bf32a7c2c1f109392761cd2a16b51115ac9 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 27 Oct 2023 10:42:26 +0100 Subject: [PATCH] chore: e2e on spots [ci rebuild] (#3068) We spent 21538 compute minutes in e2e jobs in the last 2 days. ![image](https://github.com/AztecProtocol/aztec-packages/assets/5764343/465d2a45-7572-4163-9380-011f8bc66eb5) Note that even in the reduced cost running on smallest docker instance, CCI makes up 2/3rds of the cost! Their machines are *over* 10x the cost of spot instance. Not a *huge* deal today, but if we intend on continuing to scale our collection of e2e tests it could be problematic. For reference, we've double our CCI costs every month for the last 3 months from ~$800 to now ~$5000. (Separate issue, but still hopeful we should eventually be able to run an e2e test in 10-20 seconds rather than minutes). Ignoring spot stuff and price, this PR makes an important step towards the correct build-system abstraction, i.e. build-system is responsible for: * Building docker containers. * Running containers, or compose files. * Handles the conditionality (rebuilding only what needs to be rebuilt). * Handles pulling of dependent containers to do builds or runs of compose. * Handles launching builds/runs on spots if wanting bigger cheap machines or lower cost. * Uploading processing benchmarks. Specifically this PR introduces `cond_run_compose` and `cond_spot_run_compose` scripts for launching built containers from compose scripts. We should *not* be calling custom project scripts from `config.yml`. We should run a container (or compose file), and the running of that container should run whatever script needs to be run. This takes us a step closer to the next phase, which code-generating config.yml from build-manifest.yml. **Please try to fit within this pattern and let's discuss if it doesn't work for some reason**. The code generating is more than just a convenience, but reflects that we have an abstraction that can be defined as a simple set of rules about how to build/run/deploy. It will make it much easier for engineers not familiar with build-system to add projects and for "just-the-right-thing" to happen. If we manage to get to that point, it may become relatively painless to shift the entire CI over to github actions, if so inclined. --- .circleci/config.yml | 287 ++++++++---------- build-system/scripts/build | 2 +- build-system/scripts/cond_run_compose | 35 +++ build-system/scripts/cond_spot_run_compose | 10 + build-system/scripts/query_manifest | 8 +- build-system/scripts/remote_run_script | 2 + build-system/scripts/remote_runner | 2 +- build-system/scripts/setup_env | 3 +- .../scripts/upload_logs_to_s3 | 8 +- build_manifest.yml | 2 + cspell.json | 9 +- .../end-to-end/scripts/cond_run_script.delme | 34 --- .../scripts/docker-compose-browser.yml | 5 +- .../scripts/docker-compose-no-sandbox.yml | 4 +- .../end-to-end/scripts/docker-compose.yml | 5 +- yarn-project/end-to-end/scripts/run_tests | 20 -- .../end-to-end/scripts/run_tests_local | 24 -- 17 files changed, 207 insertions(+), 253 deletions(-) create mode 100755 build-system/scripts/cond_run_compose create mode 100755 build-system/scripts/cond_spot_run_compose rename yarn-project/end-to-end/scripts/upload_logs_to_s3.sh => build-system/scripts/upload_logs_to_s3 (83%) delete mode 100755 yarn-project/end-to-end/scripts/cond_run_script.delme delete mode 100755 yarn-project/end-to-end/scripts/run_tests delete mode 100755 yarn-project/end-to-end/scripts/run_tests_local diff --git a/.circleci/config.yml b/.circleci/config.yml index 79993c70dc7..edecd9b086d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ # The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download # with just enough tools to execute the build system, and launch EC2 instances etc. # -# There are some `join` steps that are just noops. They are just used to produce cleaner graph rendering in CCI. +# There are some `join` steps that are just no-ops. They are just used to produce cleaner graph rendering in CCI. version: 2.1 @@ -61,12 +61,12 @@ checkout: &checkout retry git checkout FETCH_HEAD # Called setup_env to setup a bunch of global variables used throughout the rest of the build process. -# It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of +# It takes the required CCI environment variables as inputs, and gives them normalized names for the rest of # the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds. setup_env: &setup_env run: name: "Setup environment" - command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" + command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" # This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them save_logs: &save_logs @@ -604,378 +604,345 @@ jobs: command: build end-to-end e2e-2-pxes: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_2_pxes.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts e2e-multiple-accounts-1-enc-key: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts e2e-deploy-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_deploy_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts e2e-lending-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_lending_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts e2e-token-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_token_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts e2e-sandbox-example: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_sandbox_example.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts e2e-block-building: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_block_building.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts e2e-nested-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_nested_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts e2e-non-contract-account: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_non_contract_account.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts e2e-cross-chain-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts e2e-public-cross-chain-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts e2e-public-to-private-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts e2e-account-contracts: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_account_contracts.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts e2e-escrow-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_escrow_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts e2e-pending-commitments-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts e2e-ordering: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_ordering.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts uniswap-trade-on-l1-from-l2: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts integration-archiver-l1-to-l2: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts integration-l1-publisher: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local integration_l1_publisher.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts e2e-cli: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cli.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts e2e-p2p: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_p2p_network.test.ts ./scripts/docker-compose-no-sandbox.yml - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts e2e-browser-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-browser.yml - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts e2e-card-game: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_card_game.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts pxe-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local pxe_sandbox.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts cli-docs-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local cli_docs_sandbox.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts guides-writing-an-account-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/writing_an_account_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts guides-dapp-testing: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/dapp_testing.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts guides-sample-dapp: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local sample-dapp - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp guides-up-quick-start: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/up_quick_start.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts bench-publish-rollup: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Benchmark" - command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts ./scripts/docker-compose-no-sandbox.yml - environment: - DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees bench-process-history: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Benchmark" - command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_process_history.test.ts ./scripts/docker-compose-no-sandbox.yml - environment: - DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees build-docs: machine: diff --git a/build-system/scripts/build b/build-system/scripts/build index ff7c81d82b0..904ed2b3107 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -105,7 +105,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. TAG=$ECR_DEPLOY_URL/$PARENT_REPO - retry docker tag $PARENT_IMAGE_URI $TAG + docker tag $PARENT_IMAGE_URI $TAG done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose new file mode 100755 index 00000000000..58eef99c71a --- /dev/null +++ b/build-system/scripts/cond_run_compose @@ -0,0 +1,35 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +COMPOSE_FILE=$2 +shift 2 + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +BASE_TAG=cache-$CONTENT_HASH +SUCCESS_TAG=$BASE_TAG-$JOB_NAME + +echo "Content hash: $CONTENT_HASH" + +if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then + # Login to pull our ecr images with docker. + ecr_login + + # For each dependency and self, pull in the latest image and give it correct tag. + for REPO in $(query_manifest runDependencies $REPOSITORY $JOB_NAME) $REPOSITORY; do + echo "Pulling $REPO..." + REPO_IMAGE_URI=$(calculate_image_uri $REPO) + retry docker pull $REPO_IMAGE_URI + docker tag $REPO_IMAGE_URI aztecprotocol/$REPO + done + + cd $(query_manifest projectDir $REPOSITORY) + + export $@ + docker-compose -f $COMPOSE_FILE run $REPOSITORY + + upload_logs_to_s3 log + + retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG +fi diff --git a/build-system/scripts/cond_spot_run_compose b/build-system/scripts/cond_spot_run_compose new file mode 100755 index 00000000000..b10cc9decf5 --- /dev/null +++ b/build-system/scripts/cond_spot_run_compose @@ -0,0 +1,10 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +CPUS=$2 +shift 2 + +export TAG_POSTFIX=$JOB_NAME +cond_spot_run_script $REPOSITORY $CPUS cond_run_compose $REPOSITORY $@ diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index edf5f62b6fd..0358fbf89cb 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -75,6 +75,12 @@ case "$CMD" in dependencies) yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST ;; + runDependencies) + echo $REPO + RUN_NAME=$3 + yq -r ".\"$REPO\".runDependencies // [] | .[]" $MANIFEST + yq -r ".\"$REPO\".run.\"$RUN_NAME\".dependencies // [] | .[]" $MANIFEST + ;; allDependencies) # Get dependencies for a given repo. Inclusive of repo itself. # If a string, attempt to execute / if exists to get dependencies, else error. @@ -94,8 +100,6 @@ case "$CMD" in ALL_DEPS+=("$1") } add_deps $REPO - # Remove ourself as a dependency. - # unset ALL_DEPS[-1] for DEP in "${ALL_DEPS[@]}"; do echo $DEP done diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 5ad1a850e14..8aa4e0ea64f 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -24,6 +24,8 @@ ssh -A -F $SSH_CONFIG_PATH $IP " export COMMIT_TAG=$COMMIT_TAG export JOB_NAME=$JOB_NAME export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL + export BRANCH=${BRANCH:-} + export PULL_REQUEST=${PULL_REQUEST:-} export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD export ECR_DEPLOY_URL=$ECR_DEPLOY_URL export ECR_URL=$ECR_URL diff --git a/build-system/scripts/remote_runner b/build-system/scripts/remote_runner index 243d98e47bf..ce1a567a198 100755 --- a/build-system/scripts/remote_runner +++ b/build-system/scripts/remote_runner @@ -22,6 +22,6 @@ echo "Git checkout completed." BASH_ENV=/tmp/bash_env echo "Calling setup env..." -source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" +source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" "$BRANCH" "$PULL_REQUEST" echo "Calling $@..." $@ diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 8e436a9ca8b..60880473bac 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -13,7 +13,7 @@ COMMIT_TAG=${2##*aztec-packages-} JOB_NAME=$3 GIT_REPOSITORY_URL=${4:-} BRANCH=${5:-} -COMMIT_MESSAGE=${6:-} +PULL_REQUEST=${6:-} BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) @@ -87,6 +87,7 @@ echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV +echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV # Only run the following if we're the result of a commit (i.e. not being run manually). if [ -n "$COMMIT_HASH" ]; then diff --git a/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh b/build-system/scripts/upload_logs_to_s3 similarity index 83% rename from yarn-project/end-to-end/scripts/upload_logs_to_s3.sh rename to build-system/scripts/upload_logs_to_s3 index eb81df8bfac..ede934edae5 100755 --- a/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh +++ b/build-system/scripts/upload_logs_to_s3 @@ -8,8 +8,8 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu +LOG_FOLDER=$1 BUCKET_NAME="aztec-ci-artifacts" -LOG_FOLDER="${LOG_FOLDER:-log}" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then @@ -18,10 +18,10 @@ if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then fi # Duplicated in scripts/ci/assemble_e2e_benchmark.sh -if [ "${CIRCLE_BRANCH:-}" = "master" ]; then +if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" -elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" fi if [ -n "${TARGET_FOLDER:-}" ]; then diff --git a/build_manifest.yml b/build_manifest.yml index dc194b390e7..6bf170c5c7b 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -166,6 +166,8 @@ end-to-end: projectDir: yarn-project/end-to-end dependencies: - yarn-project + runDependencies: + - aztec-sandbox aztec-node: buildDir: yarn-project diff --git a/cspell.json b/cspell.json index b4ae2cbad6d..00506c7af67 100644 --- a/cspell.json +++ b/cspell.json @@ -27,6 +27,8 @@ "camelcase", "cbind", "cbinds", + "cimg", + "cpus", "chainsafe", "cheatcode", "cheatcodes", @@ -37,6 +39,7 @@ "comlink", "composability", "concat", + "cond", "customizability", "danlee", "Daos", @@ -73,6 +76,7 @@ "Kademlia", "keccak", "keypairs", + "keyscan", "leveldb", "leveldown", "leveljs", @@ -132,6 +136,7 @@ "sload", "snakecase", "solhint", + "stdlib", "struct", "structs", "subrepo", @@ -178,7 +183,5 @@ "*.snap", "package.json" ], - "flagWords": [ - "anonymous" - ] + "flagWords": ["anonymous"] } diff --git a/yarn-project/end-to-end/scripts/cond_run_script.delme b/yarn-project/end-to-end/scripts/cond_run_script.delme deleted file mode 100755 index df429289137..00000000000 --- a/yarn-project/end-to-end/scripts/cond_run_script.delme +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Conditionally runs a script if any dependent code has changed between -# the last successful run and the present commit. -# -# It's expected to be run from the project directory, and that there be a directory called `scripts` -# containing the given named script to execute. -# -# This script is only useful if there is nothing to do in the event there is no rebuild. This is fine -# for running a suite of tests for example, but is not useful for performing a build, as even if a -# build has nothing to do, the previous images are retagged with the new commit hash for upstream jobs. -# -# Arguments are: -# 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. -# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a -# successful run. The script will only run if there were relevant code changes since the last successful commit. -# 3... ARGS: Script to run and args. -set -eu -set -x - -REPOSITORY=$1 -shift -SUCCESS_TAG=$1 -shift -SCRIPT_TO_RUN=$1 -shift - -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" -echo "Script to run is $SCRIPT_TO_RUN $@" - -if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then - "$SCRIPT_TO_RUN" "$@" - retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG -fi diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index 5d1e8dbc13b..ddb5838000b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -30,7 +30,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -38,3 +38,6 @@ services: entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts'] volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - sandbox + - fork diff --git a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml index bd2020709e5..76d045ebc4b 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml @@ -15,7 +15,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -28,3 +28,5 @@ services: command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - fork diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index ed5880dad89..85c90b07cd6 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -30,7 +30,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' LOG_LEVL: 'debug' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 @@ -39,3 +39,6 @@ services: command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - sandbox + - fork diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests deleted file mode 100755 index 97b1effcc9e..00000000000 --- a/yarn-project/end-to-end/scripts/run_tests +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). -# It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -export TEST=$1 -export COMPOSE_FILE=${2:-docker-compose.yml} - -$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -for REPO in aztec.js end-to-end aztec-sandbox; do - retry docker pull $($calculate_image_uri $REPO) - retry docker tag $($calculate_image_uri $REPO) aztecprotocol/$REPO:latest -done - -docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local deleted file mode 100755 index 3df64ff4061..00000000000 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see config.yml and cond_run_script). -# It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -export TEST=$1 -export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} - -# Login to ECR. -ecr_login - -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -for REPO in end-to-end aztec-sandbox; do - retry docker pull $(calculate_image_uri $REPO) - retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest -done - -docker-compose -f $COMPOSE_FILE rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end - -# Upload the logs with metrics to S3 -./scripts/upload_logs_to_s3.sh \ No newline at end of file