From 52827b0849f19b82caf0da160faaeffe01ad0911 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 15 Mar 2024 14:42:13 -0300 Subject: [PATCH 001/161] chore: Load bb benches into aggregate benchmark Loads any json files in google benchmark format and adds them to the bench aggregation and to the markdown comment. For instance, given a file for ultrahonk, the markdown shows: ``` Average proving times for benchmarks measured in bb. | Circuit | 2^20 gates | | - | - | ultrahonk_proving_time_in_ms | 29.9 | ``` --- .../circuit-types/src/stats/metrics.ts | 15 ++++++++- .../scripts/src/benchmarks/aggregate.ts | 31 +++++++++++++++++++ .../scripts/src/benchmarks/markdown.ts | 8 ++++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index a27e8c6f0b4..1c9d68df540 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -7,7 +7,8 @@ export type MetricGroupBy = | 'circuit-name' | 'classes-registered' | 'leaf-count' - | 'data-writes'; + | 'data-writes' + | 'circuit-size-in-gates'; /** Definition of a metric to track in benchmarks. */ export interface Metric { @@ -23,6 +24,18 @@ export interface Metric { /** Metric definitions to track from benchmarks. */ export const Metrics = [ + { + name: 'client_ivc_proving_time_in_ms', + groupBy: 'circuit-size-in-gates', + description: 'Proving time for ClientIVC grouped by circuit size.', + events: [], + }, + { + name: 'ultrahonk_proving_time_in_ms', + groupBy: 'circuit-size-in-gates', + description: 'Proving time for UltraHonk grouped by circuit size.', + events: [], + }, { name: 'l1_rollup_calldata_size_in_bytes', groupBy: 'block-size', diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index 54bdb56dd4f..121369632d1 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -219,6 +219,15 @@ function processEntry(entry: Stats, results: BenchmarkCollectedResults) { } } +function getBarretenbergMetric(context: { executable?: string }): MetricName | undefined { + if (context.executable?.includes('ultra_honk')) { + return 'ultrahonk_proving_time_in_ms'; + } else if (context.executable?.includes('client_ivc')) { + return 'client_ivc_proving_time_in_ms'; + } + return undefined; +} + /** Array of collected raw results for a given metric. */ type BenchmarkCollectedMetricResults = Record; @@ -260,6 +269,28 @@ export async function main() { } } + // Add google benchmark json files, which have data already averaged + const googleBenchmarkFiles = fs.readdirSync(LogsDir).filter(f => f.endsWith('.json')); + for (const file of googleBenchmarkFiles) { + const data = JSON.parse(fs.readFileSync(path.join(LogsDir, file), 'utf-8')); + if (!data.context || !data.benchmarks) { + log(`Invalid google benchmark file: ${file}`); + continue; + } + const metric = getBarretenbergMetric(data.context); + if (!metric) { + log(`Unknown executable in benchmark file ${file}: ${data.context.executable}`); + continue; + } + const circuitSize = '2^20'; // Where to load size from? + const value = data.benchmarks[0]?.real_time; + if (value === undefined) { + log(`Couldn't find real_time in benchmark file ${file}`); + continue; + } + results[metric] = { [circuitSize]: value }; + } + const timestampedResults: BenchmarkResultsWithTimestamp = { ...results, timestamp: new Date().toISOString() }; // Write results to disk diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index bdd88c5f0c6..f8027ffae66 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -188,6 +188,7 @@ export function getMarkdown() { const metricsByCircuitName = Metrics.filter(m => m.groupBy === 'circuit-name').map(m => m.name); const metricsByClassesRegistered = Metrics.filter(m => m.groupBy === 'classes-registered').map(m => m.name); const metricsByLeafCount = Metrics.filter(m => m.groupBy === 'leaf-count').map(m => m.name); + const metricsByCircuitSize = Metrics.filter(m => m.groupBy === 'circuit-size-in-gates').map(m => m.name); const metricsTxPxeProcessing = Metrics.filter(m => m.name === 'tx_pxe_processing_time_ms').map(m => m.name); const metricsTxSeqProcessing = Metrics.filter(m => m.name === 'tx_sequencer_processing_time_ms').map(m => m.name); @@ -213,10 +214,15 @@ ${getWarningsSummary(benchmark, baseBenchmark)} Detailed results -All benchmarks are run on txs on the \`Benchmarking\` contract on the repository. Each tx consists of a batch call to \`create_note\` and \`increment_balance\`, which guarantees that each tx has a private call, a nested private call, a public call, and a nested public call, as well as an emitted private note, an unencrypted log, and public storage read and write. +Except for proving times, all benchmarks are run on txs on the \`Benchmarking\` contract on the repository. Each tx consists of a batch call to \`create_note\` and \`increment_balance\`, which guarantees that each tx has a private call, a nested private call, a public call, and a nested public call, as well as an emitted private note, an unencrypted log, and public storage read and write. ${prSourceDataText} ${baseCommitText} +### Proving times + +Average proving times for benchmarks measured in bb. +${getTableContent(pick(benchmark, metricsByCircuitSize), baseBenchmark, 'gates', 'Circuit')} + ### L2 block published to L1 Each column represents the number of txs on an L2 block published to L1. From e2f5f2b806dee4b767c5955e82d14fa29b23e749 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 2 Apr 2024 17:04:46 +0000 Subject: [PATCH 002/161] bb logs --- README.md | 2 +- scripts/ci/assemble_e2e_benchmark.sh | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 38e03d97604..5abbc1903d8 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ noirup -v TAG_FROM_THE_FILE This repository uses CircleCI for continuous integration. Build steps are managed using [`build-system`](https://github.com/AztecProtocol/build-system). Small packages are built and tested as part of a docker build operation, while larger ones and end-to-end tests spin up a large AWS spot instance. Each successful build step creates a new docker image that gets tagged with the package name and commit. -All packages need to be included in the [build manifest](`build_manifest.json`), which declares what paths belong to each package, as well as dependencies between packages. When the CI runs, if none of the rebuild patterns or dependencies were changed, then the build step is skipped and the last successful image is re-tagged with the current commit. Read more on the [`build-system`](https://github.com/AztecProtocol/build-system) repository README. +All packages need to be included in the [build manifest](`build_manifest.yml`), which declares what paths belong to each package, as well as dependencies between packages. When the CI runs, if none of the rebuild patterns or dependencies were changed, then the build step is skipped and the last successful image is re-tagged with the current commit. Read more on the [`build-system`](https://github.com/AztecProtocol/build-system) repository README. It is faster to debug CI failures within a persistent ssh session compared to pushing and waiting. You can create a session with "Rerun step with SSH" on CircleCI which will generate an ssh command for debugging on a worker. Run that command locally and then do diff --git a/scripts/ci/assemble_e2e_benchmark.sh b/scripts/ci/assemble_e2e_benchmark.sh index 811bef01e6f..82ccb44e5fa 100755 --- a/scripts/ci/assemble_e2e_benchmark.sh +++ b/scripts/ci/assemble_e2e_benchmark.sh @@ -7,6 +7,7 @@ set -eu BUCKET_NAME="aztec-ci-artifacts" +BARRETENBERG_BUCKET_NAME="aztec-ci-artifacts" LOG_FOLDER="${LOG_FOLDER:-log}" BENCH_FOLDER="${BENCH_FOLDER:-bench}" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" @@ -17,10 +18,12 @@ BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" # Paths from build-system/scripts/upload_logs_to_s3 if [ "${CIRCLE_BRANCH:-}" = "master" ]; then LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then LOG_SOURCE_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${CIRCLE_PULL_REQUEST##*/}.json" elif [ -n "${CIRCLE_TAG:-}" ]; then echo "Skipping benchmark run for ${CIRCLE_TAG} tagged release." @@ -44,9 +47,12 @@ EXPECTED_LOGS_COUNT=$(find yarn-project/end-to-end/src -type f -name "bench*.tes DOWNLOADED_LOGS_COUNT=$(find $LOG_FOLDER -type f -name "*.jsonl" | wc -l) if [ "$DOWNLOADED_LOGS_COUNT" -lt "$EXPECTED_LOGS_COUNT" ]; then echo Found $DOWNLOADED_LOGS_COUNT out of $EXPECTED_LOGS_COUNT benchmark log files in s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/. Exiting. - exit 0 + exit 1 fi +# Download barretenberg log files, these are direct benchmarks and separate from the above +aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive + # Generate the aggregated benchmark file mkdir -p $BENCH_FOLDER CONTAINER_BENCH_FOLDER="/usr/src/yarn-project/bench" From 20dcbaf13d157131cd798fe8a9043783ae1a8888 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 2 Apr 2024 18:09:45 +0000 Subject: [PATCH 003/161] run earthly pure local --- .github/workflows/ci.yml | 7 ++++++- build-system/scripts/upload_logs_to_s3 | 2 +- yarn-project/end-to-end/scripts/docker-compose.yml | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fead02fb6f..31ac6151094 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,12 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ run: | - earthly-cloud build x86 --no-output +${{ matrix.test }} --e2e_mode=cache + # we blank earthly token just to be sure this is a pure local run + EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + + - name: Upload logs + run: | + scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: diff --git a/build-system/scripts/upload_logs_to_s3 b/build-system/scripts/upload_logs_to_s3 index 3188a80ec0f..f06aa521725 100755 --- a/build-system/scripts/upload_logs_to_s3 +++ b/build-system/scripts/upload_logs_to_s3 @@ -21,7 +21,7 @@ fi if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" elif [ -n "${PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" + TARGET_FOLDER="logs-v1/pulls/${PAULL_REQUEST##*/}" fi if [ -n "${TARGET_FOLDER:-}" ]; then diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 8d0c71158a8..c4b424e372d 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -40,7 +40,6 @@ services: PXE_URL: http://sandbox:8080 command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - # TODO(AD) currently earthly uses /build instead of /usr/src - ../log:/usr/src/yarn-project/end-to-end/log:rw depends_on: - sandbox From ebf1fe883d5652de5bdebb93c8df8e5a2fc7d2d8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 2 Apr 2024 19:37:00 +0000 Subject: [PATCH 004/161] upload to s3 --- scripts/ci/upload_logs_to_s3 | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 scripts/ci/upload_logs_to_s3 diff --git a/scripts/ci/upload_logs_to_s3 b/scripts/ci/upload_logs_to_s3 new file mode 100755 index 00000000000..eb7e1eeb6c7 --- /dev/null +++ b/scripts/ci/upload_logs_to_s3 @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Uploads to S3 the contents of the log file mounted on the end-to-end container, +# which contains log entries with an associated event and metrics for it. +# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl +# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR + +set -eu + +LOG_FOLDER=$1 +BUCKET_NAME="aztec-ci-artifacts" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then + echo "No logs in folder $LOG_FOLDER to upload" + exit 0 +fi + +# Paths used in scripts/ci/assemble_e2e_benchmark.sh +if [ "${BRANCH:-}" = "master" ]; then + TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="logs-v1/pulls/${PAULL_REQUEST##*/}" +fi + +if [ -n "${TARGET_FOLDER:-}" ]; then + aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive +else + echo Skipping upload since no target folder was defined +fi \ No newline at end of file From 7871f491d5d7d4de3e0becf6f2c7fbdb4841860c Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 2 Apr 2024 19:54:57 +0000 Subject: [PATCH 005/161] branch --- .github/workflows/ci.yml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 066199eda34..b5495f6010e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,9 +54,8 @@ jobs: test: - e2e-card-game - e2e-crowdfunding-and-claim - # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id|| github.ref_name }}-arm + group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-arm cancel-in-progress: true steps: - name: Checkout @@ -122,7 +121,7 @@ jobs: - name: Upload logs run: | - scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: @@ -161,7 +160,7 @@ jobs: # they should use parallelism within the benchmark, but only one thing should run at a time # for accurate results # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. - bench: + bb-bench: runs-on: ubuntu-latest env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -204,3 +203,18 @@ jobs: - name: Ultrahonk Bench working-directory: ./barretenberg/cpp/ run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + + aztec-bench-summary: + runs-on: ubuntu-latest + needs: e2e-x86 + concurrency: + group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: "Assemble benchmark summary from uploaded logs" + command: ./scripts/ci/assemble_e2e_benchmark.sh From bb8b5bb39cb718a8da1a73edab41393b9634551b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 2 Apr 2024 20:09:47 +0000 Subject: [PATCH 006/161] fix --- .github/workflows/ci.yml | 1 + yarn-project/end-to-end/Earthfile | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5495f6010e..f19b1bd0719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,6 +204,7 @@ jobs: working-directory: ./barretenberg/cpp/ run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + # Post actions, deploy and summarize logs aztec-bench-summary: runs-on: ubuntu-latest needs: e2e-x86 diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index f88d7ca9b44..d646e2e9465 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -18,7 +18,7 @@ E2E_TEST_LOCAL: --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN docker compose -f $compose_file up --exit-code-from=sandbox --force-recreate + RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END # run locally and take from cache, used for our mainly x86 jobs @@ -53,7 +53,7 @@ E2E_TEST_FROM_BUILD: --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN docker compose -f $compose_file up --exit-code-from=sandbox --force-recreate + RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END E2E_TEST: @@ -259,11 +259,11 @@ bench-publish-rollup: DO +E2E_TEST --test=benchmarks/bench_publish_rollup.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml # TODO need to investigate why this isn't working -# bench-process-history: -# ARG e2e_mode=local -# DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml +bench-process-history: + ARG e2e_mode=local + DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml # TODO need to investigate why this isn't working -# bench-tx-size: -# ARG e2e_mode=local -# DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml +bench-tx-size: + ARG e2e_mode=local + DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml From 34644be4c4faed4ab06dd9293e5b36f15c8cf0cc Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 2 Apr 2024 20:25:00 +0000 Subject: [PATCH 007/161] fix --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- yarn-project/Earthfile | 5 ++++- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f19b1bd0719..d9ac6c102ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -204,18 +204,18 @@ jobs: working-directory: ./barretenberg/cpp/ run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache - # Post actions, deploy and summarize logs - aztec-bench-summary: - runs-on: ubuntu-latest - needs: e2e-x86 - concurrency: - group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: "Assemble benchmark summary from uploaded logs" - command: ./scripts/ci/assemble_e2e_benchmark.sh + # # Post actions, deploy and summarize logs + # aztec-bench-summary: + # runs-on: ubuntu-latest + # needs: e2e-x86 + # concurrency: + # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # cancel-in-progress: true + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.sha }} + + # - name: "Assemble benchmark summary from uploaded logs" + # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index b62d02875d1..4ff45e5bd57 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -86,4 +86,7 @@ build-end-to-end: FROM +end-to-end-minimal SAVE IMAGE --push aztecprotocol/end-to-end-cache:$EARTHLY_GIT_HASH FROM +aztec - SAVE IMAGE --push aztecprotocol/aztec-cache:$EARTHLY_GIT_HASH \ No newline at end of file + SAVE IMAGE --push aztecprotocol/aztec-cache:$EARTHLY_GIT_HASH + +aggregate-bench: + \ No newline at end of file From 2b90f7075aa90451b03a98574c2da2496424497c Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 03:15:44 +0000 Subject: [PATCH 008/161] working comment --- scripts/ci/assemble_e2e_benchmark.sh | 9 ++++----- scripts/ci/upload_logs_to_s3 | 2 ++ yarn-project/Earthfile | 5 +---- yarn-project/scripts/src/benchmarks/aggregate.ts | 3 ++- yarn-project/scripts/src/benchmarks/markdown.ts | 11 ++++++++--- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/scripts/ci/assemble_e2e_benchmark.sh b/scripts/ci/assemble_e2e_benchmark.sh index 82ccb44e5fa..6ba69863526 100755 --- a/scripts/ci/assemble_e2e_benchmark.sh +++ b/scripts/ci/assemble_e2e_benchmark.sh @@ -7,11 +7,10 @@ set -eu BUCKET_NAME="aztec-ci-artifacts" -BARRETENBERG_BUCKET_NAME="aztec-ci-artifacts" LOG_FOLDER="${LOG_FOLDER:-log}" BENCH_FOLDER="${BENCH_FOLDER:-bench}" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" -BASE_COMMIT_HASH="" +BASE_BENCH_PATH="" BENCHMARK_FILE_JSON="${BENCH_FOLDER}/benchmark.json" BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" @@ -62,7 +61,7 @@ export DOCKER_RUN_OPTS="\ -e BENCH_FOLDER=${CONTAINER_BENCH_FOLDER} \ -v $(realpath $LOG_FOLDER):${CONTAINER_LOG_FOLDER}:rw \ -e LOG_FOLDER=${CONTAINER_LOG_FOLDER} \ - -e BASE_COMMIT_HASH \ + -e BASE_BENCH_PATH \ -e AZTEC_BOT_COMMENTER_GITHUB_TOKEN \ -e CIRCLE_PULL_REQUEST" yarn-project/scripts/run_script.sh workspace @aztec/scripts bench-aggregate @@ -88,13 +87,13 @@ if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then aws s3 cp "s3://${BUCKET_NAME}/benchmarks-v1/master/$commit_hash.json" $BASE_BENCHMARK_FILE_JSON if [ $? -eq 0 ]; then echo "Downloaded base data from commit $commit_hash" - export BASE_COMMIT_HASH=$commit_hash + export BASE_BENCH_PATH=master/$commit_hash break; fi done set -e - if [ -z "${BASE_COMMIT_HASH:-}" ]; then + if [ -z "${BASE_BENCH_PATH:-}" ]; then echo "No base commit data found" fi diff --git a/scripts/ci/upload_logs_to_s3 b/scripts/ci/upload_logs_to_s3 index eb7e1eeb6c7..6c7ff024658 100755 --- a/scripts/ci/upload_logs_to_s3 +++ b/scripts/ci/upload_logs_to_s3 @@ -1,5 +1,7 @@ #!/usr/bin/env bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace + # Uploads to S3 the contents of the log file mounted on the end-to-end container, # which contains log entries with an associated event and metrics for it. # Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 4ff45e5bd57..f6b91b721c6 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -32,7 +32,7 @@ build: SAVE ARTIFACT /usr/src # TODO versioning flow at end before publish? -# ENV COMMIT_TAG=$EARTHLY_BUILD_SHA +# ENV COMMIT_TAG=$EARTHLY_GIT_HASH # RUN ./scripts/version_packages.sh # run: @@ -87,6 +87,3 @@ build-end-to-end: SAVE IMAGE --push aztecprotocol/end-to-end-cache:$EARTHLY_GIT_HASH FROM +aztec SAVE IMAGE --push aztecprotocol/aztec-cache:$EARTHLY_GIT_HASH - -aggregate-bench: - \ No newline at end of file diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index ac976e55050..6440cb14e36 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -262,7 +262,8 @@ export async function main() { } } - log(`Collected entries: ${JSON.stringify(collected)}`); + // Spammy if on by default + // log(`Collected entries: ${JSON.stringify(collected)}`); // For each bucket of each metric compute the average all collected data points const results: BenchmarkResults = {}; diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index 85bbbe68cc3..6bd94dc4146 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -3,11 +3,16 @@ import { BENCHMARK_HISTORY_BLOCK_SIZE, Metrics } from '@aztec/circuit-types/stats'; import { createConsoleLogger } from '@aztec/foundation/log'; + + import * as fs from 'fs'; import pick from 'lodash.pick'; + + import { BaseBenchFile, BenchFile } from './paths.js'; + // Input file paths const inputFile = BenchFile; const baseFile = BaseBenchFile; @@ -194,8 +199,8 @@ export function getMarkdown() { const metricsTxPxeProcessing = Metrics.filter(m => m.name === 'tx_pxe_processing_time_ms').map(m => m.name); const metricsTxSeqProcessing = Metrics.filter(m => m.name === 'tx_sequencer_processing_time_ms').map(m => m.name); - const baseHash = process.env.BASE_COMMIT_HASH; - const baseUrl = baseHash && `[\`${baseHash.slice(0, 8)}\`](${S3_URL}/benchmarks-v1/master/${baseHash}.json)`; + const baseUrlPath = process.env.BASE_BENCH_PATH; + const baseUrl = baseUrlPath && `[\`${baseUrlPath.slice(0, 8)}\`](${S3_URL}/benchmarks-v1/master/${baseUrlPath}.json)`; const baseCommitText = baseUrl ? `\nValues are compared against data from master at commit ${baseUrl} and shown if the difference exceeds 1%.` : ''; @@ -264,4 +269,4 @@ ${COMMENT_MARK} /** Entrypoint */ export function main() { log(getMarkdown()); -} +} \ No newline at end of file From 251cfa1c978cf610bc7532363e37b625f1c5a63a Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 03:32:49 +0000 Subject: [PATCH 009/161] typo --- build-system/scripts/upload_logs_to_s3 | 2 +- scripts/ci/upload_logs_to_s3 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/upload_logs_to_s3 b/build-system/scripts/upload_logs_to_s3 index f06aa521725..3188a80ec0f 100755 --- a/build-system/scripts/upload_logs_to_s3 +++ b/build-system/scripts/upload_logs_to_s3 @@ -21,7 +21,7 @@ fi if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" elif [ -n "${PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${PAULL_REQUEST##*/}" + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" fi if [ -n "${TARGET_FOLDER:-}" ]; then diff --git a/scripts/ci/upload_logs_to_s3 b/scripts/ci/upload_logs_to_s3 index 6c7ff024658..b13c0ebd456 100755 --- a/scripts/ci/upload_logs_to_s3 +++ b/scripts/ci/upload_logs_to_s3 @@ -22,7 +22,7 @@ fi if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" elif [ -n "${PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${PAULL_REQUEST##*/}" + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" fi if [ -n "${TARGET_FOLDER:-}" ]; then From 9af57c2bc855e8f2288e6497c0e0a7095c537a8a Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 15:14:30 +0000 Subject: [PATCH 010/161] external collab strategy --- .github/workflows/ci.yml | 26 ++++++++++++++++--- .../workflows/protocol-circuits-gate-diff.yml | 13 +++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9ac6c102ae..4a7cdbf8c66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,12 +2,24 @@ name: Earthly CI on: push: branches: [master] - pull_request: {} + pull_request_target: {types: [opened, synchronize, reopened, labeled]} workflow_dispatch: {} jobs: + check-run-condition: + runs-on: ubuntu-latest + steps: + - name: Check Condition + run: | + if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" && "${{ contains(github.event.pull_request.labels.*.name, 'external-ci') }}" != "true" ]]; then + echo "PR is not from the same repo and doesn't have the 'external-ci' label." + exit 1 + fi + # there's a lot of x86 tasks - let's split out the build step build-x86: + # IMPORTANT security flaw if we don't need 'check-run-condition' + needs: [check-run-condition] runs-on: ubuntu-latest outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} @@ -17,6 +29,8 @@ jobs: concurrency: group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 cancel-in-progress: true + # permission check + if: github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'external-ci') steps: - name: Checkout uses: actions/checkout@v4 @@ -45,6 +59,8 @@ jobs: # all the end-to-end integration tests for aztec e2e-arm: + # IMPORTANT security flaw if we don't need 'check-run-condition' + needs: [check-run-condition] runs-on: ubuntu-latest env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -80,7 +96,8 @@ jobs: # all the end-to-end integration tests for aztec e2e-x86: - needs: build-x86 + # IMPORTANT security flaw if we don't need 'check-run-condition' + needs: [build-x86, check-run-condition] runs-on: ubuntu-latest env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -125,6 +142,8 @@ jobs: # barretenberg (prover) native tests bb-native-tests: + # IMPORTANT security flaw if we don't need 'check-run-condition' + needs: [build-x86, check-run-condition] runs-on: ubuntu-latest env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -207,7 +226,8 @@ jobs: # # Post actions, deploy and summarize logs # aztec-bench-summary: # runs-on: ubuntu-latest - # needs: e2e-x86 + # # IMPORTANT security flaw if we don't need 'check-run-condition' + # needs: [e2e-x86, check-run-condition] # concurrency: # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 # cancel-in-progress: true diff --git a/.github/workflows/protocol-circuits-gate-diff.yml b/.github/workflows/protocol-circuits-gate-diff.yml index 8126a13193c..3205229cbd4 100644 --- a/.github/workflows/protocol-circuits-gate-diff.yml +++ b/.github/workflows/protocol-circuits-gate-diff.yml @@ -4,10 +4,21 @@ on: push: branches: - master - pull_request: + pull_request_target: {types: [opened, synchronize, reopened, labeled]} jobs: + check-run-condition: + runs-on: ubuntu-latest + steps: + - name: Check Condition + run: | + if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" && "${{ contains(github.event.pull_request.labels.*.name, 'external-ci') }}" != "true" ]]; then + echo "PR is not from the same repo and doesn't have the 'external-ci' label." + exit 1 + fi compare_protocol_circuits_gates: + # IMPORTANT security flaw if we don't need 'check-run-condition' + needs: [check-run-condition] concurrency: group: compare_protocol_circuits_gates-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true From 0335e57437b2e7a9dddd484f40df64f64c85edb4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 15:51:28 +0000 Subject: [PATCH 011/161] ci --- yarn-project/scripts/src/benchmarks/markdown.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index 6bd94dc4146..f803cd683ca 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -3,16 +3,11 @@ import { BENCHMARK_HISTORY_BLOCK_SIZE, Metrics } from '@aztec/circuit-types/stats'; import { createConsoleLogger } from '@aztec/foundation/log'; - - import * as fs from 'fs'; import pick from 'lodash.pick'; - - import { BaseBenchFile, BenchFile } from './paths.js'; - // Input file paths const inputFile = BenchFile; const baseFile = BaseBenchFile; @@ -269,4 +264,4 @@ ${COMMENT_MARK} /** Entrypoint */ export function main() { log(getMarkdown()); -} \ No newline at end of file +} From 03ce61528ef88fb16dff7188d19110e3d2b47da4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 19:40:23 +0000 Subject: [PATCH 012/161] grand experiment --- .github/workflows/ci.yml | 452 +++++++++--------- .../workflows/protocol-circuits-gate-diff.yml | 13 +- noir-projects/bootstrap.sh | 5 +- 3 files changed, 235 insertions(+), 235 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a7cdbf8c66..0e9a010cd89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,240 +2,254 @@ name: Earthly CI on: push: branches: [master] - pull_request_target: {types: [opened, synchronize, reopened, labeled]} + pull_request: {} workflow_dispatch: {} jobs: - check-run-condition: + start-runner: + timeout-minutes: 5 # normally it only takes 1-2 minutes + name: Start self-hosted EC2 runner runs-on: ubuntu-latest - steps: - - name: Check Condition - run: | - if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" && "${{ contains(github.event.pull_request.labels.*.name, 'external-ci') }}" != "true" ]]; then - echo "PR is not from the same repo and doesn't have the 'external-ci' label." - exit 1 - fi - - # there's a lot of x86 tasks - let's split out the build step - build-x86: - # IMPORTANT security flaw if we don't need 'check-run-condition' - needs: [check-run-condition] - runs-on: ubuntu-latest - outputs: - e2e_list: ${{ steps.e2e_list.outputs.list }} - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true - # permission check - if: github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'external-ci') - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Build - working-directory: ./yarn-project - run: | - # push to registry - earthly-cloud build x86 --push +build-end-to-end - - # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # (Note ARM uses just 2 tests as a smoketest) - - id: e2e_list - working-directory: ./yarn-project/end-to-end - run: | - echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - - # all the end-to-end integration tests for aztec - e2e-arm: - # IMPORTANT security flaw if we don't need 'check-run-condition' - needs: [check-run-condition] - runs-on: ubuntu-latest - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - strategy: - fail-fast: false - matrix: - test: - - e2e-card-game - - e2e-crowdfunding-and-claim - concurrency: - group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-arm - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Build - working-directory: ./yarn-project/end-to-end/ - # We don't do much on arm, just run it on their builder - run: - # Flags: - # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy - earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build - - # all the end-to-end integration tests for aztec - e2e-x86: - # IMPORTANT security flaw if we don't need 'check-run-condition' - needs: [build-x86, check-run-condition] - runs-on: ubuntu-latest - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - strategy: - fail-fast: false - matrix: - test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Pull E2E Images - working-directory: ./barretenberg/cpp/ - run: | - docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) - docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) - docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest - docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest - - - name: Test - working-directory: ./yarn-project/end-to-end/ - run: | - # we blank earthly token just to be sure this is a pure local run - EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - - name: Upload logs - run: | - BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - - # barretenberg (prover) native tests - bb-native-tests: - # IMPORTANT security flaw if we don't need 'check-run-condition' - needs: [build-x86, check-run-condition] - runs-on: ubuntu-latest - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - strategy: - fail-fast: false - matrix: - environment: [x86] - # pending fix for intermittent test - # environment: [x86, arm] - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }} - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: NextChapterSoftware/ec2-action-builder@v1 with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Test - working-directory: ./barretenberg/cpp/ - run: | - earthly-cloud test ${{ matrix.environment }} --no-output +test - - # All benchmarks, purposefully ran sequential on a machine - # they should use parallelism within the benchmark, but only one thing should run at a time - # for accurate results - # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. - bb-bench: - runs-on: ubuntu-latest - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - # We use a looser concurrency limit here, but we place a lock - # in the actual step action so that only one bench takes place - group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} + github_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + aws_access_key_id: ${{ secrets.DEPLOY_AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.DEPLOY_AWS_SECRET_ACCESS_KEY }} + aws_region: "us-west-2" + ec2_instance_type: c5.4xlarge + ec2_ami_id: ami-008fe2fc65df48dac + ec2_subnet_id: "SUBNET_ID_REDACTED" + ec2_security_group_id: "SECURITY_GROUP_ID_REDACTED" + ec2_instance_ttl: 40 # Optional (default is 60 minutes) + ec2_spot_instance_strategy: None # Other options are: SpotOnly, BestEffort, MaxPerformance + + # Job that runs on the self-hosted runner + run-build: + timeout-minutes: 1 + needs: + - start-runner + runs-on: ${{ github.run_id }} + steps: + - run: env + # # there's a lot of x86 tasks - let's split out the build step + # build-x86: + # runs-on: ubuntu-latest + # outputs: + # e2e_list: ${{ steps.e2e_list.outputs.list }} + # env: + # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # # cancel if reran on same PR if exists, otherwise if on same commit + # concurrency: + # group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # cancel-in-progress: true + # # permission check + # if: github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'external-ci') + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Build and Push Binaries - working-directory: ./barretenberg/cpp/ - run: earthly-cloud build x86 --push +bench-base - - # Only allow one person to use this runner - # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # github_actor: ${{ github.actor }} + + # - name: Build + # working-directory: ./yarn-project + # run: | + # # push to registry + # earthly-cloud build x86 --push +build-end-to-end + + # # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # # (Note ARM uses just 2 tests as a smoketest) + # - id: e2e_list + # working-directory: ./yarn-project/end-to-end + # run: | + # echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + + # # all the end-to-end integration tests for aztec + # e2e-arm: + # runs-on: ubuntu-latest + # env: + # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # strategy: + # fail-fast: false + # matrix: + # test: + # - e2e-card-game + # - e2e-crowdfunding-and-claim + # concurrency: + # group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-arm + # cancel-in-progress: true + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} - # Use bench_mode=cache to read the pushed build above - - name: Client IVC Bench - working-directory: ./barretenberg/cpp/ - run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # github_actor: ${{ github.actor }} + + # - name: Build + # working-directory: ./yarn-project/end-to-end/ + # # We don't do much on arm, just run it on their builder + # run: + # # Flags: + # # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy + # earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build + + # # all the end-to-end integration tests for aztec + # e2e-x86: + # needs: build-x86 + # runs-on: ubuntu-latest + # env: + # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # strategy: + # fail-fast: false + # matrix: + # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + # # cancel if reran on same PR if exists, otherwise if on same commit + # concurrency: + # group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # cancel-in-progress: true + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} - - name: Ultrahonk Bench - working-directory: ./barretenberg/cpp/ - run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # github_actor: ${{ github.actor }} + + # - name: Pull E2E Images + # working-directory: ./barretenberg/cpp/ + # run: | + # docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) + # docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) + # docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest + # docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest + + # - name: Test + # working-directory: ./yarn-project/end-to-end/ + # run: | + # # we blank earthly token just to be sure this is a pure local run + # EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + + # - name: Upload logs + # run: | + # BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + + # # barretenberg (prover) native tests + # bb-native-tests: + # needs: build-x86 + # runs-on: ubuntu-latest + # env: + # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # strategy: + # fail-fast: false + # matrix: + # environment: [x86] + # # pending fix for intermittent test + # # environment: [x86, arm] + # # cancel if reran on same PR if exists, otherwise if on same commit + # concurrency: + # group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }} + # cancel-in-progress: true + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} - # # Post actions, deploy and summarize logs - # aztec-bench-summary: + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # github_actor: ${{ github.actor }} + + # - name: Test + # working-directory: ./barretenberg/cpp/ + # run: | + # earthly-cloud test ${{ matrix.environment }} --no-output +test + + # # All benchmarks, purposefully ran sequential on a machine + # # they should use parallelism within the benchmark, but only one thing should run at a time + # # for accurate results + # # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. + # bb-bench: # runs-on: ubuntu-latest - # # IMPORTANT security flaw if we don't need 'check-run-condition' - # needs: [e2e-x86, check-run-condition] + # env: + # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # # cancel if reran on same PR if exists, otherwise if on same commit # concurrency: - # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # # We use a looser concurrency limit here, but we place a lock + # # in the actual step action so that only one bench takes place + # group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 # cancel-in-progress: true # steps: # - name: Checkout # uses: actions/checkout@v4 # with: + # # we check out submodules in ci-setup-action # ref: ${{ github.event.pull_request.head.sha }} - # - name: "Assemble benchmark summary from uploaded logs" - # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # github_actor: ${{ github.actor }} + + # - name: Build and Push Binaries + # working-directory: ./barretenberg/cpp/ + # run: earthly-cloud build x86 --push +bench-base + + # # Only allow one person to use this runner + # # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench + + # # Use bench_mode=cache to read the pushed build above + # - name: Client IVC Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache + + # - name: Ultrahonk Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + + # # # Post actions, deploy and summarize logs + # # aztec-bench-summary: + # # runs-on: ubuntu-latest + # # # IMPORTANT security flaw if we don't need 'check-run-condition' + # # needs: e2e-x86 + # # concurrency: + # # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # # cancel-in-progress: true + # # steps: + # # - name: Checkout + # # uses: actions/checkout@v4 + # # with: + # # ref: ${{ github.event.pull_request.head.sha }} + + # # - name: "Assemble benchmark summary from uploaded logs" + # # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh diff --git a/.github/workflows/protocol-circuits-gate-diff.yml b/.github/workflows/protocol-circuits-gate-diff.yml index 3205229cbd4..7905f0e6a1e 100644 --- a/.github/workflows/protocol-circuits-gate-diff.yml +++ b/.github/workflows/protocol-circuits-gate-diff.yml @@ -4,21 +4,10 @@ on: push: branches: - master - pull_request_target: {types: [opened, synchronize, reopened, labeled]} + pull_request: {} jobs: - check-run-condition: - runs-on: ubuntu-latest - steps: - - name: Check Condition - run: | - if [[ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" && "${{ contains(github.event.pull_request.labels.*.name, 'external-ci') }}" != "true" ]]; then - echo "PR is not from the same repo and doesn't have the 'external-ci' label." - exit 1 - fi compare_protocol_circuits_gates: - # IMPORTANT security flaw if we don't need 'check-run-condition' - needs: [check-run-condition] concurrency: group: compare_protocol_circuits_gates-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index c27610cb8ef..2a46c6e26f2 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -48,7 +48,4 @@ else for job in $(jobs -p); do wait $job || exit 1 done -fi - - - +fi \ No newline at end of file From f29692de2bff45bea3ad15809dd6300a6eef2a38 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 19:48:04 +0000 Subject: [PATCH 013/161] many secrets --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e9a010cd89..63be25de68a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,9 @@ jobs: id: start-ec2-runner uses: NextChapterSoftware/ec2-action-builder@v1 with: - github_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - aws_access_key_id: ${{ secrets.DEPLOY_AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.DEPLOY_AWS_SECRET_ACCESS_KEY }} + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-west-2" ec2_instance_type: c5.4xlarge ec2_ami_id: ami-008fe2fc65df48dac From 6ab7bea45ec9bdcb4d7a5e5bba049fb406a97db2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 20:49:40 +0000 Subject: [PATCH 014/161] never gonna cache again Guilty keys have got no rhythm From 727fc5ae06c840d0bd159d752c85c845328ad262 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 20:53:52 +0000 Subject: [PATCH 015/161] never gonna cache again Guilty keys have got no rhythm From bfd3dae76b5d35ec198519c47f401fcac5b40c30 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 21:46:24 +0000 Subject: [PATCH 016/161] up --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63be25de68a..e103a3cd59c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,14 +8,14 @@ on: jobs: start-runner: timeout-minutes: 5 # normally it only takes 1-2 minutes - name: Start self-hosted EC2 runner + name: Start self-hosted EC2 runner runs-on: ubuntu-latest permissions: actions: write steps: - name: Start EC2 runner id: start-ec2-runner - uses: NextChapterSoftware/ec2-action-builder@v1 + uses: AztecProtocol/ec2-action-builder@v1 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 8a2b4108c00e4ddde6adba6b9172dbdb7cf79a25 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 21:59:14 +0000 Subject: [PATCH 017/161] huh --- .github/workflows/ci.yml | 1 + .vscode/settings.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e103a3cd59c..5a20341cf6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: ec2_instance_type: c5.4xlarge ec2_ami_id: ami-008fe2fc65df48dac ec2_subnet_id: "SUBNET_ID_REDACTED" + ec2_instance_tags: [] ec2_security_group_id: "SECURITY_GROUP_ID_REDACTED" ec2_instance_ttl: 40 # Optional (default is 60 minutes) ec2_spot_instance_strategy: None # Other options are: SpotOnly, BestEffort, MaxPerformance diff --git a/.vscode/settings.json b/.vscode/settings.json index 98b15088b50..0c9b214de1e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -160,5 +160,6 @@ "noir/noir-repo/Cargo.toml", "noir/noir-repo/acvm-repo/acvm_js/Cargo.toml", "avm-transpiler/Cargo.toml" - ] + ], + "cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp" } From a76ecb4fef4574507e3fe97f5fb1928265db985f Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 22:02:56 +0000 Subject: [PATCH 018/161] fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a20341cf6b..baff6cdef74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: ec2_instance_type: c5.4xlarge ec2_ami_id: ami-008fe2fc65df48dac ec2_subnet_id: "SUBNET_ID_REDACTED" - ec2_instance_tags: [] + ec2_instance_tags: '[]' ec2_security_group_id: "SECURITY_GROUP_ID_REDACTED" ec2_instance_ttl: 40 # Optional (default is 60 minutes) ec2_spot_instance_strategy: None # Other options are: SpotOnly, BestEffort, MaxPerformance From 647a657a799b85397ae37e89cf76f41800bbbd63 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 3 Apr 2024 22:14:19 +0000 Subject: [PATCH 019/161] try --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baff6cdef74..4ccde6ed1ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: ec2_ami_id: ami-008fe2fc65df48dac ec2_subnet_id: "SUBNET_ID_REDACTED" ec2_instance_tags: '[]' + github_action_runner_version: v2.315.0 ec2_security_group_id: "SECURITY_GROUP_ID_REDACTED" ec2_instance_ttl: 40 # Optional (default is 60 minutes) ec2_spot_instance_strategy: None # Other options are: SpotOnly, BestEffort, MaxPerformance From da9990484d3b98409a2ae09baf9c29b020f5d679 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 15:52:16 +0000 Subject: [PATCH 020/161] iteation --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ccde6ed1ea..1fe895dbe76 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-west-2" + aws_region: "us-east-2" ec2_instance_type: c5.4xlarge ec2_ami_id: ami-008fe2fc65df48dac ec2_subnet_id: "SUBNET_ID_REDACTED" From 0daaad18ac3e8b4ea3b9871fed343990b715f1e6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 18:43:03 +0000 Subject: [PATCH 021/161] ami --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe895dbe76..600a03dff1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_instance_type: c5.4xlarge - ec2_ami_id: ami-008fe2fc65df48dac + ec2_ami_id: ami-0d118c6e63bcb554e ec2_subnet_id: "SUBNET_ID_REDACTED" ec2_instance_tags: '[]' github_action_runner_version: v2.315.0 From d17e6ca32147706df2d39318873af5c3dfb0640d Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 18:48:54 +0000 Subject: [PATCH 022/161] ami x2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 600a03dff1d..f2a77182bcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_instance_type: c5.4xlarge - ec2_ami_id: ami-0d118c6e63bcb554e + ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: "SUBNET_ID_REDACTED" ec2_instance_tags: '[]' github_action_runner_version: v2.315.0 From 2314e245ff03320c5dc6a5d9870d0cb708f98931 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 18:59:24 +0000 Subject: [PATCH 023/161] ncs --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2a77182bcb..4a439e269b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v1 + uses: NextChapterSoftware/ec2-action-builder@v1 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 108fdc265e756f20af2dd1df2b6d13e5f53d824f Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:06:18 +0000 Subject: [PATCH 024/161] try --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a439e269b2..eb790c61b0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,12 +23,12 @@ jobs: aws_region: "us-east-2" ec2_instance_type: c5.4xlarge ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: "SUBNET_ID_REDACTED" + ec2_subnet_id: "" ec2_instance_tags: '[]' github_action_runner_version: v2.315.0 - ec2_security_group_id: "SECURITY_GROUP_ID_REDACTED" - ec2_instance_ttl: 40 # Optional (default is 60 minutes) - ec2_spot_instance_strategy: None # Other options are: SpotOnly, BestEffort, MaxPerformance + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_instance_ttl: 40 # 40 minutes to reap + ec2_spot_instance_strategy: BestEffort # Job that runs on the self-hosted runner run-build: From af445acfa773442bcab614ec0ff410782a848b97 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:08:11 +0000 Subject: [PATCH 025/161] try --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb790c61b0d..e5594c1ac25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: aws_region: "us-east-2" ec2_instance_type: c5.4xlarge ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: "" + ec2_subnet_id: subnet-4cfabd25 ec2_instance_tags: '[]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 From 26ef204998dbe1d33b7d969df6b292d0bb1dc445 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:10:17 +0000 Subject: [PATCH 026/161] spot is hot --- .github/workflows/ci.yml | 418 +++++++++++++++++++-------------------- 1 file changed, 205 insertions(+), 213 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5594c1ac25..8730312bb85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: start-runner: timeout-minutes: 5 # normally it only takes 1-2 minutes name: Start self-hosted EC2 runner - runs-on: ubuntu-latest + runs-on: ${{ github.run_id }} permissions: actions: write steps: @@ -30,228 +30,220 @@ jobs: ec2_instance_ttl: 40 # 40 minutes to reap ec2_spot_instance_strategy: BestEffort - # Job that runs on the self-hosted runner - run-build: - timeout-minutes: 1 - needs: - - start-runner - runs-on: ${{ github.run_id }} - steps: - - run: env - # # there's a lot of x86 tasks - let's split out the build step - # build-x86: - # runs-on: ubuntu-latest - # outputs: - # e2e_list: ${{ steps.e2e_list.outputs.list }} - # env: - # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # # cancel if reran on same PR if exists, otherwise if on same commit - # concurrency: - # group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # cancel-in-progress: true - # # permission check - # if: github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'external-ci') - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # github_actor: ${{ github.actor }} - - # - name: Build - # working-directory: ./yarn-project - # run: | - # # push to registry - # earthly-cloud build x86 --push +build-end-to-end - - # # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # # (Note ARM uses just 2 tests as a smoketest) - # - id: e2e_list - # working-directory: ./yarn-project/end-to-end - # run: | - # echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - - # # all the end-to-end integration tests for aztec - # e2e-arm: - # runs-on: ubuntu-latest - # env: - # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # strategy: - # fail-fast: false - # matrix: - # test: - # - e2e-card-game - # - e2e-crowdfunding-and-claim - # concurrency: - # group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-arm - # cancel-in-progress: true - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # github_actor: ${{ github.actor }} - - # - name: Build - # working-directory: ./yarn-project/end-to-end/ - # # We don't do much on arm, just run it on their builder - # run: - # # Flags: - # # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy - # earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build - - # # all the end-to-end integration tests for aztec - # e2e-x86: - # needs: build-x86 - # runs-on: ubuntu-latest - # env: - # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # strategy: - # fail-fast: false - # matrix: - # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - # # cancel if reran on same PR if exists, otherwise if on same commit - # concurrency: - # group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # cancel-in-progress: true - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # github_actor: ${{ github.actor }} - - # - name: Pull E2E Images - # working-directory: ./barretenberg/cpp/ - # run: | - # docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) - # docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) - # docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest - # docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest - - # - name: Test - # working-directory: ./yarn-project/end-to-end/ - # run: | - # # we blank earthly token just to be sure this is a pure local run - # EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # there's a lot of x86 tasks - let's split out the build step + build-x86: + runs-on: ${{ github.run_id }} + outputs: + e2e_list: ${{ steps.e2e_list.outputs.list }} + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # cancel if reran on same PR if exists, otherwise if on same commit + concurrency: + group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true + # permission check + if: github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'external-ci') + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} - # - name: Upload logs - # run: | - # BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + + - name: Build + working-directory: ./yarn-project + run: | + # push to registry + earthly-cloud build x86 --push +build-end-to-end + + # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # (Note ARM uses just 2 tests as a smoketest) + - id: e2e_list + working-directory: ./yarn-project/end-to-end + run: | + echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + + # all the end-to-end integration tests for aztec + e2e-arm: + runs-on: ${{ github.run_id }} + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + strategy: + fail-fast: false + matrix: + test: + - e2e-card-game + - e2e-crowdfunding-and-claim + concurrency: + group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-arm + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} - # # barretenberg (prover) native tests - # bb-native-tests: - # needs: build-x86 - # runs-on: ubuntu-latest - # env: - # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # strategy: - # fail-fast: false - # matrix: - # environment: [x86] - # # pending fix for intermittent test - # # environment: [x86, arm] - # # cancel if reran on same PR if exists, otherwise if on same commit - # concurrency: - # group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }} - # cancel-in-progress: true - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + + - name: Build + working-directory: ./yarn-project/end-to-end/ + # We don't do much on arm, just run it on their builder + run: + # Flags: + # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy + earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build + + # all the end-to-end integration tests for aztec + e2e-x86: + needs: build-x86 + runs-on: ${{ github.run_id }} + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + strategy: + fail-fast: false + matrix: + test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + # cancel if reran on same PR if exists, otherwise if on same commit + concurrency: + group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # github_actor: ${{ github.actor }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + + - name: Pull E2E Images + working-directory: ./barretenberg/cpp/ + run: | + docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) + docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) + docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest + docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest + + - name: Test + working-directory: ./yarn-project/end-to-end/ + run: | + # we blank earthly token just to be sure this is a pure local run + EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + + - name: Upload logs + run: | + BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + + # barretenberg (prover) native tests + bb-native-tests: + needs: build-x86 + runs-on: ${{ github.run_id }} + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + strategy: + fail-fast: false + matrix: + environment: [x86] + # pending fix for intermittent test + # environment: [x86, arm] + # cancel if reran on same PR if exists, otherwise if on same commit + concurrency: + group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} - # - name: Test - # working-directory: ./barretenberg/cpp/ - # run: | - # earthly-cloud test ${{ matrix.environment }} --no-output +test + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + + - name: Test + working-directory: ./barretenberg/cpp/ + run: | + earthly-cloud test ${{ matrix.environment }} --no-output +test + + # All benchmarks, purposefully ran sequential on a machine + # they should use parallelism within the benchmark, but only one thing should run at a time + # for accurate results + # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. + bb-bench: + runs-on: ${{ github.run_id }} + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + # cancel if reran on same PR if exists, otherwise if on same commit + concurrency: + # We use a looser concurrency limit here, but we place a lock + # in the actual step action so that only one bench takes place + group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} - # # All benchmarks, purposefully ran sequential on a machine - # # they should use parallelism within the benchmark, but only one thing should run at a time - # # for accurate results - # # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. - # bb-bench: - # runs-on: ubuntu-latest - # env: - # EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # # cancel if reran on same PR if exists, otherwise if on same commit + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + + - name: Build and Push Binaries + working-directory: ./barretenberg/cpp/ + run: earthly-cloud build x86 --push +bench-base + + # Only allow one person to use this runner + # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench + + # Use bench_mode=cache to read the pushed build above + - name: Client IVC Bench + working-directory: ./barretenberg/cpp/ + run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache + + - name: Ultrahonk Bench + working-directory: ./barretenberg/cpp/ + run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + + # # Post actions, deploy and summarize logs + # aztec-bench-summary: + # runs-on: ${{ github.run_id }} + # # IMPORTANT security flaw if we don't need 'check-run-condition' + # needs: e2e-x86 # concurrency: - # # We use a looser concurrency limit here, but we place a lock - # # in the actual step action so that only one bench takes place - # group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 # cancel-in-progress: true # steps: # - name: Checkout # uses: actions/checkout@v4 # with: - # # we check out submodules in ci-setup-action # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # github_actor: ${{ github.actor }} - - # - name: Build and Push Binaries - # working-directory: ./barretenberg/cpp/ - # run: earthly-cloud build x86 --push +bench-base - - # # Only allow one person to use this runner - # # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench - - # # Use bench_mode=cache to read the pushed build above - # - name: Client IVC Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache - - # - name: Ultrahonk Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache - - # # # Post actions, deploy and summarize logs - # # aztec-bench-summary: - # # runs-on: ubuntu-latest - # # # IMPORTANT security flaw if we don't need 'check-run-condition' - # # needs: e2e-x86 - # # concurrency: - # # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # # cancel-in-progress: true - # # steps: - # # - name: Checkout - # # uses: actions/checkout@v4 - # # with: - # # ref: ${{ github.event.pull_request.head.sha }} - - # # - name: "Assemble benchmark summary from uploaded logs" - # # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh + # - name: "Assemble benchmark summary from uploaded logs" + # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh From b7a8ac9ed19fb913041ac0dae70e7b6da3b98d95 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:12:57 +0000 Subject: [PATCH 027/161] spot is hot --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8730312bb85..7b1390ee979 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: start-runner: timeout-minutes: 5 # normally it only takes 1-2 minutes name: Start self-hosted EC2 runner - runs-on: ${{ github.run_id }} + runs-on: ubuntu-latest permissions: actions: write steps: @@ -32,7 +32,8 @@ jobs: # there's a lot of x86 tasks - let's split out the build step build-x86: - runs-on: ${{ github.run_id }} + needs: start-runner + runs-on: ${{ github.run_id }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} env: @@ -71,6 +72,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-arm: + needs: start-runner runs-on: ${{ github.run_id }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -151,7 +153,7 @@ jobs: # barretenberg (prover) native tests bb-native-tests: - needs: build-x86 + needs: start-runner runs-on: ${{ github.run_id }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -188,7 +190,8 @@ jobs: # for accurate results # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. bb-bench: - runs-on: ${{ github.run_id }} + runs-on: ${{ github.run_id }} + needs: start-runner env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} # cancel if reran on same PR if exists, otherwise if on same commit From 80d2b518494b77eeccacdd4ff3339b868ea4cf3a Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:16:15 +0000 Subject: [PATCH 028/161] spot is hot --- .github/ci-setup-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 15df4df1a28..0c6fee9bbb5 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -18,7 +18,7 @@ runs: id: cache-submodules uses: actions/cache@v3 with: - path: .git + path: .git/modules key: submodules-${{ hashFiles('.gitmodules') }} - name: Checkout Submodules From 417f813ddb0c1a98e88c44c4ec4a231ed929b8ad Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 16:47:32 -0400 Subject: [PATCH 029/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b1390ee979..7ae69fec32e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: NextChapterSoftware/ec2-action-builder@v1 + uses: AztecProtocol/ec2-action-builder@v2 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From da59885d8ce15cb9f564d770152618ae465f3cbc Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 16:57:54 -0400 Subject: [PATCH 030/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ae69fec32e..970bd26d61d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v2 + uses: AztecProtocol/ec2-action-builder@v4 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From e6f4612ffe90b4208626d8ef8573155f52adf756 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 17:12:56 -0400 Subject: [PATCH 031/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 970bd26d61d..d083dead98c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v4 + uses: AztecProtocol/ec2-action-builder@v5 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 3b581f63abd773501377b86ac2d7425d68461347 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 17:31:49 -0400 Subject: [PATCH 032/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d083dead98c..7b1390ee979 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v5 + uses: NextChapterSoftware/ec2-action-builder@v1 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 924ac9dd8e22deb54297dba2a55df01ad2e113f1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 17:35:32 -0400 Subject: [PATCH 033/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b1390ee979..6b0857a7998 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: NextChapterSoftware/ec2-action-builder@v1 + uses: NextChapterSoftware/ec2-action-builder@v1.2 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From a7a564a74b3337c38ce33fb17259d76ca91ef446 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 17:48:53 -0400 Subject: [PATCH 034/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b0857a7998..c82d86cdad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: NextChapterSoftware/ec2-action-builder@v1.2 + uses: AztecProtocol/ec2-action-builder@v6 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From ebc1dcc9d76cdec701e5196933847da253e8bff6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 17:55:04 -0400 Subject: [PATCH 035/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c82d86cdad0..37ddc310e36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v6 + uses: AztecProtocol/ec2-action-builder@v7 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 440286f3599c3695158147fe54e311c96b1c6c44 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 18:09:11 -0400 Subject: [PATCH 036/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37ddc310e36..8dabd85721b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v7 + uses: AztecProtocol/ec2-action-builder@v9 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From db5c05952c1da11aee78779d87fabb1606d07d80 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 18:35:49 -0400 Subject: [PATCH 037/161] Update ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dabd85721b..50140449305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,9 +21,9 @@ jobs: aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" - ec2_instance_type: c5.4xlarge + ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: subnet-4cfabd25 + ec2_subnet_id: NOT_USED # commented out in forked action TODO should we? ec2_instance_tags: '[]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 From 708b7762588b94955fa3d7540a39a5b5a74ed8d4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 18:37:14 -0400 Subject: [PATCH 038/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50140449305..4973f603481 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: aws_region: "us-east-2" ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: NOT_USED # commented out in forked action TODO should we? + ec2_subnet_id: subnet-4cfabd25 ec2_instance_tags: '[]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 From 130c860ecf5c875359af91a2c41e0ff6fff8fa4e Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:46:08 -0400 Subject: [PATCH 039/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4973f603481..e1415d124e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v9 + uses: AztecProtocol/ec2-action-builder@v11 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From e1bf992b68cb0dff37c2049b576a580eccd284a9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:49:50 -0400 Subject: [PATCH 040/161] Create clear-cache.yml --- .github/workflows/clear-cache.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/clear-cache.yml diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml new file mode 100644 index 00000000000..4972601df52 --- /dev/null +++ b/.github/workflows/clear-cache.yml @@ -0,0 +1,30 @@ +name: Clear cache + +on: + workflow_dispatch: + +permissions: + actions: write + +jobs: + clear-cache: + runs-on: ubuntu-latest + steps: + - name: Clear cache + uses: actions/github-script@v6 + with: + script: | + console.log("About to clear") + const caches = await github.rest.actions.getActionsCacheList({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + for (const cache of caches.data.actions_caches) { + console.log(cache) + github.rest.actions.deleteActionsCacheById({ + owner: context.repo.owner, + repo: context.repo.repo, + cache_id: cache.id, + }) + } + console.log("Clear completed") From e3aa956deb467c5ff79ed5ec4377eae5e11c8873 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 19:56:25 -0400 Subject: [PATCH 041/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1415d124e7..d02e15584c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v11 + uses: AztecProtocol/ec2-action-builder@v12 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 2c2f7026299d71f80ac719c20d905a05bc92bf2c Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:24:12 -0400 Subject: [PATCH 042/161] Update ci.yml --- .github/workflows/ci.yml | 94 ++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d02e15584c0..edddb71ff5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: jobs: start-runner: - timeout-minutes: 5 # normally it only takes 1-2 minutes + timeout-minutes: 5 name: Start self-hosted EC2 runner runs-on: ubuntu-latest permissions: @@ -98,13 +98,13 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - - name: Build - working-directory: ./yarn-project/end-to-end/ - # We don't do much on arm, just run it on their builder - run: - # Flags: - # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy - earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build + # - name: Build + # working-directory: ./yarn-project/end-to-end/ + # # We don't do much on arm, just run it on their builder + # run: + # # Flags: + # # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy + # earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build # all the end-to-end integration tests for aztec e2e-x86: @@ -133,23 +133,23 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - - name: Pull E2E Images - working-directory: ./barretenberg/cpp/ - run: | - docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) - docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) - docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest - docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest + # - name: Pull E2E Images + # working-directory: ./barretenberg/cpp/ + # run: | + # docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) + # docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) + # docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest + # docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest - - name: Test - working-directory: ./yarn-project/end-to-end/ - run: | - # we blank earthly token just to be sure this is a pure local run - EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # - name: Test + # working-directory: ./yarn-project/end-to-end/ + # run: | + # # we blank earthly token just to be sure this is a pure local run + # EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - name: Upload logs - run: | - BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # - name: Upload logs + # run: | + # BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: @@ -180,10 +180,10 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - - name: Test - working-directory: ./barretenberg/cpp/ - run: | - earthly-cloud test ${{ matrix.environment }} --no-output +test + # - name: Test + # working-directory: ./barretenberg/cpp/ + # run: | + # earthly-cloud test ${{ matrix.environment }} --no-output +test # All benchmarks, purposefully ran sequential on a machine # they should use parallelism within the benchmark, but only one thing should run at a time @@ -213,26 +213,26 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - - name: Build and Push Binaries - working-directory: ./barretenberg/cpp/ - run: earthly-cloud build x86 --push +bench-base - - # Only allow one person to use this runner - # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench - - # Use bench_mode=cache to read the pushed build above - - name: Client IVC Bench - working-directory: ./barretenberg/cpp/ - run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache - - - name: Ultrahonk Bench - working-directory: ./barretenberg/cpp/ - run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + # - name: Build and Push Binaries + # working-directory: ./barretenberg/cpp/ + # run: earthly-cloud build x86 --push +bench-base + + # # Only allow one person to use this runner + # # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench + + # # Use bench_mode=cache to read the pushed build above + # - name: Client IVC Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache + + # - name: Ultrahonk Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache # # Post actions, deploy and summarize logs # aztec-bench-summary: From f3e605097789ebb444fed50b0ec7d93e7a29f9bb Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:26:59 -0400 Subject: [PATCH 043/161] Update ci.yml --- .github/workflows/ci.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edddb71ff5d..3c79dec8d0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,13 +98,10 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - # - name: Build - # working-directory: ./yarn-project/end-to-end/ - # # We don't do much on arm, just run it on their builder - # run: - # # Flags: - # # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy - # earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build + - name: Build + working-directory: ./yarn-project/end-to-end/ + # We don't do much on arm, just run it on their builder + run: shutdown -c # all the end-to-end integration tests for aztec e2e-x86: From aad7df9eb18b5d3e9579e1f393ecb2425a0bd9b9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:30:07 -0400 Subject: [PATCH 044/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c79dec8d0a..ada4d0bda94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,7 @@ jobs: - name: Build working-directory: ./yarn-project/end-to-end/ # We don't do much on arm, just run it on their builder - run: shutdown -c + run: shutdown # all the end-to-end integration tests for aztec e2e-x86: From 1eb9d789eca04ff81c883dbc268576e7b03a0e84 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:37:29 -0400 Subject: [PATCH 045/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ada4d0bda94..ceee8e477f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v12 + uses: AztecProtocol/ec2-action-builder@v13 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 789294f9a030764e500fb4db56c239fdbfa64136 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:39:56 -0400 Subject: [PATCH 046/161] Update ci.yml --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ceee8e477f9..fc5643c1d48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" + runner_label: ${{ github.actor }} ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 From 71fd6be1e3e5b6fbd40a6d392c34e86282c58adc Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:52:01 -0400 Subject: [PATCH 047/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc5643c1d48..4add4cef239 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v13 + uses: AztecProtocol/ec2-action-builder@v14 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From fed2328839712f44ae0ee2f82d3d040ded22b82e Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:54:09 -0400 Subject: [PATCH 048/161] Update ci.yml --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4add4cef239..dab17f215fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: # there's a lot of x86 tasks - let's split out the build step build-x86: needs: start-runner - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} env: @@ -74,7 +74,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-arm: needs: start-runner - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -107,7 +107,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-x86: needs: build-x86 - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -152,7 +152,7 @@ jobs: # barretenberg (prover) native tests bb-native-tests: needs: start-runner - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -188,7 +188,7 @@ jobs: # for accurate results # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. bb-bench: - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} needs: start-runner env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -234,7 +234,7 @@ jobs: # # Post actions, deploy and summarize logs # aztec-bench-summary: - # runs-on: ${{ github.run_id }} + # runs-on: ${{ github.actor }} # # IMPORTANT security flaw if we don't need 'check-run-condition' # needs: e2e-x86 # concurrency: From a4419170694d713906daa3ecd0d6e789cc83aa2a Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 20:56:32 -0400 Subject: [PATCH 049/161] Update ci.yml --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dab17f215fd..969013df7ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,12 @@ jobs: - name: Build working-directory: ./yarn-project/end-to-end/ # We don't do much on arm, just run it on their builder - run: shutdown + run: shutdown now + + - name: Build + working-directory: ./yarn-project/end-to-end/ + # We don't do much on arm, just run it on their builder + run: sleep 5 # all the end-to-end integration tests for aztec e2e-x86: From be0631e674f3b204e12b20b10d11ea65f28ac6c4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 21:03:50 -0400 Subject: [PATCH 050/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 969013df7ed..bf6c5c495c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v14 + uses: AztecProtocol/ec2-action-builder@v15 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 14e0763a1b5648bb37fbbaf79c39a76efff5a98d Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 21:10:02 -0400 Subject: [PATCH 051/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf6c5c495c6..4016a802472 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v15 + uses: AztecProtocol/ec2-action-builder@v16 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 39c4d791899a36fecb77886a163c1e372f9c00ae Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 21:30:32 -0400 Subject: [PATCH 052/161] Update ci.yml --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4016a802472..bd5ce943c23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,13 +15,14 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v16 + uses: AztecProtocol/ec2-action-builder@v17 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" - runner_label: ${{ github.actor }} + runner_label: ${{ github.run_id }} + runner_concurrency: 5 ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 From 671e7d853dd80888788213d7df7726e3625ffff3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 21:36:04 -0400 Subject: [PATCH 053/161] Update ci.yml --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd5ce943c23..3978044d0be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,8 +44,6 @@ jobs: concurrency: group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 cancel-in-progress: true - # permission check - if: github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'external-ci') steps: - name: Checkout uses: actions/checkout@v4 @@ -100,11 +98,6 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - - name: Build - working-directory: ./yarn-project/end-to-end/ - # We don't do much on arm, just run it on their builder - run: shutdown now - - name: Build working-directory: ./yarn-project/end-to-end/ # We don't do much on arm, just run it on their builder From 97a1c858ce039df33b0a02e71da2d3ef262dc772 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 21:38:56 -0400 Subject: [PATCH 054/161] Update ci.yml --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3978044d0be..0fe89541351 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest permissions: actions: write - steps: + steps: - name: Start EC2 runner id: start-ec2-runner uses: AztecProtocol/ec2-action-builder@v17 @@ -35,7 +35,7 @@ jobs: # there's a lot of x86 tasks - let's split out the build step build-x86: needs: start-runner - runs-on: ${{ github.actor }} + runs-on: ${{ github.run_id }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} env: @@ -73,7 +73,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-arm: needs: start-runner - runs-on: ${{ github.actor }} + runs-on: ${{ github.run_id }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -106,7 +106,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-x86: needs: build-x86 - runs-on: ${{ github.actor }} + runs-on: ${{ github.run_id }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -151,7 +151,7 @@ jobs: # barretenberg (prover) native tests bb-native-tests: needs: start-runner - runs-on: ${{ github.actor }} + runs-on: ${{ github.run_id }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -187,7 +187,7 @@ jobs: # for accurate results # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. bb-bench: - runs-on: ${{ github.actor }} + runs-on: ${{ github.run_id }} needs: start-runner env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -233,7 +233,7 @@ jobs: # # Post actions, deploy and summarize logs # aztec-bench-summary: - # runs-on: ${{ github.actor }} + # runs-on: ${{ github.run_id }} # # IMPORTANT security flaw if we don't need 'check-run-condition' # needs: e2e-x86 # concurrency: From d22daac15f02c05309f0b08fe8189773c56623f6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 21:48:34 -0400 Subject: [PATCH 055/161] Update ci.yml --- .github/workflows/ci.yml | 83 +++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fe89541351..f3e3d9786c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,7 +101,10 @@ jobs: - name: Build working-directory: ./yarn-project/end-to-end/ # We don't do much on arm, just run it on their builder - run: sleep 5 + run: + # Flags: + # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy + earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build # all the end-to-end integration tests for aztec e2e-x86: @@ -130,23 +133,23 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - # - name: Pull E2E Images - # working-directory: ./barretenberg/cpp/ - # run: | - # docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) - # docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) - # docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest - # docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest + - name: Pull E2E Images + working-directory: ./barretenberg/cpp/ + run: | + docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) + docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) + docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest + docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest - # - name: Test - # working-directory: ./yarn-project/end-to-end/ - # run: | - # # we blank earthly token just to be sure this is a pure local run - # EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + - name: Test + working-directory: ./yarn-project/end-to-end/ + run: | + # we blank earthly token just to be sure this is a pure local run + EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # - name: Upload logs - # run: | - # BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + - name: Upload logs + run: | + BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: @@ -177,10 +180,10 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - # - name: Test - # working-directory: ./barretenberg/cpp/ - # run: | - # earthly-cloud test ${{ matrix.environment }} --no-output +test + - name: Test + working-directory: ./barretenberg/cpp/ + run: | + earthly-cloud test ${{ matrix.environment }} --no-output +test # All benchmarks, purposefully ran sequential on a machine # they should use parallelism within the benchmark, but only one thing should run at a time @@ -210,26 +213,26 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} github_actor: ${{ github.actor }} - # - name: Build and Push Binaries - # working-directory: ./barretenberg/cpp/ - # run: earthly-cloud build x86 --push +bench-base - - # # Only allow one person to use this runner - # # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench - - # # Use bench_mode=cache to read the pushed build above - # - name: Client IVC Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache - - # - name: Ultrahonk Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + - name: Build and Push Binaries + working-directory: ./barretenberg/cpp/ + run: earthly-cloud build x86 --push +bench-base + + # Only allow one person to use this runner + # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench + + # Use bench_mode=cache to read the pushed build above + - name: Client IVC Bench + working-directory: ./barretenberg/cpp/ + run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache + + - name: Ultrahonk Bench + working-directory: ./barretenberg/cpp/ + run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache # # Post actions, deploy and summarize logs # aztec-bench-summary: From f40fcc15a73a389addc103fb7d28600042ca2196 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 4 Apr 2024 22:00:18 -0400 Subject: [PATCH 056/161] Update action.yml --- .github/ci-setup-action/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 0c6fee9bbb5..2c6013057c0 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -28,6 +28,8 @@ runs: - name: Setup Earthly uses: earthly/actions-setup@v1 with: + # permission issue with spot runners, simpler to leave out + use-cache: false version: 'v0.8.5' - name: Setup Env From aac343a446894c0867d23236ba79ff13cd9caf68 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 09:30:10 -0400 Subject: [PATCH 057/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3e3d9786c4..dad013017d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" runner_label: ${{ github.run_id }} - runner_concurrency: 5 + runner_concurrency: 1 ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 From f7fe004939c083382c94b371aa43462a98b8ac63 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 09:35:20 -0400 Subject: [PATCH 058/161] Update ci.yml --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dad013017d3..5a745e5a17c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,13 +217,13 @@ jobs: working-directory: ./barretenberg/cpp/ run: earthly-cloud build x86 --push +bench-base - # Only allow one person to use this runner - # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench + # # Only allow one person to use this runner + # # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench # Use bench_mode=cache to read the pushed build above - name: Client IVC Bench From 98e75766a2375f10f180adf97d362b598db484f3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 10:08:48 -0400 Subject: [PATCH 059/161] Update ci.yml --- .github/workflows/ci.yml | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a745e5a17c..c2ccc94ee38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,81 @@ jobs: ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_instance_ttl: 40 # 40 minutes to reap ec2_spot_instance_strategy: BestEffort + start-runner2: + timeout-minutes: 5 + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v17 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + runner_label: ${{ github.run_id }} + runner_concurrency: 1 + ec2_instance_type: r6a.32xlarge + ec2_ami_id: ami-04d8422a9ba4de80f + ec2_subnet_id: subnet-4cfabd25 + ec2_instance_tags: '[]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_instance_ttl: 40 # 40 minutes to reap + ec2_spot_instance_strategy: BestEffort + start-runner3: + timeout-minutes: 5 + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v17 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + runner_label: ${{ github.run_id }} + runner_concurrency: 1 + ec2_instance_type: r6a.32xlarge + ec2_ami_id: ami-04d8422a9ba4de80f + ec2_subnet_id: subnet-4cfabd25 + ec2_instance_tags: '[]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_instance_ttl: 40 # 40 minutes to reap + ec2_spot_instance_strategy: BestEffort + start-runner4: + timeout-minutes: 5 + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v17 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + runner_label: ${{ github.run_id }} + runner_concurrency: 1 + ec2_instance_type: r6a.32xlarge + ec2_ami_id: ami-04d8422a9ba4de80f + ec2_subnet_id: subnet-4cfabd25 + ec2_instance_tags: '[]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_instance_ttl: 40 # 40 minutes to reap + ec2_spot_instance_strategy: BestEffort # there's a lot of x86 tasks - let's split out the build step build-x86: From a6d719a4b478e6dae119cfa2b512eb13c1cc4e6d Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 10:10:47 -0400 Subject: [PATCH 060/161] Update ci.yml --- .github/workflows/ci.yml | 78 +--------------------------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c2ccc94ee38..0c4a95855e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" runner_label: ${{ github.run_id }} - runner_concurrency: 1 + runner_concurrency: 2 ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 @@ -31,82 +31,6 @@ jobs: ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_instance_ttl: 40 # 40 minutes to reap ec2_spot_instance_strategy: BestEffort - start-runner2: - timeout-minutes: 5 - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v17 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - runner_label: ${{ github.run_id }} - runner_concurrency: 1 - ec2_instance_type: r6a.32xlarge - ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: subnet-4cfabd25 - ec2_instance_tags: '[]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_instance_ttl: 40 # 40 minutes to reap - ec2_spot_instance_strategy: BestEffort - start-runner3: - timeout-minutes: 5 - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v17 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - runner_label: ${{ github.run_id }} - runner_concurrency: 1 - ec2_instance_type: r6a.32xlarge - ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: subnet-4cfabd25 - ec2_instance_tags: '[]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_instance_ttl: 40 # 40 minutes to reap - ec2_spot_instance_strategy: BestEffort - start-runner4: - timeout-minutes: 5 - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v17 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - runner_label: ${{ github.run_id }} - runner_concurrency: 1 - ec2_instance_type: r6a.32xlarge - ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: subnet-4cfabd25 - ec2_instance_tags: '[]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_instance_ttl: 40 # 40 minutes to reap - ec2_spot_instance_strategy: BestEffort - # there's a lot of x86 tasks - let's split out the build step build-x86: needs: start-runner From 5c71f4ed384caafc916d0ade86d1b7ee0b6ce542 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:14:05 +0000 Subject: [PATCH 061/161] hack for rerun failures --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c4a95855e0..112629b68fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,8 @@ jobs: timeout-minutes: 5 name: Start self-hosted EC2 runner runs-on: ubuntu-latest + # Only on first run + if: ${{ github.run_attempt == 1 }} permissions: actions: write steps: @@ -31,6 +33,44 @@ jobs: ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_instance_ttl: 40 # 40 minutes to reap ec2_spot_instance_strategy: BestEffort + + start-runner-retry: + timeout-minutes: 5 + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + # We avoid the issue of having no runner on a rerun-just-failed + if: ${{ github.run_attempt == 2 }} + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v17 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + runner_label: ${{ github.run_id }} + runner_concurrency: 2 + ec2_instance_type: r6a.32xlarge + ec2_ami_id: ami-04d8422a9ba4de80f + ec2_subnet_id: subnet-4cfabd25 + ec2_instance_tags: '[]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_instance_ttl: 40 # 40 minutes to reap + ec2_spot_instance_strategy: BestEffort + + start-runner-retry-sorry: + timeout-minutes: 5 + name: Start self-hosted EC2 runner + runs-on: ubuntu-latest + # We support only one rerun + if: ${{ github.run_attempt > 2 }} + steps: + - run: echo "We only support one re-run right now due to how spot tasks are cached!" && exit 1 + # there's a lot of x86 tasks - let's split out the build step build-x86: needs: start-runner From 489fc766b4c087a1337af7f66abc7224590683ba Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 12:36:43 -0400 Subject: [PATCH 062/161] Update ci.yml --- .github/workflows/ci.yml | 53 ++++++---------------------------------- 1 file changed, 8 insertions(+), 45 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 112629b68fc..c3a55b1745c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,13 +17,13 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v17 + uses: AztecProtocol/ec2-action-builder@v18 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" - runner_label: ${{ github.run_id }} + runner_label: ${{ github.actor }} runner_concurrency: 2 ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f @@ -34,47 +34,10 @@ jobs: ec2_instance_ttl: 40 # 40 minutes to reap ec2_spot_instance_strategy: BestEffort - start-runner-retry: - timeout-minutes: 5 - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - # We avoid the issue of having no runner on a rerun-just-failed - if: ${{ github.run_attempt == 2 }} - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v17 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - runner_label: ${{ github.run_id }} - runner_concurrency: 2 - ec2_instance_type: r6a.32xlarge - ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: subnet-4cfabd25 - ec2_instance_tags: '[]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_instance_ttl: 40 # 40 minutes to reap - ec2_spot_instance_strategy: BestEffort - - start-runner-retry-sorry: - timeout-minutes: 5 - name: Start self-hosted EC2 runner - runs-on: ubuntu-latest - # We support only one rerun - if: ${{ github.run_attempt > 2 }} - steps: - - run: echo "We only support one re-run right now due to how spot tasks are cached!" && exit 1 - # there's a lot of x86 tasks - let's split out the build step build-x86: needs: start-runner - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} env: @@ -112,7 +75,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-arm: needs: start-runner - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -148,7 +111,7 @@ jobs: # all the end-to-end integration tests for aztec e2e-x86: needs: build-x86 - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -193,7 +156,7 @@ jobs: # barretenberg (prover) native tests bb-native-tests: needs: start-runner - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} strategy: @@ -229,7 +192,7 @@ jobs: # for accurate results # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. bb-bench: - runs-on: ${{ github.run_id }} + runs-on: ${{ github.actor }} needs: start-runner env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -275,7 +238,7 @@ jobs: # # Post actions, deploy and summarize logs # aztec-bench-summary: - # runs-on: ${{ github.run_id }} + # runs-on: ${{ github.actor }} # # IMPORTANT security flaw if we don't need 'check-run-condition' # needs: e2e-x86 # concurrency: From 385d784745beef132b30250b5c27ff107bebef63 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 13:19:31 -0400 Subject: [PATCH 063/161] Update ci.yml --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a55b1745c..9ebc197ae45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,11 +24,12 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" runner_label: ${{ github.actor }} - runner_concurrency: 2 + runner_concurrency: 40 ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 - ec2_instance_tags: '[]' + # prevent reaping from our custom spot reaper (TODO move away from that model?) + ec2_instance_tags: '[{"Keep-Alive": true}]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_instance_ttl: 40 # 40 minutes to reap From 8ef03b6694e94bbeed59bd38363738d5588431e5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 13:29:06 -0400 Subject: [PATCH 064/161] Update ci.yml --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ebc197ae45..a7fbb521c2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,6 @@ jobs: timeout-minutes: 5 name: Start self-hosted EC2 runner runs-on: ubuntu-latest - # Only on first run - if: ${{ github.run_attempt == 1 }} permissions: actions: write steps: @@ -28,7 +26,7 @@ jobs: ec2_instance_type: r6a.32xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 - # prevent reaping from our custom spot reaper (TODO move away from that model?) + # prevent reaping from our custom spot reaper (TODO do we need to reap in the future?) ec2_instance_tags: '[{"Keep-Alive": true}]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 From afead34368068b33ce57585023f5813daa0dda61 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 13:37:15 -0400 Subject: [PATCH 065/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7fbb521c2e..d6fcd32a418 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v18 + uses: AztecProtocol/ec2-action-builder@v19 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From e5c83495d15560a19efebc72cd42b80d9ea9d47b Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 13:39:49 -0400 Subject: [PATCH 066/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6fcd32a418..da73f91578e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 # prevent reaping from our custom spot reaper (TODO do we need to reap in the future?) - ec2_instance_tags: '[{"Keep-Alive": true}]' + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": true}]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_instance_ttl: 40 # 40 minutes to reap From 467da00211e9ed712841eff0f78f2a0033b8470b Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 13:40:31 -0400 Subject: [PATCH 067/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da73f91578e..40f91039803 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 # prevent reaping from our custom spot reaper (TODO do we need to reap in the future?) - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": true}]' + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_instance_ttl: 40 # 40 minutes to reap From 2d294be26709462ef0ee2399659456b408dc8dd4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 14:11:43 -0400 Subject: [PATCH 068/161] Update ci.yml --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40f91039803..c9db163cc74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,6 +69,8 @@ jobs: - id: e2e_list working-directory: ./yarn-project/end-to-end run: | + set -eu + sudo apt-get install jq echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT # all the end-to-end integration tests for aztec From 623b200d76416d8af0752f7d073efdc92e585e46 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 14:49:52 -0400 Subject: [PATCH 069/161] Update action.yml --- .github/ci-setup-action/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 2c6013057c0..4040787fc7b 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -25,6 +25,14 @@ runs: shell: bash run: git submodule sync --recursive && git submodule update --init --recursive + # Since we use multiple runners, dont want them clashing + - name: Generate Unique Home Directory + shell: bash + run: | + RUN_HOME=~/run-$RANDOM-$(date +%s) + echo HOME=$RUN_HOME >> $GITHUB_ENV + mkdir -p $RUN_HOME + - name: Setup Earthly uses: earthly/actions-setup@v1 with: From b5b5b170637699bedb1154587192464135d0e3d3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 14:52:30 -0400 Subject: [PATCH 070/161] Update ci.yml --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9db163cc74..ecb39980fd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,8 @@ jobs: working-directory: ./yarn-project/end-to-end run: | set -eu - sudo apt-get install jq + # TODO bundle this + sudo apt-get update && sudo apt-get -y jq echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT # all the end-to-end integration tests for aztec From 1b4a95e2f5510fdf1aa3ad397cf0cdad6d8fe062 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 14:54:28 -0400 Subject: [PATCH 071/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecb39980fd7..67ca5befc4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,7 @@ jobs: run: | set -eu # TODO bundle this - sudo apt-get update && sudo apt-get -y jq + sudo apt-get update && sudo apt-get -y install jq echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT # all the end-to-end integration tests for aztec From bbddcc3e3aefa93cf45b2a4504567f5833240870 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:11:15 -0400 Subject: [PATCH 072/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67ca5befc4e..1d772e546e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v19 + uses: AztecProtocol/ec2-action-builder@v20 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 9a332701d43297d5399e6364b34e5ae7a268399a Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:25:17 -0400 Subject: [PATCH 073/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d772e546e3..d28237fb2c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v20 + uses: AztecProtocol/ec2-action-builder@v21 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 2b2621eb6bd48994a0495eb2966d18ff21a6bb02 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:31:55 -0400 Subject: [PATCH 074/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d28237fb2c9..78253fbf133 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v21 + uses: AztecProtocol/ec2-action-builder@v22 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From e57953c82ab9b372afaf4816127c8394ee9bf7cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:44:03 -0400 Subject: [PATCH 075/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78253fbf133..f66743b1ed5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v22 + uses: AztecProtocol/ec2-action-builder@v23 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 56275a5eeab5a995f72b05090bc7d3a219ba0ca7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:52:17 -0400 Subject: [PATCH 076/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f66743b1ed5..634ab748ff0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v23 + uses: AztecProtocol/ec2-action-builder@v24 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From bc93a0671033428d0fac25bbf71fcbc81ad787b4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 15:57:47 -0400 Subject: [PATCH 077/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 634ab748ff0..05dd978736a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v24 + uses: AztecProtocol/ec2-action-builder@v25 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 77413c5f88fb44201839bb34799d3283c046b22c Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 16:06:20 -0400 Subject: [PATCH 078/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05dd978736a..8db130f4378 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v25 + uses: AztecProtocol/ec2-action-builder@v26 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 0eeecfbe626efe99425417eb6494c6577cf2691e Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 16:08:38 -0400 Subject: [PATCH 079/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8db130f4378..09996f0c9e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v26 + uses: AztecProtocol/ec2-action-builder@v27 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 67a195e4ec125e4a59d46bc32f6bccc10c69bc0c Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 16:20:46 -0400 Subject: [PATCH 080/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09996f0c9e1..b4d75e4f22c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v27 + uses: AztecProtocol/ec2-action-builder@v28 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 5ec92a1aa10c66e8daa0f996f06f73552f3820d6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 16:34:38 -0400 Subject: [PATCH 081/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4d75e4f22c..3e408f73c5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v28 + uses: AztecProtocol/ec2-action-builder@v29 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 578bbf2ece10f2ac16a608b6edc493f35aa39cc1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 16:48:33 -0400 Subject: [PATCH 082/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e408f73c5d..3abfd53aede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v29 + uses: AztecProtocol/ec2-action-builder@v30 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 1b6a3e259f3a9c56c2d47aa919a25d18dd89c3e1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 17:11:07 -0400 Subject: [PATCH 083/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3abfd53aede..bff82aacce3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v30 + uses: AztecProtocol/ec2-action-builder@v31 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 1308e4bad962af85a49b225edbb031fc0e39ecd7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 17:23:25 -0400 Subject: [PATCH 084/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bff82aacce3..52286ccfbed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v31 + uses: AztecProtocol/ec2-action-builder@v32 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From fdbf2bda963f8de02b725be5d828dbb462b7c0ab Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 18:16:33 -0400 Subject: [PATCH 085/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52286ccfbed..27ccb148fd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v32 + uses: AztecProtocol/ec2-action-builder@v33 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From c220ff2b4df05e6276f9764fa4dabddb594967f1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 18:23:57 -0400 Subject: [PATCH 086/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27ccb148fd9..06acab1201b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v33 + uses: AztecProtocol/ec2-action-builder@v34 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From a045382d0ae0f4cd58f9eb6659c03ae2d1cec4fb Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 18:56:07 -0400 Subject: [PATCH 087/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06acab1201b..1d4a65bbf7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v34 + uses: AztecProtocol/ec2-action-builder@v35 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From b02faa7da5ca6caca20031b6ce922afc725222b8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 19:02:18 -0400 Subject: [PATCH 088/161] Update ci.yml --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d4a65bbf7d..96026bc6eca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,11 @@ on: push: branches: [master] pull_request: {} - workflow_dispatch: {} + workflow_dispatch: + inputs: + runner_action: + description: The action to take with the self-hosted runner (start, stop, restart). + required: false jobs: start-runner: @@ -24,6 +28,7 @@ jobs: runner_label: ${{ github.actor }} runner_concurrency: 40 ec2_instance_type: r6a.32xlarge + subaction: ${{ github.event.inputs.runner_action || 'start' }} ec2_ami_id: ami-04d8422a9ba4de80f ec2_subnet_id: subnet-4cfabd25 # prevent reaping from our custom spot reaper (TODO do we need to reap in the future?) From 76fe4ccbe06425502389aa3f89aeac35689ec7c7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 19:04:13 -0400 Subject: [PATCH 089/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96026bc6eca..f62328f0451 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v35 + uses: AztecProtocol/ec2-action-builder@v36 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From a89f486ad446f77b2b439fcf3d3e64a1c9fdd122 Mon Sep 17 00:00:00 2001 From: ludamad Date: Fri, 5 Apr 2024 19:17:18 -0400 Subject: [PATCH 090/161] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f62328f0451..1de92c1401b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v36 + uses: AztecProtocol/ec2-action-builder@v37 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From f473ecec9754e3b6ce1833c85d5dc7a626162885 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:14:31 +0000 Subject: [PATCH 091/161] refactor ci to be on same machine --- .github/ci-setup-action/action.yml | 4 + .github/workflows/ci.yml | 313 ++++++++---------- yarn-project/Earthfile | 4 +- yarn-project/end-to-end/Earthfile | 9 +- .../scripts/docker-compose-no-sandbox.yml | 2 +- .../end-to-end/scripts/docker-compose-p2p.yml | 4 +- .../end-to-end/scripts/docker-compose.yml | 6 +- 7 files changed, 148 insertions(+), 194 deletions(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 4040787fc7b..aba5d4579dd 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -33,6 +33,10 @@ runs: echo HOME=$RUN_HOME >> $GITHUB_ENV mkdir -p $RUN_HOME + # TODO reconsider how jq gets into image + - name: Setup jq + uses: dcarbone/install-jq-action@v2.1.0 + - name: Setup Earthly uses: earthly/actions-setup@v1 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1de92c1401b..9e965d3cbb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,247 +1,194 @@ -name: Earthly CI +name: Beta CI on: - push: - branches: [master] + push: {} pull_request: {} workflow_dispatch: inputs: runner_action: description: The action to take with the self-hosted runner (start, stop, restart). required: false - +concurrency: + group: ci-${{ github.ref_name == 'master' && github.run_id || github.actor }} +step_macros: + checkout: &checkout + name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + setup: &setup + name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + spot_runner_config: &spot_runner_config + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_subnet_id: subnet-4cfabd25 + subaction: ${{ github.event.inputs.runner_action || 'start' }} + # prevent reaping by mainframe spot reaper + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_spot_instance_strategy: BestEffort jobs: - start-runner: + # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders + # just for the CI job. These are specced per user and run the entire CI. + # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning + # TODO build steps that ran in previous invocations are quickly ran from cache. + start-builder: timeout-minutes: 5 - name: Start self-hosted EC2 runner + name: Start self-hosted EC2 runner (x86) runs-on: ubuntu-latest permissions: - actions: write + actions: write + strategy: + matrix: + config: + - {ec2_instance_type: m7a.32xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 2, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} steps: - name: Start EC2 runner id: start-ec2-runner uses: AztecProtocol/ec2-action-builder@v37 with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - runner_label: ${{ github.actor }} - runner_concurrency: 40 - ec2_instance_type: r6a.32xlarge - subaction: ${{ github.event.inputs.runner_action || 'start' }} - ec2_ami_id: ami-04d8422a9ba4de80f - ec2_subnet_id: subnet-4cfabd25 - # prevent reaping from our custom spot reaper (TODO do we need to reap in the future?) - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_instance_ttl: 40 # 40 minutes to reap - ec2_spot_instance_strategy: BestEffort - - # there's a lot of x86 tasks - let's split out the build step - build-x86: - needs: start-runner - runs-on: ${{ github.actor }} - outputs: - e2e_list: ${{ steps.e2e_list.outputs.list }} - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true + <<: *spot_runner_config + runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} + runner_concurrency: ${{ matrix.config.runner_concurrency }} + ec2_instance_type: ${{ matrix.config.ec2_instance_type }} + ec2_ami_id: ${{ matrix.config.ec2_ami_id }} + ec2_instance_ttl: 30 # 30 minutes to reap, refreshed by job starts + + ######################### + # START OF ARM PIPELINE # + ######################### + build-arm: + needs: start-builder + runs-on: ${{ github.actor }}-arm steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Build - working-directory: ./yarn-project - run: | - # push to registry - earthly-cloud build x86 --push +build-end-to-end - - # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # (Note ARM uses just 2 tests as a smoketest) - - id: e2e_list - working-directory: ./yarn-project/end-to-end - run: | - set -eu - # TODO bundle this - sudo apt-get update && sudo apt-get -y install jq - echo "list=$(earthly ls | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + - *checkout + - *setup + # prepare images, tagged by commit hash + - run: earthly ./yarn-project+build-end-to-end # all the end-to-end integration tests for aztec e2e-arm: - needs: start-runner - runs-on: ${{ github.actor }} - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + needs: build-arm + runs-on: ${{ github.actor }}-arm strategy: fail-fast: false matrix: test: - e2e-card-game - e2e-crowdfunding-and-claim - concurrency: - group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-arm - cancel-in-progress: true steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Build + - *checkout + - *setup + - name: Test working-directory: ./yarn-project/end-to-end/ - # We don't do much on arm, just run it on their builder - run: - # Flags: - # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy - earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + + ######################### + # START OF x86 PIPELINE # + ######################### + build-x86: + needs: start-builder + runs-on: ${{ github.actor }}-x86 + outputs: {e2e_list: ${{ steps.e2e_list.outputs.list }}} + steps: + - *checkout + - *setup + # prepare images, tagged by commit hash + - run: earthly ./yarn-project+build-end-to-end + # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # (Note ARM uses just 2 tests as a smoketest) + - name: Create list of end-to-end jobs + id: e2e_list + run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT # all the end-to-end integration tests for aztec e2e-x86: needs: build-x86 - runs-on: ${{ github.actor }} - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + runs-on: ${{ github.actor }}-x86 strategy: fail-fast: false matrix: test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Pull E2E Images - working-directory: ./barretenberg/cpp/ - run: | - docker pull aztecprotocol/aztec-cache:$(git rev-parse HEAD) - docker pull aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) - docker tag aztecprotocol/aztec-cache:$(git rev-parse HEAD) aztecprotocol/aztec:latest - docker tag aztecprotocol/end-to-end-cache:$(git rev-parse HEAD) aztecprotocol/end-to-end:latest - + - *checkout + - *setup - name: Test working-directory: ./yarn-project/end-to-end/ - run: | - # we blank earthly token just to be sure this is a pure local run - EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - name: Upload logs - run: | - BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: - needs: start-runner - runs-on: ${{ github.actor }} - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + needs: start-builder + runs-on: ${{ github.actor }}-${{matrix.environment}} strategy: fail-fast: false matrix: environment: [x86] # pending fix for intermittent test # environment: [x86, arm] - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }} - cancel-in-progress: true steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - - name: Test - working-directory: ./barretenberg/cpp/ - run: | - earthly-cloud test ${{ matrix.environment }} --no-output +test - - # All benchmarks, purposefully ran sequential on a machine - # they should use parallelism within the benchmark, but only one thing should run at a time - # for accurate results - # We don't depend on 'build' as we use a different runner and will build components on the fist step that uses them. - bb-bench: - runs-on: ${{ github.actor }} - needs: start-runner - env: - EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} - # cancel if reran on same PR if exists, otherwise if on same commit - concurrency: - # We use a looser concurrency limit here, but we place a lock - # in the actual step action so that only one bench takes place - group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - cancel-in-progress: true + - *checkout + - *setup + - working_directory: ./barretenberg/cpp/ + run: earthly --no-output +test + + # push benchmarking binaries to dockerhub registry + bb-bench-base: + runs-on: ${{ github.actor }}-x86 + needs: start-builder steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - + - *checkout + - *setup - name: Build and Push Binaries working-directory: ./barretenberg/cpp/ - run: earthly-cloud build x86 --push +bench-base + run: earthly --push +bench-base - # # Only allow one person to use this runner - # # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench + start-bb-bench-runner: + timeout-minutes: 5 + # We wait for binaries to be done for kickoff + needs: bb-bench-base + name: Start self-hosted EC2 runner (ARM) + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v37 + with: + <<: *aws_config + runner_label: ${{ github.actor }}-bench + runner_concurrency: 1 + ec2_instance_type: r6g.4xlarge + ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts + # try if spot variance too high, uses on-demand: + # ec2_spot_instance_strategy: none + bb-bench: + runs-on: ${{ github.actor }}-bench-x86 + needs: start-bb-bench-runner + steps: + - *checkout + - *setup # Use bench_mode=cache to read the pushed build above - name: Client IVC Bench working-directory: ./barretenberg/cpp/ - run: earthly-cloud bench x86 --no-output +bench-client-ivc --bench_mode=cache + run: earthly --no-output +bench-client-ivc --bench_mode=cache - name: Ultrahonk Bench working-directory: ./barretenberg/cpp/ - run: earthly-cloud bench x86 --no-output +bench-ultra-honk --bench_mode=cache + run: earthly --no-output +bench-ultra-honk --bench_mode=cache # # Post actions, deploy and summarize logs # aztec-bench-summary: diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 7c883bf9a1a..15c42bd3519 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -84,6 +84,6 @@ end-to-end-minimal: build-end-to-end: ARG EARTHLY_GIT_HASH FROM +end-to-end-minimal - SAVE IMAGE --push aztecprotocol/end-to-end-cache:$EARTHLY_GIT_HASH + SAVE IMAGE --push aztecprotocol/end-to-end:$EARTHLY_GIT_HASH FROM +aztec - SAVE IMAGE --push aztecprotocol/aztec-cache:$EARTHLY_GIT_HASH + SAVE IMAGE --push aztecprotocol/aztec:$EARTHLY_GIT_HASH diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index d646e2e9465..3fc4b518c8b 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -13,6 +13,7 @@ E2E_TEST_LOCAL: ENV ENABLE_GAS=$enable_gas ENV TEST=$test ENV DEBUG="$debug" + ENV AZTEC_DOCKER_TAG= WITH DOCKER \ --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ @@ -22,16 +23,18 @@ E2E_TEST_LOCAL: END # run locally and take from cache, used for our mainly x86 jobs -E2E_TEST_FROM_DOCKERHUB: +E2E_TEST_FROM_CACHE: FUNCTION ARG test ARG compose_file=./scripts/docker-compose.yml ARG enable_gas="" ARG debug="aztec:*" + ARG aztec_docker_tag LOCALLY ENV ENABLE_GAS=$enable_gas ENV TEST=$test ENV DEBUG="$debug" + ENV AZTEC_DOCKER_TAG=$aztec_docker_tag # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate @@ -47,7 +50,7 @@ E2E_TEST_FROM_BUILD: ENV TEST=$test ENV DEBUG=$debug COPY $compose_file $compose_file - # For ARM, we do use WITH DOCKER as we don't have many e2e tests, but note E2E_TEST_FROM_DOCKERHUB + # For ARM, we do use WITH DOCKER as we don't have many e2e tests, but note E2E_TEST_FROM_CACHE WITH DOCKER \ --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ @@ -67,7 +70,7 @@ E2E_TEST: IF [ $e2e_mode = local ] DO +E2E_TEST_LOCAL --test=$test --compose_file=$compose_file --enable_gas=$enable_gas --debug=$debug ELSE IF [ $e2e_mode = cache ] - DO +E2E_TEST_FROM_DOCKERHUB --test=$test --compose_file=$compose_file --enable_gas=$enable_gas --debug=$debug + DO +E2E_TEST_FROM_CACHE --test=$test --aztec_docker_tag=$(git rev-parse HEAD) --compose_file=$compose_file --enable_gas=$enable_gas --debug=$debug ELSE DO +E2E_TEST_FROM_BUILD --test=$test --compose_file=$compose_file --enable_gas=$enable_gas --debug=$debug END diff --git a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml index f9e1686ceff..71278ea4a81 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml @@ -13,7 +13,7 @@ services: - '8545' end-to-end: - image: aztecprotocol/end-to-end:latest + image: aztecprotocol/end-to-end:${AZTEC_DOCKER_TAG:-latest} environment: BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} diff --git a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml index 2af25f1ca0f..e3d894d0795 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-p2p.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-p2p.yml @@ -7,7 +7,7 @@ services: - '8545' p2p-bootstrap: - image: aztecprotocol/aztec:latest + image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest} command: 'start --p2p-bootstrap' expose: - '40400' @@ -20,7 +20,7 @@ services: PEER_ID_PRIVATE_KEY: '0a260024080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1224080112205ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e1a44080112402df8b977f356c6e34fa021c9647973234dff4df706c185794405aafb556723cf5ea53185db2e52dae74d0d4d6cadc494174810d0a713cd09b0ac517c38bc781e' end-to-end: - image: aztecprotocol/end-to-end:latest + image: aztecprotocol/end-to-end:${AZTEC_DOCKER_TAG:-latest} environment: BENCHMARK: true DEBUG: ${DEBUG:-'aztec:*'} diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index cc33a29709f..0d5fed41c8e 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -13,10 +13,10 @@ services: - '8545' sandbox: - image: aztecprotocol/aztec:latest + image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest} environment: DEBUG: 'aztec:*' - DEBUG_COLORS: 1 + DEBUG_COLORS: 1A ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 ARCHIVER_POLLING_INTERVAL_MS: 50 @@ -30,7 +30,7 @@ services: - '8080' end-to-end: - image: aztecprotocol/end-to-end:latest + image: aztecprotocol/end-to-end:${AZTEC_DOCKER_TAG:-latest} environment: BENCHMARK: 'true' DEBUG: ${DEBUG:-aztec:*} From a555f0fb84514ac18c7cecd6e4a962edf59454d0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:15:51 +0000 Subject: [PATCH 092/161] unfancy --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e965d3cbb8..9f5fa4b92a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,8 @@ jobs: build-x86: needs: start-builder runs-on: ${{ github.actor }}-x86 - outputs: {e2e_list: ${{ steps.e2e_list.outputs.list }}} + outputs: + e2e_list: ${{ steps.e2e_list.outputs.list }} steps: - *checkout - *setup From 09e997b387190a25e4300cd4887ab6b9d44e6827 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:24:37 +0000 Subject: [PATCH 093/161] roll back anchors --- .github/workflows/ci.yml | 125 ++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f5fa4b92a0..963c1f7f89f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,31 +9,6 @@ on: required: false concurrency: group: ci-${{ github.ref_name == 'master' && github.run_id || github.actor }} -step_macros: - checkout: &checkout - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - setup: &setup - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - spot_runner_config: &spot_runner_config - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} - # prevent reaping by mainframe spot reaper - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_spot_instance_strategy: BestEffort jobs: # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders # just for the CI job. These are specced per user and run the entire CI. @@ -55,7 +30,17 @@ jobs: id: start-ec2-runner uses: AztecProtocol/ec2-action-builder@v37 with: - <<: *spot_runner_config + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_subnet_id: subnet-4cfabd25 + subaction: ${{ github.event.inputs.runner_action || 'start' }} + # prevent reaping by mainframe spot reaper + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_spot_instance_strategy: BestEffort runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} runner_concurrency: ${{ matrix.config.runner_concurrency }} ec2_instance_type: ${{ matrix.config.ec2_instance_type }} @@ -69,8 +54,16 @@ jobs: needs: start-builder runs-on: ${{ github.actor }}-arm steps: - - *checkout - - *setup + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} # prepare images, tagged by commit hash - run: earthly ./yarn-project+build-end-to-end @@ -85,8 +78,16 @@ jobs: - e2e-card-game - e2e-crowdfunding-and-claim steps: - - *checkout - - *setup + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache @@ -98,7 +99,7 @@ jobs: needs: start-builder runs-on: ${{ github.actor }}-x86 outputs: - e2e_list: ${{ steps.e2e_list.outputs.list }} + e2e_list: ${{ steps.e2e_list.outputs.list }}} steps: - *checkout - *setup @@ -119,8 +120,16 @@ jobs: matrix: test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} steps: - - *checkout - - *setup + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache @@ -138,9 +147,17 @@ jobs: # pending fix for intermittent test # environment: [x86, arm] steps: - - *checkout - - *setup - - working_directory: ./barretenberg/cpp/ + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} + - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test # push benchmarking binaries to dockerhub registry @@ -148,8 +165,16 @@ jobs: runs-on: ${{ github.actor }}-x86 needs: start-builder steps: - - *checkout - - *setup + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - name: Build and Push Binaries working-directory: ./barretenberg/cpp/ run: earthly --push +bench-base @@ -167,7 +192,17 @@ jobs: id: start-ec2-runner uses: AztecProtocol/ec2-action-builder@v37 with: - <<: *aws_config + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_subnet_id: subnet-4cfabd25 + subaction: ${{ github.event.inputs.runner_action || 'start' }} + # prevent reaping by mainframe spot reaper + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_spot_instance_strategy: BestEffort runner_label: ${{ github.actor }}-bench runner_concurrency: 1 ec2_instance_type: r6g.4xlarge @@ -180,8 +215,16 @@ jobs: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner steps: - - *checkout - - *setup + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} # Use bench_mode=cache to read the pushed build above - name: Client IVC Bench working-directory: ./barretenberg/cpp/ From 7197b657fd5d41bc82c3ddf05a4641d23845e351 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:26:37 +0000 Subject: [PATCH 094/161] fix branch master --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 963c1f7f89f..44e1e2f8736 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: Beta CI on: - push: {} + push: + branches: [master] pull_request: {} workflow_dispatch: inputs: From 989dd0b81c5100294cf31e1ddfc9adba816b4f2a Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:27:20 +0000 Subject: [PATCH 095/161] fix branch master --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44e1e2f8736..fc7638a6e45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,7 +100,7 @@ jobs: needs: start-builder runs-on: ${{ github.actor }}-x86 outputs: - e2e_list: ${{ steps.e2e_list.outputs.list }}} + e2e_list: ${{ steps.e2e_list.outputs.list }} steps: - *checkout - *setup From f445dd831d0885118504acd64c3acbe728c9a819 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:36:57 +0000 Subject: [PATCH 096/161] lock on memory hungry prover test --- .github/workflows/ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc7638a6e45..0a4158f26a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,10 +6,12 @@ on: workflow_dispatch: inputs: runner_action: - description: The action to take with the self-hosted runner (start, stop, restart). + description: "The action to take with the self-hosted runner (start, stop, restart)." required: false concurrency: - group: ci-${{ github.ref_name == 'master' && github.run_id || github.actor }} + # force parallelism + group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} + cancel-in-progress: true jobs: # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders # just for the CI job. These are specced per user and run the entire CI. @@ -153,6 +155,14 @@ jobs: with: # we check out submodules in ci-setup-action ref: ${{ github.event.pull_request.head.sha }} + # Only allow one memory-hunger prover test to use this runner + # As detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + # so we can't use the native concurrency in GA + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench-${{ github.actor }} - name: Setup uses: ./.github/ci-setup-action with: From 553f110f7f96f5826d5fb21c0fb616611f4f947c Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:40:31 +0000 Subject: [PATCH 097/161] test --- .github/ci-setup-action/action.yml | 6 +- .github/workflows/ci.yml | 390 ++++++++++++++--------------- scripts/setup_env.sh | 3 +- 3 files changed, 194 insertions(+), 205 deletions(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index aba5d4579dd..13b6d718e8f 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -6,10 +6,6 @@ inputs: dockerhub_password: required: true description: 'DockerHub Password' - github_actor: - required: true - description: 'GitHub Actor' - runs: # define an action, runs in OS of caller using: composite @@ -46,4 +42,4 @@ runs: - name: Setup Env shell: bash - run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} ${{ inputs.github_actor }} + run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a4158f26a7..62eb3cde432 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,214 +50,208 @@ jobs: ec2_ami_id: ${{ matrix.config.ec2_ami_id }} ec2_instance_ttl: 30 # 30 minutes to reap, refreshed by job starts - ######################### - # START OF ARM PIPELINE # - ######################### - build-arm: - needs: start-builder - runs-on: ${{ github.actor }}-arm - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - # prepare images, tagged by commit hash - - run: earthly ./yarn-project+build-end-to-end - - # all the end-to-end integration tests for aztec - e2e-arm: - needs: build-arm - runs-on: ${{ github.actor }}-arm - strategy: - fail-fast: false - matrix: - test: - - e2e-card-game - - e2e-crowdfunding-and-claim - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - name: Test - working-directory: ./yarn-project/end-to-end/ - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - ######################### - # START OF x86 PIPELINE # - ######################### - build-x86: - needs: start-builder - runs-on: ${{ github.actor }}-x86 - outputs: - e2e_list: ${{ steps.e2e_list.outputs.list }} - steps: - - *checkout - - *setup - # prepare images, tagged by commit hash - - run: earthly ./yarn-project+build-end-to-end - # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # (Note ARM uses just 2 tests as a smoketest) - - name: Create list of end-to-end jobs - id: e2e_list - run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + # ######################### + # # START OF ARM PIPELINE # + # ######################### + # build-arm: + # needs: start-builder + # runs-on: ${{ github.actor }}-arm + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # # prepare images, tagged by commit hash + # - run: earthly ./yarn-project+build-end-to-end - # all the end-to-end integration tests for aztec - e2e-x86: - needs: build-x86 - runs-on: ${{ github.actor }}-x86 - strategy: - fail-fast: false - matrix: - test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - name: Test - working-directory: ./yarn-project/end-to-end/ - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - name: Upload logs - run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # # all the end-to-end integration tests for aztec + # e2e-arm: + # needs: build-arm + # runs-on: ${{ github.actor }}-arm + # strategy: + # fail-fast: false + # matrix: + # test: + # - e2e-card-game + # - e2e-crowdfunding-and-claim + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Test + # working-directory: ./yarn-project/end-to-end/ + # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # barretenberg (prover) native tests - bb-native-tests: - needs: start-builder - runs-on: ${{ github.actor }}-${{matrix.environment}} - strategy: - fail-fast: false - matrix: - environment: [x86] - # pending fix for intermittent test - # environment: [x86, arm] - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - # Only allow one memory-hunger prover test to use this runner - # As detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - # so we can't use the native concurrency in GA - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench-${{ github.actor }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +test + # ######################### + # # START OF x86 PIPELINE # + # ######################### + # build-x86: + # needs: start-builder + # runs-on: ${{ github.actor }}-x86 + # outputs: + # e2e_list: ${{ steps.e2e_list.outputs.list }} + # steps: + # - *checkout + # - *setup + # # prepare images, tagged by commit hash + # - run: earthly ./yarn-project+build-end-to-end + # # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # # (Note ARM uses just 2 tests as a smoketest) + # - name: Create list of end-to-end jobs + # id: e2e_list + # run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - # push benchmarking binaries to dockerhub registry - bb-bench-base: - runs-on: ${{ github.actor }}-x86 - needs: start-builder - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - - name: Build and Push Binaries - working-directory: ./barretenberg/cpp/ - run: earthly --push +bench-base + # # all the end-to-end integration tests for aztec + # e2e-x86: + # needs: build-x86 + # runs-on: ${{ github.actor }}-x86 + # strategy: + # fail-fast: false + # matrix: + # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Test + # working-directory: ./yarn-project/end-to-end/ + # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # - name: Upload logs + # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - start-bb-bench-runner: - timeout-minutes: 5 - # We wait for binaries to be done for kickoff - needs: bb-bench-base - name: Start self-hosted EC2 runner (ARM) - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v37 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} - # prevent reaping by mainframe spot reaper - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_spot_instance_strategy: BestEffort - runner_label: ${{ github.actor }}-bench - runner_concurrency: 1 - ec2_instance_type: r6g.4xlarge - ec2_ami_id: ami-0d8a9b0419ddb331a - ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts - # try if spot variance too high, uses on-demand: - # ec2_spot_instance_strategy: none + # # barretenberg (prover) native tests + # bb-native-tests: + # needs: start-builder + # runs-on: ${{ github.actor }}-${{matrix.environment}} + # strategy: + # fail-fast: false + # matrix: + # environment: [x86] + # # pending fix for intermittent test + # # environment: [x86, arm] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # # Only allow one memory-hunger prover test to use this runner + # # As detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # # so we can't use the native concurrency in GA + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench-${{ github.actor }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +test - bb-bench: - runs-on: ${{ github.actor }}-bench-x86 - needs: start-bb-bench-runner - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - github_actor: ${{ github.actor }} - # Use bench_mode=cache to read the pushed build above - - name: Client IVC Bench - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +bench-client-ivc --bench_mode=cache + # # push benchmarking binaries to dockerhub registry + # bb-bench-base: + # runs-on: ${{ github.actor }}-x86 + # needs: start-builder + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Build and Push Binaries + # working-directory: ./barretenberg/cpp/ + # run: earthly --push +bench-base - - name: Ultrahonk Bench - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +bench-ultra-honk --bench_mode=cache + # start-bb-bench-runner: + # timeout-minutes: 5 + # # We wait for binaries to be done for kickoff + # needs: bb-bench-base + # name: Start self-hosted EC2 runner (ARM) + # runs-on: ubuntu-latest + # permissions: + # actions: write + # steps: + # - name: Start EC2 runner + # id: start-ec2-runner + # uses: AztecProtocol/ec2-action-builder@v37 + # with: + # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws_region: "us-east-2" + # ec2_subnet_id: subnet-4cfabd25 + # subaction: ${{ github.event.inputs.runner_action || 'start' }} + # # prevent reaping by mainframe spot reaper + # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + # github_action_runner_version: v2.315.0 + # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + # ec2_spot_instance_strategy: BestEffort + # runner_label: ${{ github.actor }}-bench + # runner_concurrency: 1 + # ec2_instance_type: r6g.4xlarge + # ec2_ami_id: ami-0d8a9b0419ddb331a + # ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts + # # try if spot variance too high, uses on-demand: + # # ec2_spot_instance_strategy: none - # # Post actions, deploy and summarize logs - # aztec-bench-summary: - # runs-on: ${{ github.actor }} - # # IMPORTANT security flaw if we don't need 'check-run-condition' - # needs: e2e-x86 - # concurrency: - # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # cancel-in-progress: true + # bb-bench: + # runs-on: ${{ github.actor }}-bench-x86 + # needs: start-bb-bench-runner # steps: # - name: Checkout # uses: actions/checkout@v4 # with: + # # we check out submodules in ci-setup-action # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # # Use bench_mode=cache to read the pushed build above + # - name: Client IVC Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +bench-client-ivc --bench_mode=cache + + # - name: Ultrahonk Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +bench-ultra-honk --bench_mode=cache + + # # # Post actions, deploy and summarize logs + # # aztec-bench-summary: + # # runs-on: ${{ github.actor }} + # # # IMPORTANT security flaw if we don't need 'check-run-condition' + # # needs: e2e-x86 + # # concurrency: + # # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # # cancel-in-progress: true + # # steps: + # # - name: Checkout + # # uses: actions/checkout@v4 + # # with: + # # ref: ${{ github.event.pull_request.head.sha }} - # - name: "Assemble benchmark summary from uploaded logs" - # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh + # # - name: "Assemble benchmark summary from uploaded logs" + # # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index 35c9509dab5..9fe6201738f 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -9,5 +9,4 @@ echo "Logging in to Docker..." echo $1 | docker login -u aztecprotocolci --password-stdin # Make earthly-cloud and earthly-cloud-bench scripts available -echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV -echo "GITHUB_ACTOR=$2" >> $GITHUB_ENV \ No newline at end of file +echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV \ No newline at end of file From 4b066c3ce1a942ed99e518a5a3ea388179839d85 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:42:02 +0000 Subject: [PATCH 098/161] upload --- .github/workflows/ci.yml | 384 +++++++++++++++++++-------------------- 1 file changed, 192 insertions(+), 192 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62eb3cde432..c4f67a07d93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,208 +50,208 @@ jobs: ec2_ami_id: ${{ matrix.config.ec2_ami_id }} ec2_instance_ttl: 30 # 30 minutes to reap, refreshed by job starts - # ######################### - # # START OF ARM PIPELINE # - # ######################### - # build-arm: - # needs: start-builder - # runs-on: ${{ github.actor }}-arm - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # # prepare images, tagged by commit hash - # - run: earthly ./yarn-project+build-end-to-end + ######################### + # START OF ARM PIPELINE # + ######################### + build-arm: + needs: start-builder + runs-on: ${{ github.actor }}-arm + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # prepare images, tagged by commit hash + - run: earthly ./yarn-project+build-end-to-end - # # all the end-to-end integration tests for aztec - # e2e-arm: - # needs: build-arm - # runs-on: ${{ github.actor }}-arm - # strategy: - # fail-fast: false - # matrix: - # test: - # - e2e-card-game - # - e2e-crowdfunding-and-claim - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - name: Test - # working-directory: ./yarn-project/end-to-end/ - # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # all the end-to-end integration tests for aztec + e2e-arm: + needs: build-arm + runs-on: ${{ github.actor }}-arm + strategy: + fail-fast: false + matrix: + test: + - e2e-card-game + - e2e-crowdfunding-and-claim + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Test + working-directory: ./yarn-project/end-to-end/ + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # ######################### - # # START OF x86 PIPELINE # - # ######################### - # build-x86: - # needs: start-builder - # runs-on: ${{ github.actor }}-x86 - # outputs: - # e2e_list: ${{ steps.e2e_list.outputs.list }} - # steps: - # - *checkout - # - *setup - # # prepare images, tagged by commit hash - # - run: earthly ./yarn-project+build-end-to-end - # # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # # (Note ARM uses just 2 tests as a smoketest) - # - name: Create list of end-to-end jobs - # id: e2e_list - # run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + ######################### + # START OF x86 PIPELINE # + ######################### + build-x86: + needs: start-builder + runs-on: ${{ github.actor }}-x86 + outputs: + e2e_list: ${{ steps.e2e_list.outputs.list }} + steps: + - *checkout + - *setup + # prepare images, tagged by commit hash + - run: earthly ./yarn-project+build-end-to-end + # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # (Note ARM uses just 2 tests as a smoketest) + - name: Create list of end-to-end jobs + id: e2e_list + run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - # # all the end-to-end integration tests for aztec - # e2e-x86: - # needs: build-x86 - # runs-on: ${{ github.actor }}-x86 - # strategy: - # fail-fast: false - # matrix: - # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - name: Test - # working-directory: ./yarn-project/end-to-end/ - # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # - name: Upload logs - # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # all the end-to-end integration tests for aztec + e2e-x86: + needs: build-x86 + runs-on: ${{ github.actor }}-x86 + strategy: + fail-fast: false + matrix: + test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Test + working-directory: ./yarn-project/end-to-end/ + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + - name: Upload logs + run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - # # barretenberg (prover) native tests - # bb-native-tests: - # needs: start-builder - # runs-on: ${{ github.actor }}-${{matrix.environment}} - # strategy: - # fail-fast: false - # matrix: - # environment: [x86] - # # pending fix for intermittent test - # # environment: [x86, arm] - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # # Only allow one memory-hunger prover test to use this runner - # # As detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # # so we can't use the native concurrency in GA - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench-${{ github.actor }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +test + # barretenberg (prover) native tests + bb-native-tests: + needs: start-builder + runs-on: ${{ github.actor }}-${{matrix.environment}} + strategy: + fail-fast: false + matrix: + environment: [x86] + # pending fix for intermittent test + # environment: [x86, arm] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + # Only allow one memory-hunger prover test to use this runner + # As detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + # so we can't use the native concurrency in GA + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench-${{ github.actor }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - working-directory: ./barretenberg/cpp/ + run: earthly --no-output +test - # # push benchmarking binaries to dockerhub registry - # bb-bench-base: - # runs-on: ${{ github.actor }}-x86 - # needs: start-builder - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - name: Build and Push Binaries - # working-directory: ./barretenberg/cpp/ - # run: earthly --push +bench-base + # push benchmarking binaries to dockerhub registry + bb-bench-base: + runs-on: ${{ github.actor }}-x86 + needs: start-builder + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and Push Binaries + working-directory: ./barretenberg/cpp/ + run: earthly --push +bench-base - # start-bb-bench-runner: - # timeout-minutes: 5 - # # We wait for binaries to be done for kickoff - # needs: bb-bench-base - # name: Start self-hosted EC2 runner (ARM) - # runs-on: ubuntu-latest - # permissions: - # actions: write - # steps: - # - name: Start EC2 runner - # id: start-ec2-runner - # uses: AztecProtocol/ec2-action-builder@v37 - # with: - # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws_region: "us-east-2" - # ec2_subnet_id: subnet-4cfabd25 - # subaction: ${{ github.event.inputs.runner_action || 'start' }} - # # prevent reaping by mainframe spot reaper - # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - # github_action_runner_version: v2.315.0 - # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - # ec2_spot_instance_strategy: BestEffort - # runner_label: ${{ github.actor }}-bench - # runner_concurrency: 1 - # ec2_instance_type: r6g.4xlarge - # ec2_ami_id: ami-0d8a9b0419ddb331a - # ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts - # # try if spot variance too high, uses on-demand: - # # ec2_spot_instance_strategy: none + start-bb-bench-runner: + timeout-minutes: 5 + # We wait for binaries to be done for kickoff + needs: bb-bench-base + name: Start self-hosted EC2 runner (ARM) + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v37 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_subnet_id: subnet-4cfabd25 + subaction: ${{ github.event.inputs.runner_action || 'start' }} + # prevent reaping by mainframe spot reaper + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_spot_instance_strategy: BestEffort + runner_label: ${{ github.actor }}-bench + runner_concurrency: 1 + ec2_instance_type: r6g.4xlarge + ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts + # try if spot variance too high, uses on-demand: + # ec2_spot_instance_strategy: none + + bb-bench: + runs-on: ${{ github.actor }}-bench-x86 + needs: start-bb-bench-runner + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # Use bench_mode=cache to read the pushed build above + - name: Client IVC Bench + working-directory: ./barretenberg/cpp/ + run: earthly --no-output +bench-client-ivc --bench_mode=cache + + - name: Ultrahonk Bench + working-directory: ./barretenberg/cpp/ + run: earthly --no-output +bench-ultra-honk --bench_mode=cache - # bb-bench: - # runs-on: ${{ github.actor }}-bench-x86 - # needs: start-bb-bench-runner + # # Post actions, deploy and summarize logs + # aztec-bench-summary: + # runs-on: ${{ github.actor }} + # # IMPORTANT security flaw if we don't need 'check-run-condition' + # needs: e2e-x86 + # concurrency: + # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # cancel-in-progress: true # steps: # - name: Checkout # uses: actions/checkout@v4 # with: - # # we check out submodules in ci-setup-action # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # # Use bench_mode=cache to read the pushed build above - # - name: Client IVC Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +bench-client-ivc --bench_mode=cache - - # - name: Ultrahonk Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +bench-ultra-honk --bench_mode=cache - - # # # Post actions, deploy and summarize logs - # # aztec-bench-summary: - # # runs-on: ${{ github.actor }} - # # # IMPORTANT security flaw if we don't need 'check-run-condition' - # # needs: e2e-x86 - # # concurrency: - # # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # # cancel-in-progress: true - # # steps: - # # - name: Checkout - # # uses: actions/checkout@v4 - # # with: - # # ref: ${{ github.event.pull_request.head.sha }} - # # - name: "Assemble benchmark summary from uploaded logs" - # # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh + # - name: "Assemble benchmark summary from uploaded logs" + # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh From b4f016528ec693fc35f02103ac79ee1707f91a35 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:42:56 +0000 Subject: [PATCH 099/161] fix --- .github/workflows/ci.yml | 136 +++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4f67a07d93..a66308cc5db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,77 +167,77 @@ jobs: - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test - # push benchmarking binaries to dockerhub registry - bb-bench-base: - runs-on: ${{ github.actor }}-x86 - needs: start-builder - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build and Push Binaries - working-directory: ./barretenberg/cpp/ - run: earthly --push +bench-base + # # push benchmarking binaries to dockerhub registry + # bb-bench-base: + # runs-on: ${{ github.actor }}-x86 + # needs: start-builder + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Build and Push Binaries + # working-directory: ./barretenberg/cpp/ + # run: earthly --push +bench-base - start-bb-bench-runner: - timeout-minutes: 5 - # We wait for binaries to be done for kickoff - needs: bb-bench-base - name: Start self-hosted EC2 runner (ARM) - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v37 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} - # prevent reaping by mainframe spot reaper - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_spot_instance_strategy: BestEffort - runner_label: ${{ github.actor }}-bench - runner_concurrency: 1 - ec2_instance_type: r6g.4xlarge - ec2_ami_id: ami-0d8a9b0419ddb331a - ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts - # try if spot variance too high, uses on-demand: - # ec2_spot_instance_strategy: none + # start-bb-bench-runner: + # timeout-minutes: 5 + # # We wait for binaries to be done for kickoff + # needs: bb-bench-base + # name: Start self-hosted EC2 runner (ARM) + # runs-on: ubuntu-latest + # permissions: + # actions: write + # steps: + # - name: Start EC2 runner + # id: start-ec2-runner + # uses: AztecProtocol/ec2-action-builder@v37 + # with: + # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws_region: "us-east-2" + # ec2_subnet_id: subnet-4cfabd25 + # subaction: ${{ github.event.inputs.runner_action || 'start' }} + # # prevent reaping by mainframe spot reaper + # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + # github_action_runner_version: v2.315.0 + # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + # ec2_spot_instance_strategy: BestEffort + # runner_label: ${{ github.actor }}-bench + # runner_concurrency: 1 + # ec2_instance_type: r6g.4xlarge + # ec2_ami_id: ami-0d8a9b0419ddb331a + # ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts + # # try if spot variance too high, uses on-demand: + # # ec2_spot_instance_strategy: none - bb-bench: - runs-on: ${{ github.actor }}-bench-x86 - needs: start-bb-bench-runner - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # Use bench_mode=cache to read the pushed build above - - name: Client IVC Bench - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +bench-client-ivc --bench_mode=cache + # bb-bench: + # runs-on: ${{ github.actor }}-bench-x86 + # needs: start-bb-bench-runner + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # # Use bench_mode=cache to read the pushed build above + # - name: Client IVC Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +bench-client-ivc --bench_mode=cache - - name: Ultrahonk Bench - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +bench-ultra-honk --bench_mode=cache + # - name: Ultrahonk Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +bench-ultra-honk --bench_mode=cache # # Post actions, deploy and summarize logs # aztec-bench-summary: From 7399c2e787492749c44e0afa9696ec058bdba399 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:43:15 +0000 Subject: [PATCH 100/161] fix --- .github/workflows/ci.yml | 106 +++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a66308cc5db..21362e61de0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,60 +112,60 @@ jobs: id: e2e_list run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - # all the end-to-end integration tests for aztec - e2e-x86: - needs: build-x86 - runs-on: ${{ github.actor }}-x86 - strategy: - fail-fast: false - matrix: - test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Test - working-directory: ./yarn-project/end-to-end/ - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - name: Upload logs - run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # # all the end-to-end integration tests for aztec + # e2e-x86: + # needs: build-x86 + # runs-on: ${{ github.actor }}-x86 + # strategy: + # fail-fast: false + # matrix: + # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Test + # working-directory: ./yarn-project/end-to-end/ + # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # - name: Upload logs + # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - # barretenberg (prover) native tests - bb-native-tests: - needs: start-builder - runs-on: ${{ github.actor }}-${{matrix.environment}} - strategy: - fail-fast: false - matrix: - environment: [x86] - # pending fix for intermittent test - # environment: [x86, arm] - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - # Only allow one memory-hunger prover test to use this runner - # As detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - # so we can't use the native concurrency in GA - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench-${{ github.actor }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +test + # # barretenberg (prover) native tests + # bb-native-tests: + # needs: start-builder + # runs-on: ${{ github.actor }}-${{matrix.environment}} + # strategy: + # fail-fast: false + # matrix: + # environment: [x86] + # # pending fix for intermittent test + # # environment: [x86, arm] + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # # we check out submodules in ci-setup-action + # ref: ${{ github.event.pull_request.head.sha }} + # # Only allow one memory-hunger prover test to use this runner + # # As detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # # so we can't use the native concurrency in GA + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench-${{ github.actor }} + # - name: Setup + # uses: ./.github/ci-setup-action + # with: + # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +test # # push benchmarking binaries to dockerhub registry # bb-bench-base: From 8f766ed1cb147df022c57943cec60a77661a624f Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:43:57 +0000 Subject: [PATCH 101/161] fix --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21362e61de0..7b9b01ed114 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,7 +105,8 @@ jobs: - *checkout - *setup # prepare images, tagged by commit hash - - run: earthly ./yarn-project+build-end-to-end + - name: Build + run: earthly ./yarn-project+build-end-to-end # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end # (Note ARM uses just 2 tests as a smoketest) - name: Create list of end-to-end jobs From 368d2add5c2678f7acd5ee31f86114855af97a08 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:44:36 +0000 Subject: [PATCH 102/161] fix --- .github/workflows/ci.yml | 256 ++++++++++++++++++++------------------- 1 file changed, 131 insertions(+), 125 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b9b01ed114..e26a562a40d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,143 +102,149 @@ jobs: outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} steps: - - *checkout - - *setup + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} # prepare images, tagged by commit hash - - name: Build - run: earthly ./yarn-project+build-end-to-end + - run: earthly ./yarn-project+build-end-to-end # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end # (Note ARM uses just 2 tests as a smoketest) - name: Create list of end-to-end jobs id: e2e_list run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - # # all the end-to-end integration tests for aztec - # e2e-x86: - # needs: build-x86 - # runs-on: ${{ github.actor }}-x86 - # strategy: - # fail-fast: false - # matrix: - # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - name: Test - # working-directory: ./yarn-project/end-to-end/ - # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # - name: Upload logs - # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # all the end-to-end integration tests for aztec + e2e-x86: + needs: build-x86 + runs-on: ${{ github.actor }}-x86 + strategy: + fail-fast: false + matrix: + test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Test + working-directory: ./yarn-project/end-to-end/ + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + - name: Upload logs + run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - # # barretenberg (prover) native tests - # bb-native-tests: - # needs: start-builder - # runs-on: ${{ github.actor }}-${{matrix.environment}} - # strategy: - # fail-fast: false - # matrix: - # environment: [x86] - # # pending fix for intermittent test - # # environment: [x86, arm] - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # # Only allow one memory-hunger prover test to use this runner - # # As detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # # so we can't use the native concurrency in GA - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench-${{ github.actor }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +test + # barretenberg (prover) native tests + bb-native-tests: + needs: start-builder + runs-on: ${{ github.actor }}-${{matrix.environment}} + strategy: + fail-fast: false + matrix: + environment: [x86] + # pending fix for intermittent test + # environment: [x86, arm] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + # Only allow one memory-hunger prover test to use this runner + # As detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + # so we can't use the native concurrency in GA + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench-${{ github.actor }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - working-directory: ./barretenberg/cpp/ + run: earthly --no-output +test - # # push benchmarking binaries to dockerhub registry - # bb-bench-base: - # runs-on: ${{ github.actor }}-x86 - # needs: start-builder - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # - name: Build and Push Binaries - # working-directory: ./barretenberg/cpp/ - # run: earthly --push +bench-base + # push benchmarking binaries to dockerhub registry + bb-bench-base: + runs-on: ${{ github.actor }}-x86 + needs: start-builder + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build and Push Binaries + working-directory: ./barretenberg/cpp/ + run: earthly --push +bench-base - # start-bb-bench-runner: - # timeout-minutes: 5 - # # We wait for binaries to be done for kickoff - # needs: bb-bench-base - # name: Start self-hosted EC2 runner (ARM) - # runs-on: ubuntu-latest - # permissions: - # actions: write - # steps: - # - name: Start EC2 runner - # id: start-ec2-runner - # uses: AztecProtocol/ec2-action-builder@v37 - # with: - # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws_region: "us-east-2" - # ec2_subnet_id: subnet-4cfabd25 - # subaction: ${{ github.event.inputs.runner_action || 'start' }} - # # prevent reaping by mainframe spot reaper - # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - # github_action_runner_version: v2.315.0 - # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - # ec2_spot_instance_strategy: BestEffort - # runner_label: ${{ github.actor }}-bench - # runner_concurrency: 1 - # ec2_instance_type: r6g.4xlarge - # ec2_ami_id: ami-0d8a9b0419ddb331a - # ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts - # # try if spot variance too high, uses on-demand: - # # ec2_spot_instance_strategy: none + start-bb-bench-runner: + timeout-minutes: 5 + # We wait for binaries to be done for kickoff + needs: bb-bench-base + name: Start self-hosted EC2 runner (ARM) + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v37 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_subnet_id: subnet-4cfabd25 + subaction: ${{ github.event.inputs.runner_action || 'start' }} + # prevent reaping by mainframe spot reaper + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_spot_instance_strategy: BestEffort + runner_label: ${{ github.actor }}-bench + runner_concurrency: 1 + ec2_instance_type: r6g.4xlarge + ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts + # try if spot variance too high, uses on-demand: + # ec2_spot_instance_strategy: none - # bb-bench: - # runs-on: ${{ github.actor }}-bench-x86 - # needs: start-bb-bench-runner - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # # we check out submodules in ci-setup-action - # ref: ${{ github.event.pull_request.head.sha }} - # - name: Setup - # uses: ./.github/ci-setup-action - # with: - # dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # # Use bench_mode=cache to read the pushed build above - # - name: Client IVC Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +bench-client-ivc --bench_mode=cache + bb-bench: + runs-on: ${{ github.actor }}-bench-x86 + needs: start-bb-bench-runner + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + # we check out submodules in ci-setup-action + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + # Use bench_mode=cache to read the pushed build above + - name: Client IVC Bench + working-directory: ./barretenberg/cpp/ + run: earthly --no-output +bench-client-ivc --bench_mode=cache - # - name: Ultrahonk Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +bench-ultra-honk --bench_mode=cache + - name: Ultrahonk Bench + working-directory: ./barretenberg/cpp/ + run: earthly --no-output +bench-ultra-honk --bench_mode=cache # # Post actions, deploy and summarize logs # aztec-bench-summary: From 4aeb1185c605e33ee0412948271c8d393da02ac7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:57:37 +0000 Subject: [PATCH 103/161] concurrency fix --- .github/workflows/ci.yml | 99 ++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e26a562a40d..5682b2f4b5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,19 +53,21 @@ jobs: ######################### # START OF ARM PIPELINE # ######################### - build-arm: + # prevents concurrency issues with multiple (implicit) earthly bootstraps + setup-arm: needs: start-builder runs-on: ${{ github.actor }}-arm steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - run: earthly bootstrap + + build-arm: + needs: setup-arm + runs-on: ${{ github.actor }}-arm + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # prepare images, tagged by commit hash - run: earthly ./yarn-project+build-end-to-end @@ -80,15 +82,8 @@ jobs: - e2e-card-game - e2e-crowdfunding-and-claim steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache @@ -96,21 +91,23 @@ jobs: ######################### # START OF x86 PIPELINE # ######################### - build-x86: + # prevents concurrency issues with multiple (implicit) earthly bootstraps + setup-x86: needs: start-builder runs-on: ${{ github.actor }}-x86 + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - run: earthly bootstrap + + build-x86: + needs: setup-x86 + runs-on: ${{ github.actor }}-x86 outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # prepare images, tagged by commit hash - run: earthly ./yarn-project+build-end-to-end # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end @@ -128,15 +125,8 @@ jobs: matrix: test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache @@ -154,11 +144,8 @@ jobs: # pending fix for intermittent test # environment: [x86, arm] steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Only allow one memory-hunger prover test to use this runner # As detailed in https://github.com/ben-z/gh-action-mutex # things do not become 'pending' in github actions, and instead just cancel one another @@ -167,10 +154,6 @@ jobs: uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 with: branch: gh-action-mutex-bench-${{ github.actor }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test @@ -179,15 +162,8 @@ jobs: runs-on: ${{ github.actor }}-x86 needs: start-builder steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Build and Push Binaries working-directory: ./barretenberg/cpp/ run: earthly --push +bench-base @@ -228,15 +204,8 @@ jobs: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we check out submodules in ci-setup-action - ref: ${{ github.event.pull_request.head.sha }} - - name: Setup - uses: ./.github/ci-setup-action - with: - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Use bench_mode=cache to read the pushed build above - name: Client IVC Bench working-directory: ./barretenberg/cpp/ From fa62ec132ed7354299d52bf29639e841932feb29 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 04:58:40 +0000 Subject: [PATCH 104/161] concurrency fix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5682b2f4b5f..45e17d5b417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: # barretenberg (prover) native tests bb-native-tests: - needs: start-builder + needs: setup-x86 runs-on: ${{ github.actor }}-${{matrix.environment}} strategy: fail-fast: false @@ -160,7 +160,7 @@ jobs: # push benchmarking binaries to dockerhub registry bb-bench-base: runs-on: ${{ github.actor }}-x86 - needs: start-builder + needs: setup-x86 steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} From 9508f1f9ffbdfdd4d67420370f7e14746d26060b Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 05:09:17 +0000 Subject: [PATCH 105/161] forge fmt workaround --- .github/workflows/ci.yml | 4 ++-- l1-contracts/Earthfile | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45e17d5b417..d1fb274c5ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: # TODO build steps that ran in previous invocations are quickly ran from cache. start-builder: timeout-minutes: 5 - name: Start self-hosted EC2 runner (x86) + name: Start Build Runner (${{ matrix.config.runner_label_suffix }}) runs-on: ubuntu-latest permissions: actions: write @@ -172,7 +172,7 @@ jobs: timeout-minutes: 5 # We wait for binaries to be done for kickoff needs: bb-bench-base - name: Start self-hosted EC2 runner (ARM) + name: Start Bench Runner runs-on: ubuntu-latest permissions: actions: write diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile index c93c31bf1b3..dbf2ea0834b 100644 --- a/l1-contracts/Earthfile +++ b/l1-contracts/Earthfile @@ -19,5 +19,9 @@ COPY --dir lib scripts src terraform test *.json *.toml *.sh . build: RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff - RUN forge clean && forge fmt --check && forge build && forge test --no-match-contract UniswapPortalTest + RUN forge clean && forge build && forge test --no-match-contract UniswapPortalTest SAVE ARTIFACT out + +# not part of build as was too brittle - should CI just push if this changes? +format-check: + RUN forge fmt --check \ No newline at end of file From 9850ef6c17dea44087557681f9d5323042016686 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 05:26:25 +0000 Subject: [PATCH 106/161] work around old docker compose --- .github/workflows/ci.yml | 16 ++++++++-------- yarn-project/end-to-end/Earthfile | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1fb274c5ad..608f3cbe133 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,14 +146,14 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Only allow one memory-hunger prover test to use this runner - # As detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - # so we can't use the native concurrency in GA - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench-${{ github.actor }} + # # Only allow one memory-hunger prover test to use this runner + # # As detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # # so we can't use the native concurrency in GA + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench-${{ github.actor }} - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 3fc4b518c8b..3f20dc1a45f 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -19,7 +19,7 @@ E2E_TEST_LOCAL: --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + RUN COMPOSE_FILE=$compose_file docker compose up --exit-code-from=end-to-end --force-recreate END # run locally and take from cache, used for our mainly x86 jobs @@ -36,7 +36,7 @@ E2E_TEST_FROM_CACHE: ENV DEBUG="$debug" ENV AZTEC_DOCKER_TAG=$aztec_docker_tag # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images - RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + RUN COMPOSE_FILE=$compose_file docker compose up --exit-code-from=end-to-end --force-recreate # run on satellite and build, used for our few ARM jobs (means github runner doesn't need to be ARM) E2E_TEST_FROM_BUILD: @@ -56,7 +56,7 @@ E2E_TEST_FROM_BUILD: --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + RUN COMPOSE_FILE=$compose_file docker compose up --exit-code-from=end-to-end --force-recreate END E2E_TEST: From d8aba28793fe1d8360c61612bf96c5923414bb6e Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 05:43:09 +0000 Subject: [PATCH 107/161] work around old docker compose --- yarn-project/end-to-end/Earthfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 3f20dc1a45f..d6370e55264 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -18,8 +18,8 @@ E2E_TEST_LOCAL: --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get - # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN COMPOSE_FILE=$compose_file docker compose up --exit-code-from=end-to-end --force-recreate + # Run our docker-compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker-compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END # run locally and take from cache, used for our mainly x86 jobs @@ -36,7 +36,7 @@ E2E_TEST_FROM_CACHE: ENV DEBUG="$debug" ENV AZTEC_DOCKER_TAG=$aztec_docker_tag # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images - RUN COMPOSE_FILE=$compose_file docker compose up --exit-code-from=end-to-end --force-recreate + RUN docker-compose -f $compose_file up --exit-code-from=end-to-end --force-recreate # run on satellite and build, used for our few ARM jobs (means github runner doesn't need to be ARM) E2E_TEST_FROM_BUILD: @@ -55,8 +55,8 @@ E2E_TEST_FROM_BUILD: --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get - # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN COMPOSE_FILE=$compose_file docker compose up --exit-code-from=end-to-end --force-recreate + # Run our docker-compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker-compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END E2E_TEST: From ffac49804173b782f18fda85ec72c6b28855bedc Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 06:26:09 +0000 Subject: [PATCH 108/161] Update docker --- .github/workflows/ci.yml | 17 +++++++++-------- yarn-project/end-to-end/Earthfile | 10 +++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 608f3cbe133..abe3426390e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - run: apt-get update && apt-get -y install docker - run: earthly bootstrap build-x86: @@ -146,14 +147,14 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # # Only allow one memory-hunger prover test to use this runner - # # As detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # # so we can't use the native concurrency in GA - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench-${{ github.actor }} + # Only allow one memory-hunger prover test to use this runner + # As detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + # so we can't use the native concurrency in GA + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench-${{ github.actor }} - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index d6370e55264..3fc4b518c8b 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -18,8 +18,8 @@ E2E_TEST_LOCAL: --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get - # Run our docker-compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN docker-compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END # run locally and take from cache, used for our mainly x86 jobs @@ -36,7 +36,7 @@ E2E_TEST_FROM_CACHE: ENV DEBUG="$debug" ENV AZTEC_DOCKER_TAG=$aztec_docker_tag # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images - RUN docker-compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate # run on satellite and build, used for our few ARM jobs (means github runner doesn't need to be ARM) E2E_TEST_FROM_BUILD: @@ -55,8 +55,8 @@ E2E_TEST_FROM_BUILD: --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end-minimal \ --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+get - # Run our docker-compose, ending whenever sandbox ends, filtering out noisy eth_getLogs - RUN docker-compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs + RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END E2E_TEST: From ee4adea5dd58eba786ded7fafa1dfc57eec5bb02 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 14:15:35 +0000 Subject: [PATCH 109/161] large build, better ami --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abe3426390e..7714ef51490 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: config: - - {ec2_instance_type: m7a.32xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + - {ec2_instance_type: m7a.48xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 2, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} steps: - name: Start EC2 runner @@ -98,7 +98,6 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - run: apt-get update && apt-get -y install docker - run: earthly bootstrap build-x86: @@ -193,10 +192,10 @@ jobs: github_action_runner_version: v2.315.0 ec2_security_group_id: sg-0ccd4e5df0dcca0c9 ec2_spot_instance_strategy: BestEffort - runner_label: ${{ github.actor }}-bench + runner_label: ${{ github.actor }}-bench-x86 runner_concurrency: 1 ec2_instance_type: r6g.4xlarge - ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_ami_id: ami-04d8422a9ba4de80f ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts # try if spot variance too high, uses on-demand: # ec2_spot_instance_strategy: none From 2685d2c4ac3c3cf744dbf6a8cd870c2819bc823b Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 14:25:18 +0000 Subject: [PATCH 110/161] better benching --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7714ef51490..2aa1de7c78b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,7 +194,7 @@ jobs: ec2_spot_instance_strategy: BestEffort runner_label: ${{ github.actor }}-bench-x86 runner_concurrency: 1 - ec2_instance_type: r6g.4xlarge + ec2_instance_type: m7a.4xlarge ec2_ami_id: ami-04d8422a9ba4de80f ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts # try if spot variance too high, uses on-demand: From 2e2a526abe64849806b85846a297f0c199846fe4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 18:25:14 +0000 Subject: [PATCH 111/161] flexible with whatever docker --- .github/workflows/ci.yml | 16 ++++++++-------- yarn-project/end-to-end/Earthfile | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aa1de7c78b..d8e0469b928 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,14 +146,14 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Only allow one memory-hunger prover test to use this runner - # As detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - # so we can't use the native concurrency in GA - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench-${{ github.actor }} + # # Only allow one memory-hunger prover test to use this runner + # # As detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # # so we can't use the native concurrency in GA + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench-${{ github.actor }} - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 3fc4b518c8b..f4d1dca2376 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -35,8 +35,15 @@ E2E_TEST_FROM_CACHE: ENV TEST=$test ENV DEBUG="$debug" ENV AZTEC_DOCKER_TAG=$aztec_docker_tag + # need a different project name for each to run in parallel + LET project_name=$(echo $test | sed 's/\./_/g') + IF docker compose > /dev/null 2>&1 + LET CMD="docker compose" + ELSE + LET CMD="docker-compose" + END # In CI we do not use WITH DOCKER as we have had issues with earthly copying big images - RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate + RUN $CMD -p $project_name -f $compose_file up --exit-code-from=end-to-end --force-recreate # run on satellite and build, used for our few ARM jobs (means github runner doesn't need to be ARM) E2E_TEST_FROM_BUILD: From 669f56f3c8f91e1bef7f1ba896cf67018a992a5a Mon Sep 17 00:00:00 2001 From: ludamad Date: Sat, 6 Apr 2024 18:26:55 +0000 Subject: [PATCH 112/161] try 64-core machines --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8e0469b928..ad38c897644 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: config: - - {ec2_instance_type: m7a.48xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + - {ec2_instance_type: m7a.16xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 2, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} steps: - name: Start EC2 runner From 2fd42681a022cb71f1c290fa27b0e6a56bea88a1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 15:42:27 +0000 Subject: [PATCH 113/161] disable logs --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad38c897644..6aaf4500873 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,8 +130,8 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - name: Upload logs - run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # - name: Upload logs + # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: From c6ca2a9d9728b1d0fee7f1354829b2bcc99468bc Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 16:18:42 +0000 Subject: [PATCH 114/161] merge fix --- yarn-project/Earthfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index b2c304a702e..c0f563e5c7f 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -42,16 +42,10 @@ build: ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true RUN ./bootstrap.sh full -<<<<<<< HEAD -# TODO versioning flow at end before publish? -# ENV COMMIT_TAG=$EARTHLY_GIT_HASH -# RUN ./scripts/version_packages.sh -======= aztec-prod: FROM +build RUN yarn workspaces focus @aztec/cli @aztec/aztec --production && yarn cache clean SAVE ARTIFACT /usr/src /usr/src ->>>>>>> origin/master aztec-prod-slim: FROM node:18.19.1-slim From 38f3c5e10b9230c22a99bdcc28288ccf8280a5d1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 17:34:38 +0000 Subject: [PATCH 115/161] attach ebs --- .github/workflows/ci.yml | 11 ++++- scripts/attach_ebs_cache.sh | 98 +++++++++++++++++++++++++++++++++++++ scripts/setup_env.sh | 3 +- 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100755 scripts/attach_ebs_cache.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad38c897644..01ca4b8a76f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,14 +57,19 @@ jobs: setup-arm: needs: start-builder runs-on: ${{ github.actor }}-arm + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - run: ./scripts/attach_ebs_cache.sh - run: earthly bootstrap build-arm: needs: setup-arm runs-on: ${{ github.actor }}-arm + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} @@ -98,6 +103,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - run: ./scripts/attach_ebs_cache.sh - run: earthly bootstrap build-x86: @@ -130,8 +136,9 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - - name: Upload logs - run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # TODO + # - name: Upload logs + # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log # barretenberg (prover) native tests bb-native-tests: diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh new file mode 100755 index 00000000000..da0e61b2a5f --- /dev/null +++ b/scripts/attach_ebs_cache.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -eu + +GITHUB_ACTOR=$1 +REGION="us-east-2" +AVAILABILITY_ZONE="us-east-2c" +SIZE=128 +VOLUME_TYPE="gp2" +INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) + +# Check for existing volume +# we don't filter by available - we want to just error if it's attached already +# this means we are in a weird state (two spot instances running etc) +EXISTING_VOLUME=$(aws ec2 describe-volumes \ + --region $REGION \ + --filters "Name=tag:username,Values=$GITHUB_ACTOR" \ + --query "Volumes[0].VolumeId" \ + --output text) + +# If no existing volume, create one +if [ "$EXISTING_VOLUME" == "None" ]; then + VOLUME_ID=$(aws ec2 create-volume \ + --region $REGION \ + --availability-zone $AVAILABILITY_ZONE \ + --size $SIZE \ + --volume-type $VOLUME_TYPE \ + --tag-specifications "ResourceType=volume,Tags=[{Key=username,Value=$GITHUB_ACTOR}]" \ + --query "VolumeId" \ + --output text) + MAX_WAIT_TIME=300 # Maximum wait time in seconds + WAIT_INTERVAL=10 # Interval between checks in seconds + elapsed_time=0 + # Wait for the volume to become available + echo "Waiting for volume $VOLUME_ID to become available..." + while [ "$(aws ec2 describe-volumes \ + --region $REGION \ + --volume-ids $VOLUME_ID \ + --query "Volumes[0].State" \ + --output text)" != "available" ]; do + sleep 1 + if [ $elapsed_time -ge $MAX_WAIT_TIME ]; then + echo "Volume $VOLUME_ID did not become available within $MAX_WAIT_TIME seconds." + exit 1 + fi + + sleep $WAIT_INTERVAL + elapsed_time=$((elapsed_time + WAIT_INTERVAL)) + done +else + VOLUME_ID=$EXISTING_VOLUME +fi + +# Attach volume to the instance +aws ec2 attach-volume \ + --region $REGION \ + --volume-id $VOLUME_ID \ + --instance-id $INSTANCE_ID \ + --device /dev/xvdb + +# Wait for the volume to be attached +while [ "$(aws ec2 describe-volumes \ + --region $REGION \ + --volume-ids $VOLUME_ID \ + --query "Volumes[0].Attachments[0].State" \ + --output text)" != "attached" ]; do + sleep 1 +done + +# We are expecting the device to come up as /dev/nvme1n1, but include generic code from +# https://github.com/slavivanov/ec2-spotter/blob/master/ec2spotter-remount-root +while true; do + if lsblk /dev/nvme1n1; then + BLKDEVICE=/dev/nvme1n1 + # DEVICE=/dev/nvme1n1p1 + break + fi + if lsblk /dev/xvdb; then + BLKDEVICE=/dev/xvdb + # DEVICE=/dev/xvdb1 + break + fi + echo "waiting for device to attach" + sleep 5 +done + +# Create a file system if it does not exist +if ! file -s $BLKDEVICE | grep -q ext4; then + mkfs -t ext4 $BLKDEVICE +fi + +# Unmount existing /var if it's already mounted +if mount | grep -q /var; then + umount /var +fi + +# Create a mount point and mount the volume +mkdir -p /var +mount $BLKDEVICE /var \ No newline at end of file diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index 9fe6201738f..e0a6988b2b6 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -9,4 +9,5 @@ echo "Logging in to Docker..." echo $1 | docker login -u aztecprotocolci --password-stdin # Make earthly-cloud and earthly-cloud-bench scripts available -echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV \ No newline at end of file +echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV +echo "EARTHLY_CONFIG=.github/earthly-ci-config.yml" >> $GITHUB_ENV \ No newline at end of file From d7f40471dc35e5a52743facd71e4c6cb0486bd64 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 17:38:06 +0000 Subject: [PATCH 116/161] fix script evoke --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01ca4b8a76f..fe48ec4097b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - run: ./scripts/attach_ebs_cache.sh + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }} - run: earthly bootstrap build-arm: @@ -103,7 +103,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - run: ./scripts/attach_ebs_cache.sh + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }} - run: earthly bootstrap build-x86: From 5c787c6a878ddeaf512b544f35acecf8053b785b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 17:41:24 +0000 Subject: [PATCH 117/161] fix env and check mount --- .github/workflows/ci.yml | 4 ++++ scripts/attach_ebs_cache.sh | 16 +++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe48ec4097b..a0c52590674 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: runs-on: ${{ github.actor }}-arm env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} @@ -100,6 +101,9 @@ jobs: setup-x86: needs: start-builder runs-on: ${{ github.actor }}-x86 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index da0e61b2a5f..e179f994338 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -8,6 +8,13 @@ SIZE=128 VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) +# Unmount existing /var if it's already mounted +if mount | grep -q /var/lib/docker/volumes; then + echo "Detected mount existing on /var/lib/docker/volumes already" + echo "Continuing..." + exit 0 +fi + # Check for existing volume # we don't filter by available - we want to just error if it's attached already # this means we are in a weird state (two spot instances running etc) @@ -88,11 +95,6 @@ if ! file -s $BLKDEVICE | grep -q ext4; then mkfs -t ext4 $BLKDEVICE fi -# Unmount existing /var if it's already mounted -if mount | grep -q /var; then - umount /var -fi - # Create a mount point and mount the volume -mkdir -p /var -mount $BLKDEVICE /var \ No newline at end of file +mkdir -p /var/lib/docker/volumes +mount $BLKDEVICE /var/lib/docker/volumes \ No newline at end of file From fade3b9cab1e2d40f2532386a7008603e909cf71 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 17:43:04 +0000 Subject: [PATCH 118/161] missing file --- .github/earthly-ci-config.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/earthly-ci-config.yml diff --git a/.github/earthly-ci-config.yml b/.github/earthly-ci-config.yml new file mode 100644 index 00000000000..c95d838b97d --- /dev/null +++ b/.github/earthly-ci-config.yml @@ -0,0 +1,5 @@ +global: + cache_size_mb: 250000 + buildkit_max_parallelism: 50 + container_frontend: docker-shell + buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]A \ No newline at end of file From 2420a1e08b53920dcca3ae984a578069269e3e04 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 17:44:06 +0000 Subject: [PATCH 119/161] fix syntax --- .github/earthly-ci-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/earthly-ci-config.yml b/.github/earthly-ci-config.yml index c95d838b97d..ffea16b6d09 100644 --- a/.github/earthly-ci-config.yml +++ b/.github/earthly-ci-config.yml @@ -2,4 +2,4 @@ global: cache_size_mb: 250000 buildkit_max_parallelism: 50 container_frontend: docker-shell - buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]A \ No newline at end of file + buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"] \ No newline at end of file From 611a12e0cb6279e04214fccf67aad3308aa00d76 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 17:58:43 +0000 Subject: [PATCH 120/161] restart with runner action --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c52590674..3a4a0b2ed33 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: description: "The action to take with the self-hosted runner (start, stop, restart)." required: false concurrency: - # force parallelism + # force parallelism group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: @@ -38,7 +38,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} + subaction: ${{ github.event.inputs.runner_action || 'restart' }} # prevent reaping by mainframe spot reaper ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 @@ -190,7 +190,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v37 + uses: AztecProtocol/ec2-action-builder@v0.1 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 563c2876ecb10c09eefc94387054556eff069156 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:00:25 +0000 Subject: [PATCH 121/161] fix avail zone --- scripts/attach_ebs_cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index e179f994338..dee3f72eb07 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -3,7 +3,7 @@ set -eu GITHUB_ACTOR=$1 REGION="us-east-2" -AVAILABILITY_ZONE="us-east-2c" +AVAILABILITY_ZONE="us-east-2a" SIZE=128 VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) From 3b9657e8c215920d50708a75777ded324a42080a Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:06:37 +0000 Subject: [PATCH 122/161] spec bench x86 --- .github/workflows/ci.yml | 15 ++++++++++++--- scripts/attach_ebs_cache.sh | 9 ++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a4a0b2ed33..0afb795d3ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }} + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - run: earthly bootstrap build-arm: @@ -107,7 +107,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }} + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 - run: earthly bootstrap build-x86: @@ -211,9 +211,18 @@ jobs: # try if spot variance too high, uses on-demand: # ec2_spot_instance_strategy: none - bb-bench: + setup-bb-bench: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 + - run: earthly bootstrap + + bb-bench: + runs-on: ${{ github.actor }}-bench-x86 + needs: setup-bb-bench steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index dee3f72eb07..a0f279584ff 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -1,10 +1,9 @@ #!/bin/bash set -eu -GITHUB_ACTOR=$1 -REGION="us-east-2" +EBS_CACHE_TAG=$1 +SIZE=$2 AVAILABILITY_ZONE="us-east-2a" -SIZE=128 VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) @@ -20,7 +19,7 @@ fi # this means we are in a weird state (two spot instances running etc) EXISTING_VOLUME=$(aws ec2 describe-volumes \ --region $REGION \ - --filters "Name=tag:username,Values=$GITHUB_ACTOR" \ + --filters "Name=tag:username,Values=$EBS_CACHE_TAG" \ --query "Volumes[0].VolumeId" \ --output text) @@ -31,7 +30,7 @@ if [ "$EXISTING_VOLUME" == "None" ]; then --availability-zone $AVAILABILITY_ZONE \ --size $SIZE \ --volume-type $VOLUME_TYPE \ - --tag-specifications "ResourceType=volume,Tags=[{Key=username,Value=$GITHUB_ACTOR}]" \ + --tag-specifications "ResourceType=volume,Tags=[{Key=username,Value=$EBS_CACHE_TAG}]" \ --query "VolumeId" \ --output text) MAX_WAIT_TIME=300 # Maximum wait time in seconds From 7583aad83b6ad8aeaece2870eaf0bf72660337f4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:09:36 +0000 Subject: [PATCH 123/161] fix region --- .github/workflows/ci.yml | 3 +++ scripts/attach_ebs_cache.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0afb795d3ce..ff137d880c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - run: earthly bootstrap @@ -106,6 +107,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # Attach our 128gb cache disk - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 - run: earthly bootstrap @@ -217,6 +219,7 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - run: earthly bootstrap diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index a0f279584ff..c10b555cbc6 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -3,6 +3,7 @@ set -eu EBS_CACHE_TAG=$1 SIZE=$2 +REGION="us-east-2" AVAILABILITY_ZONE="us-east-2a" VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) From 6439d073f54aba58e29e6e1106a9384d92af806d Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:14:22 +0000 Subject: [PATCH 124/161] fix x86 startup --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff137d880c0..d5b3c3efd90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,8 @@ jobs: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} # Attach our 128gb cache disk - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 + # Attach our 128gb cache disk + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - run: earthly bootstrap build-x86: From 7bea72ac018539e0421a43a419addf68c1405000 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:19:18 +0000 Subject: [PATCH 125/161] fix paths --- scripts/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index e0a6988b2b6..97fb8fd68a1 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -10,4 +10,4 @@ echo $1 | docker login -u aztecprotocolci --password-stdin # Make earthly-cloud and earthly-cloud-bench scripts available echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV -echo "EARTHLY_CONFIG=.github/earthly-ci-config.yml" >> $GITHUB_ENV \ No newline at end of file +echo "EARTHLY_CONFIG=$(git rev-parse --show-toplevel)/.github/earthly-ci-config.yml" >> $GITHUB_ENV \ No newline at end of file From 3b9e6a3d86173493f1d3727607a487686cee5c48 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:21:26 +0000 Subject: [PATCH 126/161] try old instance type --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b3c3efd90..7a5d138db36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: config: - - {ec2_instance_type: m7a.16xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + - {ec2_instance_type: m6a.16xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 2, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} steps: - name: Start EC2 runner From c5d2faf71a79ff7dcdef0341c9137273016d4329 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 14:25:40 -0400 Subject: [PATCH 127/161] Delete .github/workflows/clear-cache.yml obsolete --- .github/workflows/clear-cache.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 .github/workflows/clear-cache.yml diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml deleted file mode 100644 index 4972601df52..00000000000 --- a/.github/workflows/clear-cache.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Clear cache - -on: - workflow_dispatch: - -permissions: - actions: write - -jobs: - clear-cache: - runs-on: ubuntu-latest - steps: - - name: Clear cache - uses: actions/github-script@v6 - with: - script: | - console.log("About to clear") - const caches = await github.rest.actions.getActionsCacheList({ - owner: context.repo.owner, - repo: context.repo.repo, - }) - for (const cache of caches.data.actions_caches) { - console.log(cache) - github.rest.actions.deleteActionsCacheById({ - owner: context.repo.owner, - repo: context.repo.repo, - cache_id: cache.id, - }) - } - console.log("Clear completed") From ed3766687088ae3c0eb27eb13da41d85750c5cd4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 14:26:48 -0400 Subject: [PATCH 128/161] Update attach_ebs_cache.sh --- scripts/attach_ebs_cache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index c10b555cbc6..147559660d1 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -8,7 +8,7 @@ AVAILABILITY_ZONE="us-east-2a" VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) -# Unmount existing /var if it's already mounted +# Check for existing mount, assume we can continue if existing if mount | grep -q /var/lib/docker/volumes; then echo "Detected mount existing on /var/lib/docker/volumes already" echo "Continuing..." @@ -97,4 +97,4 @@ fi # Create a mount point and mount the volume mkdir -p /var/lib/docker/volumes -mount $BLKDEVICE /var/lib/docker/volumes \ No newline at end of file +mount $BLKDEVICE /var/lib/docker/volumes From 4d8e0392acbbc51268053d2d49acbd5c71771732 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 19:32:44 +0000 Subject: [PATCH 129/161] avail zones --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a5d138db36..f60a1c66e43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v37 + uses: AztecProtocol/ec2-action-builder@v0.2 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -193,7 +193,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v0.1 + uses: AztecProtocol/ec2-action-builder@v0.2 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 59d1fe8f62f54f70c4318d0905124844562d8149 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 19:42:04 +0000 Subject: [PATCH 130/161] revert --- .github/workflows/ci.yml | 5 ++- .../workflows/protocol-circuits-gate-diff.yml | 2 +- .vscode/settings.json | 3 +- l1-contracts/Earthfile | 2 +- .../circuit-types/src/stats/metrics.ts | 15 +------- .../scripts/src/benchmarks/aggregate.ts | 34 +------------------ .../scripts/src/benchmarks/markdown.ts | 12 ++----- 7 files changed, 12 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f60a1c66e43..4a2d3d8a9a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Beta CI +name: CI on: push: branches: [master] @@ -217,6 +217,9 @@ jobs: setup-bb-bench: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} diff --git a/.github/workflows/protocol-circuits-gate-diff.yml b/.github/workflows/protocol-circuits-gate-diff.yml index 7905f0e6a1e..8126a13193c 100644 --- a/.github/workflows/protocol-circuits-gate-diff.yml +++ b/.github/workflows/protocol-circuits-gate-diff.yml @@ -4,7 +4,7 @@ on: push: branches: - master - pull_request: {} + pull_request: jobs: compare_protocol_circuits_gates: diff --git a/.vscode/settings.json b/.vscode/settings.json index 0c9b214de1e..98b15088b50 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -160,6 +160,5 @@ "noir/noir-repo/Cargo.toml", "noir/noir-repo/acvm-repo/acvm_js/Cargo.toml", "avm-transpiler/Cargo.toml" - ], - "cmake.sourceDirectory": "/mnt/user-data/adam/aztec-packages/barretenberg/cpp" + ] } diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile index a93c88ba5cf..efa14c84819 100644 --- a/l1-contracts/Earthfile +++ b/l1-contracts/Earthfile @@ -20,4 +20,4 @@ build: COPY --dir lib scripts src terraform test *.json *.toml *.sh . RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff RUN forge clean && forge fmt --check && forge build && forge test --no-match-contract UniswapPortalTest - SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts + SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts \ No newline at end of file diff --git a/yarn-project/circuit-types/src/stats/metrics.ts b/yarn-project/circuit-types/src/stats/metrics.ts index 467c80c7839..4ee8ea7e04c 100644 --- a/yarn-project/circuit-types/src/stats/metrics.ts +++ b/yarn-project/circuit-types/src/stats/metrics.ts @@ -8,8 +8,7 @@ export type MetricGroupBy = | 'classes-registered' | 'leaf-count' | 'data-writes' - | 'fee-payment-method' - | 'circuit-size-in-gates'; + | 'fee-payment-method'; /** Definition of a metric to track in benchmarks. */ export interface Metric { @@ -25,18 +24,6 @@ export interface Metric { /** Metric definitions to track from benchmarks. */ export const Metrics = [ - { - name: 'client_ivc_proving_time_in_ms', - groupBy: 'circuit-size-in-gates', - description: 'Proving time for ClientIVC grouped by circuit size.', - events: [], - }, - { - name: 'ultrahonk_proving_time_in_ms', - groupBy: 'circuit-size-in-gates', - description: 'Proving time for UltraHonk grouped by circuit size.', - events: [], - }, { name: 'l1_rollup_calldata_size_in_bytes', groupBy: 'block-size', diff --git a/yarn-project/scripts/src/benchmarks/aggregate.ts b/yarn-project/scripts/src/benchmarks/aggregate.ts index 6440cb14e36..ce1d8905463 100644 --- a/yarn-project/scripts/src/benchmarks/aggregate.ts +++ b/yarn-project/scripts/src/benchmarks/aggregate.ts @@ -228,15 +228,6 @@ function processEntry(entry: Stats, results: BenchmarkCollectedResults, fileName } } -function getBarretenbergMetric(context: { executable?: string }): MetricName | undefined { - if (context.executable?.includes('ultra_honk')) { - return 'ultrahonk_proving_time_in_ms'; - } else if (context.executable?.includes('client_ivc')) { - return 'client_ivc_proving_time_in_ms'; - } - return undefined; -} - /** Array of collected raw results for a given metric. */ type BenchmarkCollectedMetricResults = Record; @@ -262,8 +253,7 @@ export async function main() { } } - // Spammy if on by default - // log(`Collected entries: ${JSON.stringify(collected)}`); + log(`Collected entries: ${JSON.stringify(collected)}`); // For each bucket of each metric compute the average all collected data points const results: BenchmarkResults = {}; @@ -279,28 +269,6 @@ export async function main() { } } - // Add google benchmark json files, which have data already averaged - const googleBenchmarkFiles = fs.readdirSync(LogsDir).filter(f => f.endsWith('.json')); - for (const file of googleBenchmarkFiles) { - const data = JSON.parse(fs.readFileSync(path.join(LogsDir, file), 'utf-8')); - if (!data.context || !data.benchmarks) { - log(`Invalid google benchmark file: ${file}`); - continue; - } - const metric = getBarretenbergMetric(data.context); - if (!metric) { - log(`Unknown executable in benchmark file ${file}: ${data.context.executable}`); - continue; - } - const circuitSize = '2^20'; // Where to load size from? - const value = data.benchmarks[0]?.real_time; - if (value === undefined) { - log(`Couldn't find real_time in benchmark file ${file}`); - continue; - } - results[metric] = { [circuitSize]: value }; - } - const timestampedResults: BenchmarkResultsWithTimestamp = { ...results, timestamp: new Date().toISOString() }; // Write results to disk diff --git a/yarn-project/scripts/src/benchmarks/markdown.ts b/yarn-project/scripts/src/benchmarks/markdown.ts index f803cd683ca..2887805cfdc 100644 --- a/yarn-project/scripts/src/benchmarks/markdown.ts +++ b/yarn-project/scripts/src/benchmarks/markdown.ts @@ -189,13 +189,12 @@ export function getMarkdown() { const metricsByClassesRegistered = Metrics.filter(m => m.groupBy === 'classes-registered').map(m => m.name); const metricsByFeePaymentMethod = Metrics.filter(m => m.groupBy === 'fee-payment-method').map(m => m.name); const metricsByLeafCount = Metrics.filter(m => m.groupBy === 'leaf-count').map(m => m.name); - const metricsByCircuitSize = Metrics.filter(m => m.groupBy === 'circuit-size-in-gates').map(m => m.name); const metricsTxPxeProcessing = Metrics.filter(m => m.name === 'tx_pxe_processing_time_ms').map(m => m.name); const metricsTxSeqProcessing = Metrics.filter(m => m.name === 'tx_sequencer_processing_time_ms').map(m => m.name); - const baseUrlPath = process.env.BASE_BENCH_PATH; - const baseUrl = baseUrlPath && `[\`${baseUrlPath.slice(0, 8)}\`](${S3_URL}/benchmarks-v1/master/${baseUrlPath}.json)`; + const baseHash = process.env.BASE_COMMIT_HASH; + const baseUrl = baseHash && `[\`${baseHash.slice(0, 8)}\`](${S3_URL}/benchmarks-v1/master/${baseHash}.json)`; const baseCommitText = baseUrl ? `\nValues are compared against data from master at commit ${baseUrl} and shown if the difference exceeds 1%.` : ''; @@ -215,15 +214,10 @@ ${getWarningsSummary(benchmark, baseBenchmark)} Detailed results -Except for proving times, all benchmarks are run on txs on the \`Benchmarking\` contract on the repository. Each tx consists of a batch call to \`create_note\` and \`increment_balance\`, which guarantees that each tx has a private call, a nested private call, a public call, and a nested public call, as well as an emitted private note, an unencrypted log, and public storage read and write. +All benchmarks are run on txs on the \`Benchmarking\` contract on the repository. Each tx consists of a batch call to \`create_note\` and \`increment_balance\`, which guarantees that each tx has a private call, a nested private call, a public call, and a nested public call, as well as an emitted private note, an unencrypted log, and public storage read and write. ${prSourceDataText} ${baseCommitText} -### Proving times - -Average proving times for benchmarks measured in bb. -${getTableContent(pick(benchmark, metricsByCircuitSize), baseBenchmark, 'gates', 'Circuit')} - ### L2 block published to L1 Each column represents the number of txs on an L2 block published to L1. From 34666943319d83601493df8ae3a506230a5188c4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 19:42:28 +0000 Subject: [PATCH 131/161] revert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5abbc1903d8..38e03d97604 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ noirup -v TAG_FROM_THE_FILE This repository uses CircleCI for continuous integration. Build steps are managed using [`build-system`](https://github.com/AztecProtocol/build-system). Small packages are built and tested as part of a docker build operation, while larger ones and end-to-end tests spin up a large AWS spot instance. Each successful build step creates a new docker image that gets tagged with the package name and commit. -All packages need to be included in the [build manifest](`build_manifest.yml`), which declares what paths belong to each package, as well as dependencies between packages. When the CI runs, if none of the rebuild patterns or dependencies were changed, then the build step is skipped and the last successful image is re-tagged with the current commit. Read more on the [`build-system`](https://github.com/AztecProtocol/build-system) repository README. +All packages need to be included in the [build manifest](`build_manifest.json`), which declares what paths belong to each package, as well as dependencies between packages. When the CI runs, if none of the rebuild patterns or dependencies were changed, then the build step is skipped and the last successful image is re-tagged with the current commit. Read more on the [`build-system`](https://github.com/AztecProtocol/build-system) repository README. It is faster to debug CI failures within a persistent ssh session compared to pushing and waiting. You can create a session with "Rerun step with SSH" on CircleCI which will generate an ssh command for debugging on a worker. Run that command locally and then do From 6b96870123290d48ee6dafa7a181dc0264d194ed Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 19:43:59 +0000 Subject: [PATCH 132/161] Don't restart --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a2d3d8a9a3..c20a95dd468 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'restart' }} + subaction: ${{ github.event.inputs.runner_action || 'start' }} # prevent reaping by mainframe spot reaper ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 From 922b6fe1c6df3b3e21969f98cf55491443e577a5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 19:44:52 +0000 Subject: [PATCH 133/161] Use 100gb cache --- .github/earthly-ci-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/earthly-ci-config.yml b/.github/earthly-ci-config.yml index ffea16b6d09..453eebea37c 100644 --- a/.github/earthly-ci-config.yml +++ b/.github/earthly-ci-config.yml @@ -1,5 +1,5 @@ global: - cache_size_mb: 250000 + cache_size_mb: 100000 buildkit_max_parallelism: 50 container_frontend: docker-shell buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"] \ No newline at end of file From 49ab388fc10215bcd0d8087918e9d414d97ce58f Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 20:02:48 +0000 Subject: [PATCH 134/161] default cache probably fine --- .github/earthly-ci-config.yml | 3 +-- scripts/ci/upload_logs_to_s3 | 32 -------------------------------- 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100755 scripts/ci/upload_logs_to_s3 diff --git a/.github/earthly-ci-config.yml b/.github/earthly-ci-config.yml index 453eebea37c..0c041246568 100644 --- a/.github/earthly-ci-config.yml +++ b/.github/earthly-ci-config.yml @@ -1,5 +1,4 @@ global: - cache_size_mb: 100000 buildkit_max_parallelism: 50 container_frontend: docker-shell - buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"] \ No newline at end of file +buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"] \ No newline at end of file diff --git a/scripts/ci/upload_logs_to_s3 b/scripts/ci/upload_logs_to_s3 deleted file mode 100755 index b13c0ebd456..00000000000 --- a/scripts/ci/upload_logs_to_s3 +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace - -# Uploads to S3 the contents of the log file mounted on the end-to-end container, -# which contains log entries with an associated event and metrics for it. -# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl -# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR - -set -eu - -LOG_FOLDER=$1 -BUCKET_NAME="aztec-ci-artifacts" -COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" - -if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then - echo "No logs in folder $LOG_FOLDER to upload" - exit 0 -fi - -# Paths used in scripts/ci/assemble_e2e_benchmark.sh -if [ "${BRANCH:-}" = "master" ]; then - TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" -elif [ -n "${PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" -fi - -if [ -n "${TARGET_FOLDER:-}" ]; then - aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive -else - echo Skipping upload since no target folder was defined -fi \ No newline at end of file From 5fb872c8cf9f2d783c3e5ce3ac65ad0dc385c160 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 20:22:31 +0000 Subject: [PATCH 135/161] [ci rebuild] redo --- .github/ci-setup-action/action.yml | 11 +++++++++++ .github/workflows/ci.yml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 13b6d718e8f..4e81173acdb 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -43,3 +43,14 @@ runs: - name: Setup Env shell: bash run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} + + - name: Setup Docker + shell: bash + run: | + if ! [ -f /etc/docker/daemon.json ] ; then + echo '{"default-address-pools":[{"base":"172.17.0.0/12","size":20}, {"base":"10.99.0.0/12","size":20}, {"base":"192.168.0.0/16","size":24}]}' > /etc/docker/daemon.json + sudo service docker restart + echo "Configured docker daemon for making many networks." + else + echo "Docker daemon already configured." + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c20a95dd468..4a2d3d8a9a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} + subaction: ${{ github.event.inputs.runner_action || 'restart' }} # prevent reaping by mainframe spot reaper ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 From 5bf145e6ee48703bbc3c4b7e2c84ecb6b07c30c0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 20:28:46 +0000 Subject: [PATCH 136/161] Always wait for ebs --- .github/workflows/ci.yml | 6 +++--- scripts/attach_ebs_cache.sh | 40 +++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a2d3d8a9a3..eaa27361560 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,14 +31,14 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v0.2 + uses: AztecProtocol/ec2-action-builder@v0.3 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'restart' }} + subaction: ${{ github.event.inputs.runner_action || 'start' }} # prevent reaping by mainframe spot reaper ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 @@ -193,7 +193,7 @@ jobs: steps: - name: Start EC2 runner id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v0.2 + uses: AztecProtocol/ec2-action-builder@v0.3 with: github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index 147559660d1..a871eaf2547 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -34,30 +34,32 @@ if [ "$EXISTING_VOLUME" == "None" ]; then --tag-specifications "ResourceType=volume,Tags=[{Key=username,Value=$EBS_CACHE_TAG}]" \ --query "VolumeId" \ --output text) - MAX_WAIT_TIME=300 # Maximum wait time in seconds - WAIT_INTERVAL=10 # Interval between checks in seconds - elapsed_time=0 - # Wait for the volume to become available - echo "Waiting for volume $VOLUME_ID to become available..." - while [ "$(aws ec2 describe-volumes \ - --region $REGION \ - --volume-ids $VOLUME_ID \ - --query "Volumes[0].State" \ - --output text)" != "available" ]; do - sleep 1 - if [ $elapsed_time -ge $MAX_WAIT_TIME ]; then - echo "Volume $VOLUME_ID did not become available within $MAX_WAIT_TIME seconds." - exit 1 - fi - - sleep $WAIT_INTERVAL - elapsed_time=$((elapsed_time + WAIT_INTERVAL)) - done else VOLUME_ID=$EXISTING_VOLUME fi +MAX_WAIT_TIME=300 # Maximum wait time in seconds +WAIT_INTERVAL=10 # Interval between checks in seconds +elapsed_time=0 +# Wait for the volume to become available +echo "Waiting for volume $VOLUME_ID to become available..." +while [ "$(aws ec2 describe-volumes \ + --region $REGION \ + --volume-ids $VOLUME_ID \ + --query "Volumes[0].State" \ + --output text)" != "available" ]; do + sleep 1 + if [ $elapsed_time -ge $MAX_WAIT_TIME ]; then + echo "Volume $VOLUME_ID did not become available within $MAX_WAIT_TIME seconds." + exit 1 + fi + + sleep $WAIT_INTERVAL + elapsed_time=$((elapsed_time + WAIT_INTERVAL)) +done + # Attach volume to the instance + aws ec2 attach-volume \ --region $REGION \ --volume-id $VOLUME_ID \ From fe45bfba7127caaf17da10533f1c51e094d719b2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 21:04:16 +0000 Subject: [PATCH 137/161] runner --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaa27361560..cb00cfbbae1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: config: - - {ec2_instance_type: m6a.16xlarge, runner_concurrency: 45, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + - {ec2_instance_type: m6a.16xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 2, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} steps: - name: Start EC2 runner From fb95343f74202cdd5fb0ec703263f8eb6a01073d Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 21:08:16 +0000 Subject: [PATCH 138/161] [ci rebuild] fix bench-tx-size --- yarn-project/end-to-end/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index d78a452164d..2f49410dede 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -276,4 +276,4 @@ bench-process-history: # TODO need to investigate why this isn't working bench-tx-size: ARG e2e_mode=local - DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=true --compose_file=./scripts/docker-compose-no-sandbox.yml From ab909ee724e77d58be9052818b23330aad826a06 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:00:54 +0000 Subject: [PATCH 139/161] try better cache settings --- .github/earthly-ci-config.yml | 4 ---- .github/workflows/ci.yml | 4 +++- barretenberg/cpp/Earthfile | 14 +++++++++++++- scripts/setup_env.sh | 3 +-- yarn-project/end-to-end/Earthfile | 2 -- 5 files changed, 17 insertions(+), 10 deletions(-) delete mode 100644 .github/earthly-ci-config.yml diff --git a/.github/earthly-ci-config.yml b/.github/earthly-ci-config.yml deleted file mode 100644 index 0c041246568..00000000000 --- a/.github/earthly-ci-config.yml +++ /dev/null @@ -1,4 +0,0 @@ -global: - buildkit_max_parallelism: 50 - container_frontend: docker-shell -buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"] \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb00cfbbae1..654891d602e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 + - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' - run: earthly bootstrap build-arm: @@ -107,10 +108,10 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # Attach our 128gb cache disk - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 + - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-x86: @@ -225,6 +226,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 + - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap bb-bench: diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index de96f18f227..ffb37203b7f 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -210,8 +210,20 @@ test-clang-format: COPY format.sh . RUN ./format.sh check +test-base: + ARG EARTHLY_GIT_HASH + ARG TARGETARCH + ARG test_mode=build + LOCALLY + IF [ $test_mode = cache ] + FROM aztecprotocol/test-base:$TARGETARCH-$EARTHLY_GIT_HASH + ELSE + FROM +preset-release-assert-test + SAVE IMAGE --push aztecprotocol/test-base:$TARGETARCH-$EARTHLY_GIT_HASH + END + test: BUILD +test-clang-format - FROM +preset-release-assert-test + FROM test-base COPY --dir ./srs_db/+build/. srs_db RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index 97fb8fd68a1..9fe6201738f 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -9,5 +9,4 @@ echo "Logging in to Docker..." echo $1 | docker login -u aztecprotocolci --password-stdin # Make earthly-cloud and earthly-cloud-bench scripts available -echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV -echo "EARTHLY_CONFIG=$(git rev-parse --show-toplevel)/.github/earthly-ci-config.yml" >> $GITHUB_ENV \ No newline at end of file +echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV \ No newline at end of file diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 2f49410dede..2aa42c7e138 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -268,12 +268,10 @@ bench-publish-rollup: ARG e2e_mode=local DO +E2E_TEST --test=benchmarks/bench_publish_rollup.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml -# TODO need to investigate why this isn't working bench-process-history: ARG e2e_mode=local DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml -# TODO need to investigate why this isn't working bench-tx-size: ARG e2e_mode=local DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=true --compose_file=./scripts/docker-compose-no-sandbox.yml From c6a7a08657b710d63e5060bee9b12969ae330e67 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:07:35 +0000 Subject: [PATCH 140/161] fix earthly config --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 654891d602e..40b3b53b10a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} + subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} # prevent reaping by mainframe spot reaper ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 @@ -65,7 +65,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' + - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' - run: earthly bootstrap build-arm: @@ -111,7 +111,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-x86: @@ -226,7 +226,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap bb-bench: From 3870ace2a7bd1bd8bee263edab71021ad610b28e Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:07:35 +0000 Subject: [PATCH 141/161] fix earthly config [ci restart-spot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 654891d602e..40b3b53b10a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: "us-east-2" ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} + subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} # prevent reaping by mainframe spot reaper ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' github_action_runner_version: v2.315.0 @@ -65,7 +65,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' + - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' - run: earthly bootstrap build-arm: @@ -111,7 +111,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-x86: @@ -226,7 +226,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - - run: '"global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap bb-bench: From fb6079f999d3bdb9d904730849c15b7121a96b7f Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:09:00 +0000 Subject: [PATCH 142/161] [ci rebuild][ci restart-spot] From 425566d21af1be0580432962b7065cb172e5eb16 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:10:29 +0000 Subject: [PATCH 143/161] earthly path fix [ci rebuild][ci restart-spot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40b3b53b10a..55dd832b180 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' + - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' - run: earthly bootstrap build-arm: @@ -111,7 +111,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-x86: @@ -226,7 +226,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - - run: 'echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap bb-bench: From 13aff198b1ee60ecb9a0329fa3c46c1e9e7dc0d1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:19:44 +0000 Subject: [PATCH 144/161] try fix cache setting [ci spot-restart] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55dd832b180..a583095db96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50}" > ~/.earthly/config.yml' + - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_pct: 90, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-arm: @@ -111,7 +111,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_pct: 90, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-x86: @@ -226,7 +226,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_mb: 110000, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' + - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_pct: 90, buildkit_max_parallelism: 1, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap bb-bench: From 336cab1f9fb25929f1eac4bdb70d7f56345c00a8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 18:20:36 -0400 Subject: [PATCH 145/161] Update docker-compose.yml --- yarn-project/end-to-end/scripts/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index 0d5fed41c8e..3987fd269b9 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -16,7 +16,7 @@ services: image: aztecprotocol/aztec:${AZTEC_DOCKER_TAG:-latest} environment: DEBUG: 'aztec:*' - DEBUG_COLORS: 1A + DEBUG_COLORS: 1 ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 ARCHIVER_POLLING_INTERVAL_MS: 50 From 3cb0b413c14cf479c1877c371fe6d57882560bc7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:24:02 +0000 Subject: [PATCH 146/161] reinstate static config file after research [ci restart-spot] --- .github/earthly-ci-config.yml | 5 +++++ .github/workflows/ci.yml | 3 --- scripts/setup_env.sh | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 .github/earthly-ci-config.yml diff --git a/.github/earthly-ci-config.yml b/.github/earthly-ci-config.yml new file mode 100644 index 00000000000..398a2717c67 --- /dev/null +++ b/.github/earthly-ci-config.yml @@ -0,0 +1,5 @@ +global: + cache_size_pct: 90 + buildkit_max_parallelism: 50 + container_frontend: docker-shell +buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"] \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a583095db96..cfdd562a850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,6 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_pct: 90, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-arm: @@ -111,7 +110,6 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 128gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_pct: 90, buildkit_max_parallelism: 50, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap build-x86: @@ -226,7 +224,6 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # Attach our 328gb cache disk - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - - run: 'mkdir -p ~/.earthly && echo "global: {cache_size_pct: 90, buildkit_max_parallelism: 1, buildkit_additional_args: ["-e", "BUILDKIT_STEP_LOG_MAX_SIZE=-1"]}" > ~/.earthly/config.yml' - run: earthly bootstrap bb-bench: diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index 9fe6201738f..97fb8fd68a1 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -9,4 +9,5 @@ echo "Logging in to Docker..." echo $1 | docker login -u aztecprotocolci --password-stdin # Make earthly-cloud and earthly-cloud-bench scripts available -echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV \ No newline at end of file +echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV +echo "EARTHLY_CONFIG=$(git rev-parse --show-toplevel)/.github/earthly-ci-config.yml" >> $GITHUB_ENV \ No newline at end of file From ada047ea5951c91f63b71dec034e029f2db722d3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 22:59:27 +0000 Subject: [PATCH 147/161] [ci restart-spot] From e63c46059006c06420de4b5051d7f295a2d0de5e Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:21:32 +0000 Subject: [PATCH 148/161] change mountpoint. [ci restart-spot] --- .github/workflows/ci.yml | 8 ++++---- barretenberg/cpp/Earthfile | 14 +------------- scripts/attach_ebs_cache.sh | 8 ++++---- yarn-project/Earthfile | 2 +- yarn-project/end-to-end/Earthfile | 7 +++---- 5 files changed, 13 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfdd562a850..2a0bdce9ca0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,8 +75,8 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # prepare images, tagged by commit hash - - run: earthly ./yarn-project+build-end-to-end + # prepare images locally, tagged by commit hash + - run: earthly ./yarn-project+push-end-to-end # all the end-to-end integration tests for aztec e2e-arm: @@ -120,8 +120,8 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # prepare images, tagged by commit hash - - run: earthly ./yarn-project+build-end-to-end + # prepare images locally, tagged by commit hash + - run: earthly ./yarn-project+push-end-to-end # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end # (Note ARM uses just 2 tests as a smoketest) - name: Create list of end-to-end jobs diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index ffb37203b7f..de96f18f227 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -210,20 +210,8 @@ test-clang-format: COPY format.sh . RUN ./format.sh check -test-base: - ARG EARTHLY_GIT_HASH - ARG TARGETARCH - ARG test_mode=build - LOCALLY - IF [ $test_mode = cache ] - FROM aztecprotocol/test-base:$TARGETARCH-$EARTHLY_GIT_HASH - ELSE - FROM +preset-release-assert-test - SAVE IMAGE --push aztecprotocol/test-base:$TARGETARCH-$EARTHLY_GIT_HASH - END - test: BUILD +test-clang-format - FROM test-base + FROM +preset-release-assert-test COPY --dir ./srs_db/+build/. srs_db RUN cd build && GTEST_COLOR=1 ctest -j$(nproc) --output-on-failure diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index a871eaf2547..e5c8d77b66d 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -9,8 +9,8 @@ VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) # Check for existing mount, assume we can continue if existing -if mount | grep -q /var/lib/docker/volumes; then - echo "Detected mount existing on /var/lib/docker/volumes already" +if mount | grep -q /var/lib/docker; then + echo "Detected mount existing on /var/lib/docker already" echo "Continuing..." exit 0 fi @@ -98,5 +98,5 @@ if ! file -s $BLKDEVICE | grep -q ext4; then fi # Create a mount point and mount the volume -mkdir -p /var/lib/docker/volumes -mount $BLKDEVICE /var/lib/docker/volumes +mkdir -p /var/lib/docker +mount $BLKDEVICE /var/lib/docker diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index c0f563e5c7f..e2b7e9e6de1 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -79,7 +79,7 @@ all: BUILD +cli BUILD +end-to-end -build-end-to-end: +push-end-to-end: ARG EARTHLY_GIT_HASH # pushes the foundry image BUILD ../foundry/+build diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 2aa42c7e138..6ed128441b7 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -# requires first saving the images locally with ../+build-end-to-end +# requires first saving the images locally with ../+push-end-to-end # run locally and build E2E_TEST_LOCAL: @@ -12,8 +12,7 @@ E2E_TEST_LOCAL: LOCALLY ENV ENABLE_GAS=$enable_gas ENV TEST=$test - ENV DEBUG="$debug" - ENV AZTEC_DOCKER_TAG= + ENV DEBUG=$debug WITH DOCKER \ --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end \ @@ -33,7 +32,7 @@ E2E_TEST_FROM_CACHE: LOCALLY ENV ENABLE_GAS=$enable_gas ENV TEST=$test - ENV DEBUG="$debug" + ENV DEBUG=$debug ENV AZTEC_DOCKER_TAG=$aztec_docker_tag # need a different project name for each to run in parallel LET project_name=$(echo $test | sed 's/\./_/g') From 7141e25c6ef0c9692d18b2e3d2a2ca3619f40169 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:25:09 +0000 Subject: [PATCH 149/161] export e2e locally [ci restart-spot] --- .github/workflows/ci.yml | 4 ++-- foundry/Earthfile | 8 +------- yarn-project/Earthfile | 12 +++++++----- yarn-project/end-to-end/Earthfile | 2 +- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a0bdce9ca0..b88e08f4780 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # prepare images locally, tagged by commit hash - - run: earthly ./yarn-project+push-end-to-end + - run: earthly ./yarn-project+export-end-to-end # all the end-to-end integration tests for aztec e2e-arm: @@ -121,7 +121,7 @@ jobs: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} # prepare images locally, tagged by commit hash - - run: earthly ./yarn-project+push-end-to-end + - run: earthly ./yarn-project+export-end-to-end # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end # (Note ARM uses just 2 tests as a smoketest) - name: Create list of end-to-end jobs diff --git a/foundry/Earthfile b/foundry/Earthfile index f14a5502e5a..8742867c672 100644 --- a/foundry/Earthfile +++ b/foundry/Earthfile @@ -13,10 +13,4 @@ build: ARG TARGETARCH COPY --dir +builder/usr/src/foundry/bin /usr/src/foundry/bin SAVE ARTIFACT /usr/src/foundry/bin /usr/src/foundry/bin - SAVE IMAGE --push aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991-$TARGETARCH:latest - -build-save-local: - # for renaming the image as a local image, used with e2e runners - ARG image - FROM +build - SAVE IMAGE $image \ No newline at end of file + SAVE IMAGE --push aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991-$TARGETARCH:latest \ No newline at end of file diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index e2b7e9e6de1..75316122d4e 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -79,11 +79,13 @@ all: BUILD +cli BUILD +end-to-end -push-end-to-end: +# for use with yarn-project/end-to-end and its e2e_mode=cache option +export-end-to-end: ARG EARTHLY_GIT_HASH - # pushes the foundry image - BUILD ../foundry/+build + # pushes the foundry image to local docker images + FROM ../foundry/+build + SAVE IMAGE aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991:latest FROM +end-to-end - SAVE IMAGE --push aztecprotocol/end-to-end:$EARTHLY_GIT_HASH + SAVE IMAGE aztecprotocol/end-to-end:$EARTHLY_GIT_HASH FROM +aztec - SAVE IMAGE --push aztecprotocol/aztec:$EARTHLY_GIT_HASH + SAVE IMAGE aztecprotocol/aztec:$EARTHLY_GIT_HASH diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 6ed128441b7..9fdf96a53a6 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -1,6 +1,6 @@ VERSION 0.8 -# requires first saving the images locally with ../+push-end-to-end +# requires first saving the images locally with ../+export-end-to-end # run locally and build E2E_TEST_LOCAL: From e996164bc9375ddd565e01e4b316e86995aa30cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:27:55 +0000 Subject: [PATCH 150/161] replace with todo --- scripts/attach_ebs_cache.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/attach_ebs_cache.sh b/scripts/attach_ebs_cache.sh index e5c8d77b66d..4bce6c775ba 100755 --- a/scripts/attach_ebs_cache.sh +++ b/scripts/attach_ebs_cache.sh @@ -8,9 +8,11 @@ AVAILABILITY_ZONE="us-east-2a" VOLUME_TYPE="gp2" INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) +# TODO also mount various other aspects of docker image metadata + # Check for existing mount, assume we can continue if existing -if mount | grep -q /var/lib/docker; then - echo "Detected mount existing on /var/lib/docker already" +if mount | grep -q /var/lib/docker/volumes; then + echo "Detected mount existing on /var/lib/docker/volumes already" echo "Continuing..." exit 0 fi @@ -98,5 +100,5 @@ if ! file -s $BLKDEVICE | grep -q ext4; then fi # Create a mount point and mount the volume -mkdir -p /var/lib/docker -mount $BLKDEVICE /var/lib/docker +mkdir -p /var/lib/docker/volumes +mount $BLKDEVICE /var/lib/docker/volumes From e3b6bafa814ceabf643a2c8e0742e019210ca5e8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:30:18 +0000 Subject: [PATCH 151/161] fix path --- foundry/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/foundry/Earthfile b/foundry/Earthfile index 8742867c672..14262b0b3fb 100644 --- a/foundry/Earthfile +++ b/foundry/Earthfile @@ -13,4 +13,5 @@ build: ARG TARGETARCH COPY --dir +builder/usr/src/foundry/bin /usr/src/foundry/bin SAVE ARTIFACT /usr/src/foundry/bin /usr/src/foundry/bin + ENV PATH="${PATH}:/usr/src/foundry/bin" SAVE IMAGE --push aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991-$TARGETARCH:latest \ No newline at end of file From 44ce605748d2469fd8c9b00e5682a198b58de54b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:36:33 +0000 Subject: [PATCH 152/161] try build mode --- .github/workflows/ci.yml | 2 +- yarn-project/end-to-end/Earthfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b88e08f4780..3f4d5e28a42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Test working-directory: ./yarn-project/end-to-end/ - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=build #--e2e_mode=cache ######################### # START OF x86 PIPELINE # diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 9fdf96a53a6..222c5ffc99c 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -273,4 +273,4 @@ bench-process-history: bench-tx-size: ARG e2e_mode=local - DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=true --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml From bb9551a056a5c320ede0459acd8698d0d7477a9c Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:40:34 +0000 Subject: [PATCH 153/161] retry with proper fork --- yarn-project/end-to-end/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 222c5ffc99c..d9b8aaef243 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -16,7 +16,7 @@ E2E_TEST_LOCAL: WITH DOCKER \ --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end \ - --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+build + --load aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991:latest=../../foundry/+build # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END @@ -60,7 +60,7 @@ E2E_TEST_FROM_BUILD: WITH DOCKER \ --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end \ - --load ghcr.io/foundry-rs/foundry:nightly-de33b6af53005037b463318d2628b5cfcaf39916=../../foundry/+build + --load aztecprotocol/foundry-nightly-de33b6af53005037b463318d2628b5cfcaf3991:latest=../../foundry/+build # Run our docker compose, ending whenever sandbox ends, filtering out noisy eth_getLogs RUN docker compose -f $compose_file up --exit-code-from=end-to-end --force-recreate END From 1fb6317c997a8d50531f954c24fe29c30237870d Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 9 Apr 2024 23:44:11 +0000 Subject: [PATCH 154/161] no dice --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f4d5e28a42..b88e08f4780 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Test working-directory: ./yarn-project/end-to-end/ - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=build #--e2e_mode=cache + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache ######################### # START OF x86 PIPELINE # From dca8838cf6ccaa30acecb0cd890fbd17ee4e48b2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:05:27 +0000 Subject: [PATCH 155/161] test ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b88e08f4780..65d422cded2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,7 +188,7 @@ jobs: name: Start Bench Runner runs-on: ubuntu-latest permissions: - actions: write + actions: write steps: - name: Start EC2 runner id: start-ec2-runner From 5b6f4cefc61cd4a93c8bb7c9a076420bf7d4a540 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:19:41 +0000 Subject: [PATCH 156/161] [ci restart-spot] From 67bb7990fc2ff8ec81066404168225703d4decf0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:38:05 +0000 Subject: [PATCH 157/161] add ability to just start spot --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65d422cded2..aecc755270f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,10 @@ on: runner_action: description: "The action to take with the self-hosted runner (start, stop, restart)." required: false + just_start_spot: + description: "Should we just run spots?" + type: boolean + required: false concurrency: # force parallelism group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} @@ -57,6 +61,7 @@ jobs: setup-arm: needs: start-builder runs-on: ${{ github.actor }}-arm + if: ${{ github.event.inputs.just_start_spot }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -102,6 +107,7 @@ jobs: setup-x86: needs: start-builder runs-on: ${{ github.actor }}-x86 + if: ${{ github.event.inputs.just_start_spot }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -216,6 +222,7 @@ jobs: setup-bb-bench: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner + if: ${{ github.event.inputs.just_start_spot }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From cd84a130a2cbed97efb05faa9dbf84fec967f63c Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:48:43 +0000 Subject: [PATCH 158/161] fix start spot check --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aecc755270f..03b5d87ff5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: setup-arm: needs: start-builder runs-on: ${{ github.actor }}-arm - if: ${{ github.event.inputs.just_start_spot }} + if: ${{ github.event.inputs.just_start_spot == 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -107,7 +107,7 @@ jobs: setup-x86: needs: start-builder runs-on: ${{ github.actor }}-x86 - if: ${{ github.event.inputs.just_start_spot }} + if: ${{ github.event.inputs.just_start_spot == 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -222,7 +222,7 @@ jobs: setup-bb-bench: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner - if: ${{ github.event.inputs.just_start_spot }} + if: ${{ github.event.inputs.just_start_spot == 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 42a79d2d9fa70a84a289064da282e2d13f1072bb Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:51:52 +0000 Subject: [PATCH 159/161] better spot only mode --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03b5d87ff5d..6c556a783c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,6 @@ jobs: setup-arm: needs: start-builder runs-on: ${{ github.actor }}-arm - if: ${{ github.event.inputs.just_start_spot == 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -75,6 +74,7 @@ jobs: build-arm: needs: setup-arm runs-on: ${{ github.actor }}-arm + if: ${{ github.event.inputs.just_start_spot == 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} steps: @@ -107,7 +107,6 @@ jobs: setup-x86: needs: start-builder runs-on: ${{ github.actor }}-x86 - if: ${{ github.event.inputs.just_start_spot == 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -121,6 +120,7 @@ jobs: build-x86: needs: setup-x86 runs-on: ${{ github.actor }}-x86 + if: ${{ github.event.inputs.just_start_spot == 'true' }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} steps: @@ -184,6 +184,7 @@ jobs: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Build and Push Binaries + if: ${{ github.event.inputs.just_start_spot == 'false' }} working-directory: ./barretenberg/cpp/ run: earthly --push +bench-base From df04897fb02cc0adec977d4b288dae8258ca363e Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:54:12 +0000 Subject: [PATCH 160/161] fix just_start_spot --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c556a783c7..2427e132a48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: build-arm: needs: setup-arm runs-on: ${{ github.actor }}-arm - if: ${{ github.event.inputs.just_start_spot == 'true' }} + if: ${{ github.event.inputs.just_start_spot == 'false' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} steps: @@ -120,7 +120,7 @@ jobs: build-x86: needs: setup-x86 runs-on: ${{ github.actor }}-x86 - if: ${{ github.event.inputs.just_start_spot == 'true' }} + if: ${{ github.event.inputs.just_start_spot == 'false' }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} steps: @@ -223,7 +223,7 @@ jobs: setup-bb-bench: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner - if: ${{ github.event.inputs.just_start_spot == 'true' }} + if: ${{ github.event.inputs.just_start_spot == 'false' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} From 2c8266cb3578a7e53ff678c975a0c88294e11796 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 10 Apr 2024 12:57:12 +0000 Subject: [PATCH 161/161] fix --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2427e132a48..77f4301befd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: build-arm: needs: setup-arm runs-on: ${{ github.actor }}-arm - if: ${{ github.event.inputs.just_start_spot == 'false' }} + if: ${{ github.event.inputs.just_start_spot != 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} steps: @@ -120,7 +120,7 @@ jobs: build-x86: needs: setup-x86 runs-on: ${{ github.actor }}-x86 - if: ${{ github.event.inputs.just_start_spot == 'false' }} + if: ${{ github.event.inputs.just_start_spot != 'true' }} outputs: e2e_list: ${{ steps.e2e_list.outputs.list }} steps: @@ -184,7 +184,7 @@ jobs: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - name: Build and Push Binaries - if: ${{ github.event.inputs.just_start_spot == 'false' }} + if: ${{ github.event.inputs.just_start_spot != 'true' }} working-directory: ./barretenberg/cpp/ run: earthly --push +bench-base @@ -223,7 +223,7 @@ jobs: setup-bb-bench: runs-on: ${{ github.actor }}-bench-x86 needs: start-bb-bench-runner - if: ${{ github.event.inputs.just_start_spot == 'false' }} + if: ${{ github.event.inputs.just_start_spot != 'true' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}