From a4e9214f3cd4cca6cd77cc5df7f748e02e6aebe5 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 9 Oct 2024 13:57:52 +0000 Subject: [PATCH 01/19] feat: use s3 cache in bootstrap fast --- avm-transpiler/Earthfile | 7 +++++- avm-transpiler/bootstrap_cache.sh | 6 ++--- barretenberg/cpp/bootstrap_cache.sh | 23 ++++++++++++------- bootstrap.sh | 2 +- .../s3-cache-scripts/cache-download.sh | 6 +++-- build-system/s3-cache-scripts/cache-upload.sh | 2 +- build-system/scripts/can_use_ci_cache | 10 +++++++- build-system/scripts/setup_env | 4 ++-- l1-contracts/.rebuild_patterns | 1 + l1-contracts/Earthfile | 23 ++++++++++++++----- l1-contracts/bootstrap_cache.sh | 5 ++-- noir/bootstrap_cache.sh | 10 ++++---- scripts/earthly-local | 3 +-- yarn-project/Earthfile | 2 +- 14 files changed, 67 insertions(+), 37 deletions(-) create mode 100644 l1-contracts/.rebuild_patterns diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile index 95199fbe3d2..5f60d7f14bb 100644 --- a/avm-transpiler/Earthfile +++ b/avm-transpiler/Earthfile @@ -8,11 +8,16 @@ source: WORKDIR /usr/src/avm-transpiler COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .rustfmt.toml . + DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns" build: FROM +source # build avm transpiler, and make sure the big build and deps folders don't hit cache - RUN ./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps} + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="avm-transpiler" \ + --rebuild_patterns=".rebuild_patterns" \ + --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \ + --build_artifacts="target" SAVE ARTIFACT target/release/avm-transpiler avm-transpiler SAVE ARTIFACT scripts/compile_then_transpile.sh diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index 4959fd6b980..0546cb0d10b 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -5,7 +5,5 @@ cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m" -extract_repo_if_working_copy_clean avm-transpiler \ - /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/ - -remove_old_images avm-transpiler +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh) +cache-download.sh avm-transpiler-$HASH.tar.gz > /dev/null diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index 0066d75a3d9..ea821104428 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -5,13 +5,20 @@ cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving bb binary from remote cache...\033[0m" -extract_repo_if_working_copy_clean barretenberg-x86_64-linux-clang \ - /usr/src/barretenberg/cpp/build/bin ./build \ - /usr/src/barretenberg/cpp/build-pic/lib ./build-pic +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh) +TMP=$(mktemp -d) -echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m" -extract_repo_if_working_copy_clean barretenberg-wasm-linux-clang \ - /usr/src/barretenberg/cpp/build-wasm/bin ./build-wasm \ - /usr/src/barretenberg/cpp/build-wasm-threads/bin ./build-wasm-threads +function on_exit() { + rm -rf "$TMP" +} +trap on_exit EXIT -remove_old_images barretenberg-wasm-linux-clang +cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm > /dev/null +cache-download.sh barretenberg-preset-wasm-threads-v1-$HASH.tar.gz $TMP/build-wasm-threads > /dev/null +cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release > /dev/null +cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state > /dev/null + +mv -n $TMP/build-wasm/build build-wasm/ +mv -n $TMP/build-wasm-threads/build build-wasm-threads/ +mv -n $TMP/build-release/build build/ +mv -n $TMP/build-world-state/build/bin/* build/bin/ diff --git a/bootstrap.sh b/bootstrap.sh index 2378c85ec1e..b0341af730c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -128,7 +128,7 @@ elif [ "$CMD" = "full" ]; then elif [ "$CMD" = "fast" ]; then export USE_CACHE=1 if ! can_use_ci_cache; then - echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}" + echo -e "${BOLD}${YELLOW}WARNING: AWS credentials are missing. Note this is for internal aztec devs only.${RESET}" exit 1 fi elif [ "$CMD" = "check" ]; then diff --git a/build-system/s3-cache-scripts/cache-download.sh b/build-system/s3-cache-scripts/cache-download.sh index 3fbbf43503e..ec1b6e240bd 100755 --- a/build-system/s3-cache-scripts/cache-download.sh +++ b/build-system/s3-cache-scripts/cache-download.sh @@ -1,13 +1,14 @@ #!/bin/bash set -eu -if [ "$#" -ne 1 ]; then +if [ "$#" -lt 1 ]; then echo "Usage: $0 " exit 1 fi # Get the tar.gz file name from the argument TAR_FILE="$1" +OUT_DIR="${2:-.}" function on_exit() { # Cleanup the temporary tar.gz file @@ -20,6 +21,7 @@ trap on_exit EXIT aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "$TAR_FILE" --quiet # Extract the cache file -tar -xzf "$TAR_FILE" +mkdir -p "$OUT_DIR" +tar -xzf "$TAR_FILE" -C "$OUT_DIR" echo "Cache download and extraction complete." diff --git a/build-system/s3-cache-scripts/cache-upload.sh b/build-system/s3-cache-scripts/cache-upload.sh index d574d246e90..3dd2397db83 100755 --- a/build-system/s3-cache-scripts/cache-upload.sh +++ b/build-system/s3-cache-scripts/cache-upload.sh @@ -24,4 +24,4 @@ trap on_exit EXIT # Rest of args are our binary paths tar -czf "$TAR_FILE" $@ -aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME" \ No newline at end of file +aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME" diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache index 2831dabb2bb..f5bd1a90e0c 100755 --- a/build-system/scripts/can_use_ci_cache +++ b/build-system/scripts/can_use_ci_cache @@ -1,3 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1 \ No newline at end of file +if [ -n $AWS_ACCESS_KEY_ID ] && [ -n $AWS_SECRET_ACCESS_KEY ]; then + exit 0 +elif [ -f ~/.aws/credentials ]; then + echo "export AWS_ACCESS_KEY_ID=\$(aws configure get default.aws_access_key_id)" >> $BASH_ENV + echo "export AWS_SECRET_ACCESS_KEY=\$(aws configure get default.aws_secret_access_key)" >> $BASH_ENV + exit 0 +else + exit 1 +fi diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index b98486cef0d..adb78741e57 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -25,7 +25,7 @@ ROOT_PATH=$(git rev-parse --show-toplevel) BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat $ROOT_PATH/PROJECT) COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH) -PATH=$PATH:$BUILD_SYSTEM_PATH/scripts +PATH=$PATH:$BUILD_SYSTEM_PATH/scripts:$BUILD_SYSTEM_PATH/s3-cache-scripts check_env @@ -81,7 +81,7 @@ echo export BUILD_SYSTEM_PATH=$BUILD_SYSTEM_PATH >> $BASH_ENV echo export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} >> $BASH_ENV echo export BUILDKIT_PROGRESS=plain >> $BASH_ENV echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV -echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV +echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts:$BUILD_SYSTEM_PATH/s3-cache-scripts >> $BASH_ENV echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV echo export ECR_REGION=us-east-2 >> $BASH_ENV echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV diff --git a/l1-contracts/.rebuild_patterns b/l1-contracts/.rebuild_patterns new file mode 100644 index 00000000000..09a492e2260 --- /dev/null +++ b/l1-contracts/.rebuild_patterns @@ -0,0 +1 @@ +^l1-contracts/src/.*\.sol$ diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile index 9ad5d1000f6..115371bd12d 100644 --- a/l1-contracts/Earthfile +++ b/l1-contracts/Earthfile @@ -1,16 +1,27 @@ VERSION 0.8 -build: +src: FROM ../build-images+from-registry WORKDIR /usr/src/l1-contracts COPY --dir lib src terraform test *.json *.toml *.sh . - #RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff - # "slither": "forge clean && forge build --build-info --skip '*/test/**' --force && slither . --checklist --ignore-compile --show-ignored-findings --config-file ./slither.config.json | tee slither_output.md", - # "slither-has-diff": "./slither_has_diff.sh" + DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns" + +lint: + FROM +src RUN solhint --config ./.solhint.json --fix "src/**/*.sol" RUN forge clean && forge fmt --check - RUN forge build - SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts + +build: + FROM +lint + WORKDIR /usr/src/l1-contracts + + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="l1-contracts" \ + --rebuild_patterns=".rebuild_patterns" \ + --command="forge build" \ + --build_artifacts="out" + + SAVE ARTIFACT /usr/src/l1-contracts/out /usr/src/l1-contracts/out test: FROM +build diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 2cb1f5ad049..2c0b9ad04ed 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -5,6 +5,5 @@ cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" -extract_repo_if_working_copy_clean l1-contracts /usr/src/l1-contracts/out . - -remove_old_images l1-contracts +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh) +cache-download.sh l1-contracts-$HASH.tar.gz > /dev/null diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index d4f2d006401..7abbeae2bcb 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -5,9 +5,9 @@ cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" -extract_repo_if_working_copy_clean noir-packages /usr/src/noir/packages ./ -echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" -extract_repo_if_working_copy_clean noir /usr/src/noir/noir-repo/target/release ./noir-repo/target/ +NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native compute-content-hash.sh) +cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz > /dev/null -remove_old_images noir-packages -remove_old_images noir +echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" +PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_packages compute-content-hash.sh) +cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz diff --git a/scripts/earthly-local b/scripts/earthly-local index 50bf07187b6..3b7fb2030d2 100755 --- a/scripts/earthly-local +++ b/scripts/earthly-local @@ -10,8 +10,7 @@ function start_minio() { # Already started return fi - mkdir -p ~/.minio/data - docker run -d -p 12000:9000 -p 12001:12001 -v ~/.minio/data:/data \ + docker run -d -p 12000:9000 -p 12001:12001 -v minio-data:/data \ quay.io/minio/minio server /data --console-address ":12001" # make our cache bucket AWS_ACCESS_KEY_ID="minioadmin" AWS_SECRET_ACCESS_KEY="minioadmin" aws --endpoint-url http://localhost:12000 s3 mb s3://aztec-ci-artifacts 2>/dev/null || true diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index 6b6abd466ed..c3278f79c71 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -45,7 +45,7 @@ build: COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node COPY ../noir/+nargo/acvm /usr/src/noir/noir-repo/target/release/acvm COPY --dir ../noir-projects/+build/. /usr/src/noir-projects - COPY ../l1-contracts/+build/usr/src/l1-contracts /usr/src/l1-contracts + COPY ../l1-contracts/+build/usr/src/l1-contracts/out /usr/src/l1-contracts/out WORKDIR /usr/src/yarn-project COPY . . From 5955d7b39c144296d657dcb939810420579c93ee Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 9 Oct 2024 14:17:41 +0000 Subject: [PATCH 02/19] fix: bb.js caching --- barretenberg/ts/Earthfile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/barretenberg/ts/Earthfile b/barretenberg/ts/Earthfile index e4df38e4c5d..798bb685e37 100644 --- a/barretenberg/ts/Earthfile +++ b/barretenberg/ts/Earthfile @@ -6,12 +6,11 @@ CACHE: ARG build_artifacts ARG prefix # TODO(#8929): reinstate bb.js caching - RUN $command - # DO ../../build-system/s3-cache-scripts/+WITH_CACHE \ - # --prefix="bb.js-$prefix" \ - # --command="$command" \ - # --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \ - # --build_artifacts="$build_artifacts" + DO ../../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="bb.js-$prefix" \ + --command="$command" \ + --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \ + --build_artifacts="$build_artifacts" deps: FROM ../../build-images+from-registry @@ -34,7 +33,7 @@ deps: esm: FROM +deps DO +CACHE \ - --prefix="browser" \ + --prefix="esm" \ --command="yarn build:esm" \ --build_artifacts="." SAVE ARTIFACT /usr/src/barretenberg/ts build From 7897420d3025baad9d617150633f12c3620b1894 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 9 Oct 2024 16:28:54 +0000 Subject: [PATCH 03/19] fix: rebuild patterns --- avm-transpiler/Earthfile | 2 +- avm-transpiler/bootstrap_cache.sh | 2 +- barretenberg/ts/bootstrap_cache.sh | 9 ++++++--- noir/bootstrap_cache.sh | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile index 5f60d7f14bb..e02b70282a0 100644 --- a/avm-transpiler/Earthfile +++ b/avm-transpiler/Earthfile @@ -15,7 +15,7 @@ build: # build avm transpiler, and make sure the big build and deps folders don't hit cache DO ../build-system/s3-cache-scripts/+WITH_CACHE \ --prefix="avm-transpiler" \ - --rebuild_patterns=".rebuild_patterns" \ + --rebuild_patterns="../noir/.rebuild_patterns_native .rebuild_patterns" \ --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \ --build_artifacts="target" SAVE ARTIFACT target/release/avm-transpiler avm-transpiler diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index 0546cb0d10b..dc90c97f58c 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -5,5 +5,5 @@ cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m" -HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh) +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" compute-content-hash.sh) cache-download.sh avm-transpiler-$HASH.tar.gz > /dev/null diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index 8cfbfabaaa6..05e88024bb6 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -5,8 +5,11 @@ cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m" -extract_repo_if_working_copy_clean bb.js /usr/src/barretenberg/ts/dest . +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" compute-content-hash.sh) + +cache-download.sh bb.js-esm-$HASH.tar.gz > /dev/null +cache-download.sh bb.js-cjs-$HASH.tar.gz > /dev/null +cache-download.sh bb.js-browser-$HASH.tar.gz > /dev/null + # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. yarn install - -remove_old_images bb.js diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 7abbeae2bcb..dd30c9af542 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -9,5 +9,5 @@ NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native compute-cont cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz > /dev/null echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" -PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_packages compute-content-hash.sh) +PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../barretenberg/cpp/.rebuild_patterns .rebuild_patterns_packages" compute-content-hash.sh) cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz From d9e6864eec7c8dd52ce7f2102463f554fae124c4 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 10:28:45 +0000 Subject: [PATCH 04/19] fix: cache noir projects --- noir-projects/Earthfile | 91 +++++++++++-------- noir-projects/bootstrap_cache.sh | 15 +-- .../mock-protocol-circuits/.rebuild_patterns | 3 + .../noir-contracts/.rebuild_patterns | 4 + .../noir-protocol-circuits/.rebuild_patterns | 2 + 5 files changed, 69 insertions(+), 46 deletions(-) create mode 100644 noir-projects/mock-protocol-circuits/.rebuild_patterns create mode 100644 noir-projects/noir-contracts/.rebuild_patterns create mode 100644 noir-projects/noir-protocol-circuits/.rebuild_patterns diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 4345a432089..392a4758eaf 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -1,70 +1,81 @@ VERSION 0.8 -source: - FROM ../build-images+from-registry +deps: + LOCALLY + LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin) - # Install nargo + FROM ../build-images+from-registry COPY ../noir/+nargo/nargo /usr/bin/nargo - # Install world state napi - COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node + COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \ + bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials' + ENV BB_HASH=$bb_source_hash + ENV NARGO=nargo +source: + FROM +deps WORKDIR /usr/src/noir-projects - # Copy source. - COPY package.json yarn.lock mega_honk_circuits.json . + COPY package.json yarn.lock . + RUN yarn + + COPY mega_honk_circuits.json . COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts . build-contracts: ARG RAYON_NUM_THREADS - LOCALLY - LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin) - FROM +source - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \ - bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials' - # Install bb - COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb + FROM +source + ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS # Install transpiler COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler + ENV TRANSPILER=avm-transpiler - RUN yarn - ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS - RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler ./bootstrap.sh - SAVE ARTIFACT noir-contracts + WORKDIR /usr/src/noir-projects/noir-contracts + + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="noir-projects-noir-contracts" \ + --rebuild_patterns="../../noir/.rebuild_patterns_native ../../avm-transpiler/.rebuild_patterns ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ + --command="./bootstrap.sh" \ + --build_artifacts="target" + + SAVE ARTIFACT target build-protocol-circuits: ARG RAYON_NUM_THREADS LOCALLY - LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin) FROM +source - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \ - bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials' - - # Install bb - COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb - RUN yarn + WORKDIR /usr/src/noir-projects/noir-protocol-circuits ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS - RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh - SAVE ARTIFACT noir-protocol-circuits + ENV PARALLEL_VK=false + + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="noir-projects-noir-protocol-circuits" \ + --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ + --command="yarn && ./bootstrap.sh" \ + --build_artifacts="target" + + SAVE ARTIFACT target build-mock-protocol-circuits: ARG RAYON_NUM_THREADS LOCALLY - LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin) FROM +source - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \ - bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials' - # Install bb - COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb - RUN yarn + WORKDIR /usr/src/noir-projects/mock-protocol-circuits ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS - RUN echo "building with num threads $RAYON_NUM_THREADS" - RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh - SAVE ARTIFACT mock-protocol-circuits + ENV PARALLEL_VK=false + + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ + --prefix="noir-projects-mock-protocol-circuits" \ + --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ + --command="./bootstrap.sh" \ + --build_artifacts="target" + + SAVE ARTIFACT target build: FROM +source @@ -72,9 +83,9 @@ build: BUILD +build-protocol-circuits BUILD +build-mock-protocol-circuits - COPY +build-contracts/noir-contracts ./noir-contracts - COPY +build-protocol-circuits/noir-protocol-circuits ./noir-protocol-circuits - COPY +build-mock-protocol-circuits/mock-protocol-circuits ./mock-protocol-circuits + COPY --dir +build-contracts/target ./noir-contracts + COPY --dir +build-protocol-circuits/target ./noir-protocol-circuits + COPY --dir +build-mock-protocol-circuits/target ./mock-protocol-circuits SAVE ARTIFACT aztec-nr SAVE ARTIFACT noir-contracts @@ -109,7 +120,7 @@ format: FROM +source WORKDIR /usr/src/noir-projects/noir-protocol-circuits - RUN yarn && node ./scripts/generate_variants.js + RUN node ./scripts/generate_variants.js RUN nargo fmt --check WORKDIR /usr/src/noir-projects/mock-protocol-circuits diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 97cf5134182..02c3125baf7 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -5,13 +5,16 @@ cd "$(dirname "$0")" source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" -extract_repo_if_working_copy_clean noir-projects \ - /usr/src/noir-projects/noir-contracts/target ./noir-contracts \ - /usr/src/noir-projects/noir-protocol-circuits/target ./noir-protocol-circuits -remove_old_images noir-projects +PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" compute-content-hash.sh) +cache-download.sh noir-projects-noir-protocol-circuits-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null + +MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" compute-content-hash.sh) +cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null + +CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" compute-content-hash.sh) +cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts > /dev/null yarn -./mock-protocol-circuits/bootstrap.sh -(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js) \ No newline at end of file +(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js) diff --git a/noir-projects/mock-protocol-circuits/.rebuild_patterns b/noir-projects/mock-protocol-circuits/.rebuild_patterns new file mode 100644 index 00000000000..caf8875c067 --- /dev/null +++ b/noir-projects/mock-protocol-circuits/.rebuild_patterns @@ -0,0 +1,3 @@ +^noir-projects/mock-protocol-circuits/.*\.(nr|toml)$ +^noir-projects/mock-protocol-circuits/bootstrap.sh$ +^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$ diff --git a/noir-projects/noir-contracts/.rebuild_patterns b/noir-projects/noir-contracts/.rebuild_patterns new file mode 100644 index 00000000000..dca838646cb --- /dev/null +++ b/noir-projects/noir-contracts/.rebuild_patterns @@ -0,0 +1,4 @@ +^noir-projects/noir-contracts/.*\.(nr|toml)$ +^noir-projects/noir-contracts/bootstrap.sh$ +^noir-projects/aztec-nr/.*\.(nr|toml)$ +^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$ diff --git a/noir-projects/noir-protocol-circuits/.rebuild_patterns b/noir-projects/noir-protocol-circuits/.rebuild_patterns new file mode 100644 index 00000000000..24b422f31d9 --- /dev/null +++ b/noir-projects/noir-protocol-circuits/.rebuild_patterns @@ -0,0 +1,2 @@ +^noir-projects/noir-protocol-circuits/.*\.(nr|toml)$ +^noir-projects/noir-protocol-circuits/bootstrap.sh$ From 5f9391c2fa96bcc5c0bdf7c006a9e71de9ebf946 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 10:40:35 +0000 Subject: [PATCH 05/19] refactor: remove reliance on setup_env --- avm-transpiler/bootstrap_cache.sh | 5 ++--- barretenberg/cpp/Earthfile | 2 +- barretenberg/cpp/bootstrap_cache.sh | 13 +++++++------ build-system/scripts/setup_env | 4 ++-- l1-contracts/bootstrap_cache.sh | 5 ++--- noir-projects/bootstrap_cache.sh | 15 ++++++++------- noir/bootstrap_cache.sh | 9 ++++----- 7 files changed, 26 insertions(+), 27 deletions(-) diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index dc90c97f58c..5f93779832f 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -2,8 +2,7 @@ set -eu cd "$(dirname "$0")" -source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m" -HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" compute-content-hash.sh) -cache-download.sh avm-transpiler-$HASH.tar.gz > /dev/null +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" ../build-system/s3-cache-scripts/compute-content-hash.sh) +../build-system/s3-cache-scripts/cache-download.sh avm-transpiler-$HASH.tar.gz > /dev/null diff --git a/barretenberg/cpp/Earthfile b/barretenberg/cpp/Earthfile index 3c2c7ad5b51..72b3fb7f218 100644 --- a/barretenberg/cpp/Earthfile +++ b/barretenberg/cpp/Earthfile @@ -131,7 +131,7 @@ preset-sol: preset-wasm-threads: FROM +source - DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads-v1 \ + DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads \ --command="cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm" SAVE ARTIFACT build/bin diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index ea821104428..6dedb530e2f 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -2,10 +2,11 @@ set -eu cd "$(dirname "$0")" -source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving bb binary from remote cache...\033[0m" -HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh) + +SCRIPTS_PATH=../../build-system/s3-cache-scripts/ +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns $SCRIPTS_PATH/compute-content-hash.sh) TMP=$(mktemp -d) function on_exit() { @@ -13,10 +14,10 @@ function on_exit() { } trap on_exit EXIT -cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm > /dev/null -cache-download.sh barretenberg-preset-wasm-threads-v1-$HASH.tar.gz $TMP/build-wasm-threads > /dev/null -cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release > /dev/null -cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state > /dev/null +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm > /dev/null +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-threads-$HASH.tar.gz $TMP/build-wasm-threads > /dev/null +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release > /dev/null +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state > /dev/null mv -n $TMP/build-wasm/build build-wasm/ mv -n $TMP/build-wasm-threads/build build-wasm-threads/ diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index adb78741e57..b98486cef0d 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -25,7 +25,7 @@ ROOT_PATH=$(git rev-parse --show-toplevel) BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) PROJECT=$(cat $ROOT_PATH/PROJECT) COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH) -PATH=$PATH:$BUILD_SYSTEM_PATH/scripts:$BUILD_SYSTEM_PATH/s3-cache-scripts +PATH=$PATH:$BUILD_SYSTEM_PATH/scripts check_env @@ -81,7 +81,7 @@ echo export BUILD_SYSTEM_PATH=$BUILD_SYSTEM_PATH >> $BASH_ENV echo export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} >> $BASH_ENV echo export BUILDKIT_PROGRESS=plain >> $BASH_ENV echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV -echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts:$BUILD_SYSTEM_PATH/s3-cache-scripts >> $BASH_ENV +echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV echo export ECR_REGION=us-east-2 >> $BASH_ENV echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 2c0b9ad04ed..73580b21b5f 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -2,8 +2,7 @@ set -eu cd "$(dirname "$0")" -source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" -HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh) -cache-download.sh l1-contracts-$HASH.tar.gz > /dev/null +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns ../build-system/s3-cache-scripts/compute-content-hash.sh) +../build-system/s3-cache-scripts/cache-download.sh l1-contracts-$HASH.tar.gz > /dev/null diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 02c3125baf7..9d0296dccff 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -2,18 +2,19 @@ set -eu cd "$(dirname "$0")" -source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +SCRIPTS_PATH=../build-system/s3-cache-scripts/ echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" -PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" compute-content-hash.sh) -cache-download.sh noir-projects-noir-protocol-circuits-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null +PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) +$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null -MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" compute-content-hash.sh) -cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null +MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) +$SCRIPTS_PATH/cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null -CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" compute-content-hash.sh) -cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts > /dev/null +CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) +$SCRIPTS_PATH/cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts > /dev/null yarn diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index dd30c9af542..22bab48283f 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -2,12 +2,11 @@ set -eu cd "$(dirname "$0")" -source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" -NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native compute-content-hash.sh) -cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz > /dev/null +NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native ../../build-system/s3-cache-scripts/compute-content-hash.sh) +../../build-system/s3-cache-scripts/cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz > /dev/null echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" -PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../barretenberg/cpp/.rebuild_patterns .rebuild_patterns_packages" compute-content-hash.sh) -cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz +PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../barretenberg/cpp/.rebuild_patterns .rebuild_patterns_packages" ../../build-system/s3-cache-scripts/compute-content-hash.sh) +../../build-system/s3-cache-scripts/cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz From 36b7dbcd62084371fc57c5624da1465ff1a945f4 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 10:41:25 +0000 Subject: [PATCH 06/19] fix: remove aws creds from BASH_ENV --- build-system/scripts/can_use_ci_cache | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache index f5bd1a90e0c..af235562170 100755 --- a/build-system/scripts/can_use_ci_cache +++ b/build-system/scripts/can_use_ci_cache @@ -3,8 +3,6 @@ set -euo pipefail if [ -n $AWS_ACCESS_KEY_ID ] && [ -n $AWS_SECRET_ACCESS_KEY ]; then exit 0 elif [ -f ~/.aws/credentials ]; then - echo "export AWS_ACCESS_KEY_ID=\$(aws configure get default.aws_access_key_id)" >> $BASH_ENV - echo "export AWS_SECRET_ACCESS_KEY=\$(aws configure get default.aws_secret_access_key)" >> $BASH_ENV exit 0 else exit 1 From 7cb9e9cdf7945f6f91576b374fae1e8bb6ab38cb Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 11:53:29 +0000 Subject: [PATCH 07/19] fix: variants --- avm-transpiler/Earthfile | 1 - noir-projects/mock-protocol-circuits/.rebuild_patterns | 1 + noir-projects/noir-contracts/.rebuild_patterns | 1 + noir-projects/noir-protocol-circuits/.rebuild_patterns | 2 ++ .../noir-protocol-circuits/scripts/generate_variants.js | 4 +++- noir-projects/scripts/generate_vk_json.js | 3 +++ scripts/earthly-local | 5 +++++ .../src/scripts/generate_private_kernel_reset_data.ts | 5 ++++- .../src/scripts/generate_ts_from_abi.ts | 5 ++++- 9 files changed, 23 insertions(+), 4 deletions(-) diff --git a/avm-transpiler/Earthfile b/avm-transpiler/Earthfile index e02b70282a0..e4db70ea091 100644 --- a/avm-transpiler/Earthfile +++ b/avm-transpiler/Earthfile @@ -8,7 +8,6 @@ source: WORKDIR /usr/src/avm-transpiler COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .rustfmt.toml . - DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns" build: FROM +source diff --git a/noir-projects/mock-protocol-circuits/.rebuild_patterns b/noir-projects/mock-protocol-circuits/.rebuild_patterns index caf8875c067..332509323a3 100644 --- a/noir-projects/mock-protocol-circuits/.rebuild_patterns +++ b/noir-projects/mock-protocol-circuits/.rebuild_patterns @@ -1,3 +1,4 @@ ^noir-projects/mock-protocol-circuits/.*\.(nr|toml)$ ^noir-projects/mock-protocol-circuits/bootstrap.sh$ ^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$ +^noir-projects/scripts/generate_vk_json.js$ diff --git a/noir-projects/noir-contracts/.rebuild_patterns b/noir-projects/noir-contracts/.rebuild_patterns index dca838646cb..b48553b97c8 100644 --- a/noir-projects/noir-contracts/.rebuild_patterns +++ b/noir-projects/noir-contracts/.rebuild_patterns @@ -2,3 +2,4 @@ ^noir-projects/noir-contracts/bootstrap.sh$ ^noir-projects/aztec-nr/.*\.(nr|toml)$ ^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$ +^noir-projects/scripts/generate_vk_json.js$ diff --git a/noir-projects/noir-protocol-circuits/.rebuild_patterns b/noir-projects/noir-protocol-circuits/.rebuild_patterns index 24b422f31d9..cb909eba88b 100644 --- a/noir-projects/noir-protocol-circuits/.rebuild_patterns +++ b/noir-projects/noir-protocol-circuits/.rebuild_patterns @@ -1,2 +1,4 @@ ^noir-projects/noir-protocol-circuits/.*\.(nr|toml)$ ^noir-projects/noir-protocol-circuits/bootstrap.sh$ +^noir-projects/noir-protocol-circuits/scripts/generate_variants.js$ +^noir-projects/scripts/generate_vk_json.js$ diff --git a/noir-projects/noir-protocol-circuits/scripts/generate_variants.js b/noir-projects/noir-protocol-circuits/scripts/generate_variants.js index 94b27106cf0..b57d0812b61 100644 --- a/noir-projects/noir-protocol-circuits/scripts/generate_variants.js +++ b/noir-projects/noir-protocol-circuits/scripts/generate_variants.js @@ -69,8 +69,10 @@ function generateDimensions(dimensionsList) { const fileContent = ["[", dimensionValuesStrings.join(",\n"), "]"].join("\n"); + const outputFilePath = path.join(root, "target", "dimensions", "private_kernel_reset_dimensions.json"); + fs.mkdirSync(path.dirname(outputFilePath), { recursive: true }); fs.writeFileSync( - path.join(root, "private_kernel_reset_dimensions.json"), + outputFilePath, fileContent ); } diff --git a/noir-projects/scripts/generate_vk_json.js b/noir-projects/scripts/generate_vk_json.js index 6f4c6a3da88..ca3bc6b3008 100644 --- a/noir-projects/scripts/generate_vk_json.js +++ b/noir-projects/scripts/generate_vk_json.js @@ -56,6 +56,9 @@ async function removeFunctionArtifact(artifactPath) { async function getBytecodeHash(artifactPath) { const { bytecode } = JSON.parse(await fs.readFile(artifactPath)); + if (!bytecode) { + throw new Error("No bytecode found in artifact: " + artifactPath); + } return crypto.createHash("md5").update(bytecode).digest("hex"); } diff --git a/scripts/earthly-local b/scripts/earthly-local index 3b7fb2030d2..bb9991fb8e4 100755 --- a/scripts/earthly-local +++ b/scripts/earthly-local @@ -29,6 +29,11 @@ if ! git diff-index --quiet HEAD --; then S3_BUILD_CACHE_MINIO_URL="" elif [ ! -z "${AWS_ACCESS_KEY_ID:-}" ] ; then S3_BUILD_CACHE_DOWNLOAD=true +elif [ -f ~/.aws/credentials ]; then + # make credentials avaialble to earthly + AWS_ACCESS_KEY_ID=$(aws configure get default.aws_access_key_id) + AWS_SECRET_ACCESS_KEY=$(aws configure get default.aws_secret_access_key) + S3_BUILD_CACHE_DOWNLOAD=true else S3_BUILD_CACHE_UPLOAD=false S3_BUILD_CACHE_DOWNLOAD=false diff --git a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts index d6d5b118b9c..b5b33034373 100644 --- a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts +++ b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts @@ -153,7 +153,10 @@ const main = async () => { checkDimensionNames(config); const dimensionsList = JSON.parse( - await fs.readFile('../../noir-projects/noir-protocol-circuits/private_kernel_reset_dimensions.json', 'utf8'), + await fs.readFile( + '../../noir-projects/noir-protocol-circuits/target/dimensions/private_kernel_reset_dimensions.json', + 'utf8', + ), ) as number[][]; checkMaxDimensions(dimensionsList); diff --git a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts index 7cf03712bd1..cb8251c1a27 100644 --- a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts +++ b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts @@ -30,7 +30,10 @@ const circuits = [ const main = async () => { const dimensionsLists = JSON.parse( - await fs.readFile('../../noir-projects/noir-protocol-circuits/private_kernel_reset_dimensions.json', 'utf8'), + await fs.readFile( + '../../noir-projects/noir-protocol-circuits/target/dimensions/private_kernel_reset_dimensions.json', + 'utf8', + ), ) as number[][]; // Need any variant in the set so that the type will be rendered with generics. circuits.push(`private_kernel_reset_${dimensionsLists[0].join('_')}`); From a774265adb52ff55e39a790cc3ffc668a47a17c0 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 10 Oct 2024 15:27:54 +0100 Subject: [PATCH 08/19] Update cache-download.sh --- build-system/s3-cache-scripts/cache-download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/s3-cache-scripts/cache-download.sh b/build-system/s3-cache-scripts/cache-download.sh index ec1b6e240bd..1277ec15b8e 100755 --- a/build-system/s3-cache-scripts/cache-download.sh +++ b/build-system/s3-cache-scripts/cache-download.sh @@ -24,4 +24,4 @@ aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$T mkdir -p "$OUT_DIR" tar -xzf "$TAR_FILE" -C "$OUT_DIR" -echo "Cache download and extraction complete." +echo "Cache download and extraction of $TAR_FILE complete." From d002b8f6bc7df5a86521ebb41376a8f48c4ded18 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 14:44:42 +0000 Subject: [PATCH 09/19] fix: save whole l1-contracts folder as artifact --- l1-contracts/Earthfile | 2 +- yarn-project/Earthfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/l1-contracts/Earthfile b/l1-contracts/Earthfile index 115371bd12d..011d4b98215 100644 --- a/l1-contracts/Earthfile +++ b/l1-contracts/Earthfile @@ -21,7 +21,7 @@ build: --command="forge build" \ --build_artifacts="out" - SAVE ARTIFACT /usr/src/l1-contracts/out /usr/src/l1-contracts/out + SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts test: FROM +build diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile index c3278f79c71..6b6abd466ed 100644 --- a/yarn-project/Earthfile +++ b/yarn-project/Earthfile @@ -45,7 +45,7 @@ build: COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node COPY ../noir/+nargo/acvm /usr/src/noir/noir-repo/target/release/acvm COPY --dir ../noir-projects/+build/. /usr/src/noir-projects - COPY ../l1-contracts/+build/usr/src/l1-contracts/out /usr/src/l1-contracts/out + COPY ../l1-contracts/+build/usr/src/l1-contracts /usr/src/l1-contracts WORKDIR /usr/src/yarn-project COPY . . From 041461fb922598f579138fd4bf6ea14b544f59b0 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 15:37:06 +0000 Subject: [PATCH 10/19] fix: more fixes --- noir-projects/Earthfile | 21 ++++++++++--------- .../noir-contracts/.rebuild_patterns | 3 +-- .../noir-protocol-circuits/.rebuild_patterns | 2 +- .../generate_private_kernel_reset_data.ts | 5 +---- .../src/scripts/generate_ts_from_abi.ts | 5 +---- 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 392a4758eaf..0a9262ee263 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -33,49 +33,50 @@ build-contracts: ENV TRANSPILER=avm-transpiler WORKDIR /usr/src/noir-projects/noir-contracts - DO ../build-system/s3-cache-scripts/+WITH_CACHE \ --prefix="noir-projects-noir-contracts" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../avm-transpiler/.rebuild_patterns ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="./bootstrap.sh" \ --build_artifacts="target" - SAVE ARTIFACT target + WORKDIR /usr/src/noir-projects + SAVE ARTIFACT noir-contracts build-protocol-circuits: ARG RAYON_NUM_THREADS LOCALLY FROM +source - WORKDIR /usr/src/noir-projects/noir-protocol-circuits ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS ENV PARALLEL_VK=false + WORKDIR /usr/src/noir-projects/noir-protocol-circuits DO ../build-system/s3-cache-scripts/+WITH_CACHE \ --prefix="noir-projects-noir-protocol-circuits" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="yarn && ./bootstrap.sh" \ --build_artifacts="target" - SAVE ARTIFACT target + WORKDIR /usr/src/noir-projects + SAVE ARTIFACT noir-protocol-circuits build-mock-protocol-circuits: ARG RAYON_NUM_THREADS LOCALLY FROM +source - WORKDIR /usr/src/noir-projects/mock-protocol-circuits - ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS ENV PARALLEL_VK=false + WORKDIR /usr/src/noir-projects/mock-protocol-circuits DO ../build-system/s3-cache-scripts/+WITH_CACHE \ --prefix="noir-projects-mock-protocol-circuits" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="./bootstrap.sh" \ --build_artifacts="target" - SAVE ARTIFACT target + WORKDIR /usr/src/noir-projects + SAVE ARTIFACT mock-protocol-circuits build: FROM +source @@ -83,9 +84,9 @@ build: BUILD +build-protocol-circuits BUILD +build-mock-protocol-circuits - COPY --dir +build-contracts/target ./noir-contracts - COPY --dir +build-protocol-circuits/target ./noir-protocol-circuits - COPY --dir +build-mock-protocol-circuits/target ./mock-protocol-circuits + COPY +build-contracts/noir-contracts ./noir-contracts + COPY +build-protocol-circuits/noir-protocol-circuits ./noir-protocol-circuits + COPY +build-mock-protocol-circuits/mock-protocol-circuits ./mock-protocol-circuits SAVE ARTIFACT aztec-nr SAVE ARTIFACT noir-contracts diff --git a/noir-projects/noir-contracts/.rebuild_patterns b/noir-projects/noir-contracts/.rebuild_patterns index b48553b97c8..7feb1d9bfbf 100644 --- a/noir-projects/noir-contracts/.rebuild_patterns +++ b/noir-projects/noir-contracts/.rebuild_patterns @@ -1,5 +1,4 @@ -^noir-projects/noir-contracts/.*\.(nr|toml)$ -^noir-projects/noir-contracts/bootstrap.sh$ +^noir-projects/noir-contracts/.*\.(nr|toml|sh|json|js)$ ^noir-projects/aztec-nr/.*\.(nr|toml)$ ^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$ ^noir-projects/scripts/generate_vk_json.js$ diff --git a/noir-projects/noir-protocol-circuits/.rebuild_patterns b/noir-projects/noir-protocol-circuits/.rebuild_patterns index cb909eba88b..110634d45d7 100644 --- a/noir-projects/noir-protocol-circuits/.rebuild_patterns +++ b/noir-projects/noir-protocol-circuits/.rebuild_patterns @@ -1,4 +1,4 @@ -^noir-projects/noir-protocol-circuits/.*\.(nr|toml)$ +^noir-projects/noir-protocol-circuits/.*\.(nr|toml|json)$ ^noir-projects/noir-protocol-circuits/bootstrap.sh$ ^noir-projects/noir-protocol-circuits/scripts/generate_variants.js$ ^noir-projects/scripts/generate_vk_json.js$ diff --git a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts index b5b33034373..d6d5b118b9c 100644 --- a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts +++ b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_private_kernel_reset_data.ts @@ -153,10 +153,7 @@ const main = async () => { checkDimensionNames(config); const dimensionsList = JSON.parse( - await fs.readFile( - '../../noir-projects/noir-protocol-circuits/target/dimensions/private_kernel_reset_dimensions.json', - 'utf8', - ), + await fs.readFile('../../noir-projects/noir-protocol-circuits/private_kernel_reset_dimensions.json', 'utf8'), ) as number[][]; checkMaxDimensions(dimensionsList); diff --git a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts index cb8251c1a27..7cf03712bd1 100644 --- a/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts +++ b/yarn-project/noir-protocol-circuits-types/src/scripts/generate_ts_from_abi.ts @@ -30,10 +30,7 @@ const circuits = [ const main = async () => { const dimensionsLists = JSON.parse( - await fs.readFile( - '../../noir-projects/noir-protocol-circuits/target/dimensions/private_kernel_reset_dimensions.json', - 'utf8', - ), + await fs.readFile('../../noir-projects/noir-protocol-circuits/private_kernel_reset_dimensions.json', 'utf8'), ) as number[][]; // Need any variant in the set so that the type will be rendered with generics. circuits.push(`private_kernel_reset_${dimensionsLists[0].join('_')}`); From da8e1fd857b58df40ef7550cc1ca57361260f21a Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 15:55:25 +0000 Subject: [PATCH 11/19] fix: cache scripts --- barretenberg/ts/bootstrap_cache.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index 05e88024bb6..d7782fa1128 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -4,12 +4,14 @@ set -eu cd "$(dirname "$0")" source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null +CACHE_SCRIPTS=../../build-system/s3-cache-scripts + echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m" -HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" compute-content-hash.sh) +HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" $CACHE_SCRIPTS/compute-content-hash.sh) -cache-download.sh bb.js-esm-$HASH.tar.gz > /dev/null -cache-download.sh bb.js-cjs-$HASH.tar.gz > /dev/null -cache-download.sh bb.js-browser-$HASH.tar.gz > /dev/null +$CACHE_SCRIPTS/cache-download.sh bb.js-esm-$HASH.tar.gz > /dev/null +$CACHE_SCRIPTS/cache-download.sh bb.js-cjs-$HASH.tar.gz > /dev/null +$CACHE_SCRIPTS/cache-download.sh bb.js-browser-$HASH.tar.gz > /dev/null # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. yarn install From 5aa28826aeaab68fdf3833653237a243f8dbfee6 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 16:21:12 +0000 Subject: [PATCH 12/19] chore: revert dimensions --- .../noir-protocol-circuits/scripts/generate_variants.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/noir-projects/noir-protocol-circuits/scripts/generate_variants.js b/noir-projects/noir-protocol-circuits/scripts/generate_variants.js index b57d0812b61..94b27106cf0 100644 --- a/noir-projects/noir-protocol-circuits/scripts/generate_variants.js +++ b/noir-projects/noir-protocol-circuits/scripts/generate_variants.js @@ -69,10 +69,8 @@ function generateDimensions(dimensionsList) { const fileContent = ["[", dimensionValuesStrings.join(",\n"), "]"].join("\n"); - const outputFilePath = path.join(root, "target", "dimensions", "private_kernel_reset_dimensions.json"); - fs.mkdirSync(path.dirname(outputFilePath), { recursive: true }); fs.writeFileSync( - outputFilePath, + path.join(root, "private_kernel_reset_dimensions.json"), fileContent ); } From 980d6f322efeeafdc895e5f97c10bc562b60a228 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 16:47:53 +0000 Subject: [PATCH 13/19] fix: noir proto circuits --- noir-projects/Earthfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 0a9262ee263..34a29f6852c 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -51,10 +51,14 @@ build-protocol-circuits: ENV PARALLEL_VK=false WORKDIR /usr/src/noir-projects/noir-protocol-circuits + + # NOTE: this step generate a Nargo.toml from the template file + RUN yarn && node ./scripts/generate_variants.js + DO ../build-system/s3-cache-scripts/+WITH_CACHE \ --prefix="noir-projects-noir-protocol-circuits" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ - --command="yarn && ./bootstrap.sh" \ + --command="./bootstrap.sh" \ --build_artifacts="target" WORKDIR /usr/src/noir-projects @@ -101,6 +105,7 @@ test: COPY +build/. /usr/src/noir-projects + RUN exit 1 RUN cd /usr/src/noir-projects/noir-protocol-circuits && nargo test --silence-warnings RUN cd /usr/src/yarn-project/txe && yarn start & \ From cc5d511e1eacc9ed7a0e789f0f2c3c853138bf22 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 17:10:05 +0000 Subject: [PATCH 14/19] fix: don't exit --- noir-projects/Earthfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 34a29f6852c..e89321906cb 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -52,14 +52,11 @@ build-protocol-circuits: WORKDIR /usr/src/noir-projects/noir-protocol-circuits - # NOTE: this step generate a Nargo.toml from the template file - RUN yarn && node ./scripts/generate_variants.js - DO ../build-system/s3-cache-scripts/+WITH_CACHE \ --prefix="noir-projects-noir-protocol-circuits" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="./bootstrap.sh" \ - --build_artifacts="target" + --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json" WORKDIR /usr/src/noir-projects SAVE ARTIFACT noir-protocol-circuits @@ -105,7 +102,6 @@ test: COPY +build/. /usr/src/noir-projects - RUN exit 1 RUN cd /usr/src/noir-projects/noir-protocol-circuits && nargo test --silence-warnings RUN cd /usr/src/yarn-project/txe && yarn start & \ From 0c72b5cac66c9472f27f02d684a989c0fd45c1a2 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 17:15:13 +0000 Subject: [PATCH 15/19] chore: bump the cache --- noir-projects/Earthfile | 2 +- noir-projects/bootstrap_cache.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index e89321906cb..ca5d21ad289 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -53,7 +53,7 @@ build-protocol-circuits: WORKDIR /usr/src/noir-projects/noir-protocol-circuits DO ../build-system/s3-cache-scripts/+WITH_CACHE \ - --prefix="noir-projects-noir-protocol-circuits" \ + --prefix="noir-projects-noir-protocol-circuits-1" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="./bootstrap.sh" \ --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json" diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 9d0296dccff..1e1cf548831 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -8,7 +8,7 @@ SCRIPTS_PATH=../build-system/s3-cache-scripts/ echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null +$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-1-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) $SCRIPTS_PATH/cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null From 516cb91d57861a7166b18dd084404b0991d36777 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 17:32:05 +0000 Subject: [PATCH 16/19] fix: cache autogenerated --- noir-projects/Earthfile | 4 ++-- noir-projects/bootstrap_cache.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index ca5d21ad289..7883cb970de 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -53,10 +53,10 @@ build-protocol-circuits: WORKDIR /usr/src/noir-projects/noir-protocol-circuits DO ../build-system/s3-cache-scripts/+WITH_CACHE \ - --prefix="noir-projects-noir-protocol-circuits-1" \ + --prefix="noir-projects-noir-protocol-circuits-2" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="./bootstrap.sh" \ - --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json" + --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json crates/autogenerated" WORKDIR /usr/src/noir-projects SAVE ARTIFACT noir-protocol-circuits diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 1e1cf548831..5d630d4aa98 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -8,7 +8,7 @@ SCRIPTS_PATH=../build-system/s3-cache-scripts/ echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-1-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null +$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-2-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) $SCRIPTS_PATH/cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null From 3f297e7cf12fa5d761d740c913f9ae19b62be3b0 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 10 Oct 2024 18:09:38 +0000 Subject: [PATCH 17/19] fix: bootstrap_cache scripts --- avm-transpiler/bootstrap_cache.sh | 2 +- barretenberg/cpp/bootstrap_cache.sh | 17 +++++++++-------- barretenberg/ts/bootstrap_cache.sh | 17 ++++++++++++++--- build-system/scripts/can_use_ci_cache | 2 +- l1-contracts/bootstrap_cache.sh | 2 +- noir-projects/bootstrap_cache.sh | 6 +++--- noir/bootstrap_cache.sh | 8 ++++---- 7 files changed, 33 insertions(+), 21 deletions(-) diff --git a/avm-transpiler/bootstrap_cache.sh b/avm-transpiler/bootstrap_cache.sh index 5f93779832f..fdedcb627fd 100755 --- a/avm-transpiler/bootstrap_cache.sh +++ b/avm-transpiler/bootstrap_cache.sh @@ -5,4 +5,4 @@ cd "$(dirname "$0")" echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m" HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" ../build-system/s3-cache-scripts/compute-content-hash.sh) -../build-system/s3-cache-scripts/cache-download.sh avm-transpiler-$HASH.tar.gz > /dev/null +../build-system/s3-cache-scripts/cache-download.sh avm-transpiler-$HASH.tar.gz diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index 6dedb530e2f..4b674f79026 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -14,12 +14,13 @@ function on_exit() { } trap on_exit EXIT -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm > /dev/null -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-threads-$HASH.tar.gz $TMP/build-wasm-threads > /dev/null -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release > /dev/null -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state > /dev/null +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-threads-$HASH.tar.gz $TMP/build-wasm-threads +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release +$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state -mv -n $TMP/build-wasm/build build-wasm/ -mv -n $TMP/build-wasm-threads/build build-wasm-threads/ -mv -n $TMP/build-release/build build/ -mv -n $TMP/build-world-state/build/bin/* build/bin/ +# clobber the existing build with the cached build +mv -f $TMP/build-wasm/build build-wasm/ +mv -f $TMP/build-wasm-threads/build build-wasm-threads/ +mv -f $TMP/build-release/build build/ +mv -f $TMP/build-world-state/build/bin/* build/bin/ diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index d7782fa1128..2ad816066db 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -7,11 +7,22 @@ source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null CACHE_SCRIPTS=../../build-system/s3-cache-scripts echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m" +TMP=$(mktemp -d) + +function on_exit() { + rm -rf "$TMP" +} +trap on_exit EXIT + HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" $CACHE_SCRIPTS/compute-content-hash.sh) -$CACHE_SCRIPTS/cache-download.sh bb.js-esm-$HASH.tar.gz > /dev/null -$CACHE_SCRIPTS/cache-download.sh bb.js-cjs-$HASH.tar.gz > /dev/null -$CACHE_SCRIPTS/cache-download.sh bb.js-browser-$HASH.tar.gz > /dev/null +$CACHE_SCRIPTS/cache-download.sh bb.js-esm-$HASH.tar.gz $TMP/esm +$CACHE_SCRIPTS/cache-download.sh bb.js-cjs-$HASH.tar.gz $TMP/cjs +$CACHE_SCRIPTS/cache-download.sh bb.js-browser-$HASH.tar.gz $TMP/browser + +cp -r $TMP/esm/dest dest +cp -r $TMP/cjs/dest dest +cp -r $TMP/browser/dest dest # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. yarn install diff --git a/build-system/scripts/can_use_ci_cache b/build-system/scripts/can_use_ci_cache index af235562170..b3dd6bdd240 100755 --- a/build-system/scripts/can_use_ci_cache +++ b/build-system/scripts/can_use_ci_cache @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -if [ -n $AWS_ACCESS_KEY_ID ] && [ -n $AWS_SECRET_ACCESS_KEY ]; then +if [ ! -z ${AWS_ACCESS_KEY_ID:-} ] && [ ! -z ${AWS_SECRET_ACCESS_KEY:-} ]; then exit 0 elif [ -f ~/.aws/credentials ]; then exit 0 diff --git a/l1-contracts/bootstrap_cache.sh b/l1-contracts/bootstrap_cache.sh index 73580b21b5f..6f509bc0574 100755 --- a/l1-contracts/bootstrap_cache.sh +++ b/l1-contracts/bootstrap_cache.sh @@ -5,4 +5,4 @@ cd "$(dirname "$0")" echo -e "\033[1mRetrieving contracts from remote cache...\033[0m" HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns ../build-system/s3-cache-scripts/compute-content-hash.sh) -../build-system/s3-cache-scripts/cache-download.sh l1-contracts-$HASH.tar.gz > /dev/null +../build-system/s3-cache-scripts/cache-download.sh l1-contracts-$HASH.tar.gz diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 5d630d4aa98..b6f14ec0aa7 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -8,13 +8,13 @@ SCRIPTS_PATH=../build-system/s3-cache-scripts/ echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-2-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null +$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-2-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null +$SCRIPTS_PATH/cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts > /dev/null +$SCRIPTS_PATH/cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts yarn diff --git a/noir/bootstrap_cache.sh b/noir/bootstrap_cache.sh index 22bab48283f..8f7eee62d18 100755 --- a/noir/bootstrap_cache.sh +++ b/noir/bootstrap_cache.sh @@ -4,9 +4,9 @@ set -eu cd "$(dirname "$0")" echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" -NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native ../../build-system/s3-cache-scripts/compute-content-hash.sh) -../../build-system/s3-cache-scripts/cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz > /dev/null +NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native ../build-system/s3-cache-scripts/compute-content-hash.sh) +../build-system/s3-cache-scripts/cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" -PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../barretenberg/cpp/.rebuild_patterns .rebuild_patterns_packages" ../../build-system/s3-cache-scripts/compute-content-hash.sh) -../../build-system/s3-cache-scripts/cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz +PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../barretenberg/cpp/.rebuild_patterns .rebuild_patterns_packages" ../build-system/s3-cache-scripts/compute-content-hash.sh) +../build-system/s3-cache-scripts/cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz From 71c005aa29b42d36ca4c1f853b4a3437b36210cc Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 11 Oct 2024 08:04:55 +0000 Subject: [PATCH 18/19] fix: parallel download --- barretenberg/cpp/bootstrap_cache.sh | 21 +++++++++++-------- barretenberg/ts/bootstrap_cache.sh | 17 ++++++++------- noir-projects/Earthfile | 2 +- noir-projects/bootstrap_cache.sh | 13 ++++++------ .../noir-protocol-circuits/bootstrap.sh | 2 -- 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index 4b674f79026..a33362f895b 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -14,13 +14,16 @@ function on_exit() { } trap on_exit EXIT -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-wasm-threads-$HASH.tar.gz $TMP/build-wasm-threads -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release -$SCRIPTS_PATH/cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state +# Parallel download of all the cached builds because they're quite big +echo " +barretenberg-preset-wasm +barretenberg-preset-wasm-threads +barretenberg-preset-release +barretenberg-preset-release-world-state +" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh {}-$HASH.tar.gz $TMP/{}" -# clobber the existing build with the cached build -mv -f $TMP/build-wasm/build build-wasm/ -mv -f $TMP/build-wasm-threads/build build-wasm-threads/ -mv -f $TMP/build-release/build build/ -mv -f $TMP/build-world-state/build/bin/* build/bin/ +# # clobber the existing build with the cached build +cp -r $TMP/barretenberg-preset-wasm/build build-wasm/ +cp -r $TMP/barretenberg-preset-wasm-threads/build build-wasm-threads/ +cp -r $TMP/barretenberg-preset-release/build build/ +cp -r $TMP/barretenberg-preset-release-world-state/build/* build/ diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index 2ad816066db..de6f0978fd0 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -16,13 +16,16 @@ trap on_exit EXIT HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" $CACHE_SCRIPTS/compute-content-hash.sh) -$CACHE_SCRIPTS/cache-download.sh bb.js-esm-$HASH.tar.gz $TMP/esm -$CACHE_SCRIPTS/cache-download.sh bb.js-cjs-$HASH.tar.gz $TMP/cjs -$CACHE_SCRIPTS/cache-download.sh bb.js-browser-$HASH.tar.gz $TMP/browser - -cp -r $TMP/esm/dest dest -cp -r $TMP/cjs/dest dest -cp -r $TMP/browser/dest dest +# Parallel download of all the cached builds because they're quite big +echo " +bb.js-esm +bb.js-cjs +bb.js-browser +" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh {}-$HASH.tar.gz $TMP/{}" + +cp -r $TMP/bb.js-esm/dest dest +cp -r $TMP/bb.js-cjs/dest dest +cp -r $TMP/bb.js-browser/dest dest # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. yarn install diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile index 7883cb970de..3cd70133ddb 100644 --- a/noir-projects/Earthfile +++ b/noir-projects/Earthfile @@ -53,7 +53,7 @@ build-protocol-circuits: WORKDIR /usr/src/noir-projects/noir-protocol-circuits DO ../build-system/s3-cache-scripts/+WITH_CACHE \ - --prefix="noir-projects-noir-protocol-circuits-2" \ + --prefix="noir-projects-noir-protocol-circuits" \ --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \ --command="./bootstrap.sh" \ --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json crates/autogenerated" diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index b6f14ec0aa7..0bef716df91 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -8,14 +8,13 @@ SCRIPTS_PATH=../build-system/s3-cache-scripts/ echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m" PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-noir-protocol-circuits-2-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits - MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits - CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) -$SCRIPTS_PATH/cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts -yarn +echo " +noir-protocol-circuits +mock-protocol-circuits +noir-contracts +" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh noir-projects-{}-$HASH.tar.gz {}" -(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js) +yarn diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index f1e2f981940..c90b7e00328 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -58,6 +58,4 @@ else for pathname in "./target"/*.json; do BB_HASH=$BB_HASH node ../scripts/generate_vk_json.js "$pathname" "./target/keys" done - - fi From d887bbe6183dfdc71c46432db9256629ca214e4f Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 11 Oct 2024 10:22:56 +0000 Subject: [PATCH 19/19] fix: cache --- barretenberg/cpp/bootstrap_cache.sh | 4 +++- barretenberg/ts/bootstrap_cache.sh | 9 +++++---- noir-projects/bootstrap_cache.sh | 8 ++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/barretenberg/cpp/bootstrap_cache.sh b/barretenberg/cpp/bootstrap_cache.sh index a33362f895b..062551fa4ce 100755 --- a/barretenberg/cpp/bootstrap_cache.sh +++ b/barretenberg/cpp/bootstrap_cache.sh @@ -25,5 +25,7 @@ barretenberg-preset-release-world-state # # clobber the existing build with the cached build cp -r $TMP/barretenberg-preset-wasm/build build-wasm/ cp -r $TMP/barretenberg-preset-wasm-threads/build build-wasm-threads/ -cp -r $TMP/barretenberg-preset-release/build build/ + +mkdir -p build +cp -r $TMP/barretenberg-preset-release/build/* build/ cp -r $TMP/barretenberg-preset-release-world-state/build/* build/ diff --git a/barretenberg/ts/bootstrap_cache.sh b/barretenberg/ts/bootstrap_cache.sh index de6f0978fd0..06967c486a3 100755 --- a/barretenberg/ts/bootstrap_cache.sh +++ b/barretenberg/ts/bootstrap_cache.sh @@ -21,11 +21,12 @@ echo " bb.js-esm bb.js-cjs bb.js-browser -" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh {}-$HASH.tar.gz $TMP/{}" +" | xargs --max-procs 0 -I {} bash -c "$CACHE_SCRIPTS/cache-download.sh {}-$HASH.tar.gz $TMP/{}" -cp -r $TMP/bb.js-esm/dest dest -cp -r $TMP/bb.js-cjs/dest dest -cp -r $TMP/bb.js-browser/dest dest +mkdir -p dest +cp -r $TMP/bb.js-esm/dest/* dest/ +cp -r $TMP/bb.js-cjs/dest/* dest/ +cp -r $TMP/bb.js-browser/dest/* dest/ # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled. yarn install diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh index 0bef716df91..df73bf279a4 100755 --- a/noir-projects/bootstrap_cache.sh +++ b/noir-projects/bootstrap_cache.sh @@ -12,9 +12,9 @@ MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_nat CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh) echo " -noir-protocol-circuits -mock-protocol-circuits -noir-contracts -" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh noir-projects-{}-$HASH.tar.gz {}" +noir-protocol-circuits $PROTOCOL_CIRCUITS_HASH +mock-protocol-circuits $MOCK_CIRCUITS_HASH +noir-contracts $CONTRACTS_HASH +" | xargs --max-procs 0 --max-args 2 bash -c "$SCRIPTS_PATH/cache-download.sh noir-projects-\$0-\$1.tar.gz \$0" yarn