fix: Revert "fix: Revert "feat: use s3 cache in bootstrap fast"" (#9182)

Reverts #9181

---------

Co-authored-by: ludamad <adam.domurad@gmail.com>

avm-transpiler/.rebuild_patterns

@@ -1 +1 @@
-^avm-transpiler/.*$
+^avm-transpiler/.*$

avm-transpiler/Earthfile

@@ -12,7 +12,11 @@ source:
 build:
     FROM +source
     # build avm transpiler, and make sure the big build and deps folders don't hit cache
-    RUN ./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}
+    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+        --prefix="avm-transpiler" \
+        --rebuild_patterns="../noir/.rebuild_patterns_native .rebuild_patterns" \
+        --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \
+        --build_artifacts="target"
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
     SAVE ARTIFACT scripts/compile_then_transpile.sh
 

avm-transpiler/bootstrap_cache.sh

@@ -2,10 +2,7 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m"
-extract_repo_if_working_copy_clean avm-transpiler \
-  /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/
-
-remove_old_images avm-transpiler
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native .rebuild_patterns" ../build-system/s3-cache-scripts/compute-content-hash.sh)
+../build-system/s3-cache-scripts/cache-download.sh avm-transpiler-$HASH.tar.gz

barretenberg/cpp/Earthfile

@@ -131,7 +131,7 @@ preset-sol:
 
 preset-wasm-threads:
   FROM +source
-  DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads-v1 \
+  DO +CACHE_BUILD_BIN --prefix=preset-wasm-threads \
     --command="cmake --preset wasm-threads -Bbuild && cmake --build build --target barretenberg.wasm"
   SAVE ARTIFACT build/bin
 

barretenberg/cpp/bootstrap_cache.sh

@@ -2,16 +2,30 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving bb binary from remote cache...\033[0m"
-extract_repo_if_working_copy_clean barretenberg-x86_64-linux-clang \
-  /usr/src/barretenberg/cpp/build/bin ./build \
-  /usr/src/barretenberg/cpp/build-pic/lib ./build-pic
 
-echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m"
-extract_repo_if_working_copy_clean barretenberg-wasm-linux-clang \
-  /usr/src/barretenberg/cpp/build-wasm/bin ./build-wasm \
-  /usr/src/barretenberg/cpp/build-wasm-threads/bin ./build-wasm-threads
+SCRIPTS_PATH=../../build-system/s3-cache-scripts/
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns $SCRIPTS_PATH/compute-content-hash.sh)
+TMP=$(mktemp -d)
 
-remove_old_images barretenberg-wasm-linux-clang
+function on_exit() {
+  rm -rf "$TMP"
+}
+trap on_exit EXIT
+
+# Parallel download of all the cached builds because they're quite big
+echo "
+barretenberg-preset-wasm
+barretenberg-preset-wasm-threads
+barretenberg-preset-release
+barretenberg-preset-release-world-state
+" | xargs --max-procs 0 -I {} bash -c "$SCRIPTS_PATH/cache-download.sh {}-$HASH.tar.gz $TMP/{}"
+
+# # clobber the existing build with the cached build
+cp -r $TMP/barretenberg-preset-wasm/build build-wasm/
+cp -r $TMP/barretenberg-preset-wasm-threads/build build-wasm-threads/
+
+mkdir -p build
+cp -r $TMP/barretenberg-preset-release/build/* build/
+cp -r $TMP/barretenberg-preset-release-world-state/build/* build/

barretenberg/ts/.rebuild_patterns

@@ -1 +1 @@
-^barretenberg/ts/.*$
+^barretenberg/ts/.*$

barretenberg/ts/Earthfile

@@ -6,12 +6,11 @@ CACHE:
   ARG build_artifacts
   ARG prefix
   # TODO(#8929): reinstate bb.js caching
-  RUN $command
-  # DO ../../build-system/s3-cache-scripts/+WITH_CACHE \
-  #   --prefix="bb.js-$prefix" \
-  #   --command="$command" \
-  #   --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \
-  #   --build_artifacts="$build_artifacts"
+  DO ../../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="bb.js-$prefix" \
+    --command="$command" \
+    --rebuild_patterns="../cpp/.rebuild_patterns .rebuild_patterns" \
+    --build_artifacts="$build_artifacts"
 
 deps:
     FROM ../../build-images+from-registry
@@ -34,7 +33,7 @@ deps:
 esm:
     FROM +deps
     DO +CACHE \
-      --prefix="browser" \
+      --prefix="esm" \
       --command="yarn build:esm" \
       --build_artifacts="."
     SAVE ARTIFACT /usr/src/barretenberg/ts build

barretenberg/ts/bootstrap_cache.sh

@@ -4,9 +4,29 @@ set -eu
 cd "$(dirname "$0")"
 source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
+CACHE_SCRIPTS=../../build-system/s3-cache-scripts
+
 echo -e "\033[1mRetrieving bb.js from remote cache...\033[0m"
-extract_repo_if_working_copy_clean bb.js /usr/src/barretenberg/ts/dest .
+TMP=$(mktemp -d)
+
+function on_exit() {
+  rm -rf "$TMP"
+}
+trap on_exit EXIT
+
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../cpp/.rebuild_patterns .rebuild_patterns" $CACHE_SCRIPTS/compute-content-hash.sh)
+
+# Parallel download of all the cached builds because they're quite big
+echo "
+bb.js-esm
+bb.js-cjs
+bb.js-browser
+" | xargs --max-procs 0 -I {} bash -c "$CACHE_SCRIPTS/cache-download.sh {}-$HASH.tar.gz $TMP/{}"
+
+mkdir -p dest
+cp -r $TMP/bb.js-esm/dest/* dest/
+cp -r $TMP/bb.js-cjs/dest/* dest/
+cp -r $TMP/bb.js-browser/dest/* dest/
+
 # Annoyingly we still need to install modules, so they can be found as part of module resolution when portalled.
 yarn install
-
-remove_old_images bb.js

bootstrap.sh

@@ -128,7 +128,7 @@ elif [ "$CMD" = "full" ]; then
 elif [ "$CMD" = "fast" ]; then
   export USE_CACHE=1
   if ! can_use_ci_cache; then
-    echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}"
+    echo -e "${BOLD}${YELLOW}WARNING: AWS credentials are missing. Note this is for internal aztec devs only.${RESET}"
     exit 1
   fi
 elif [ "$CMD" = "check" ]; then

build-system/s3-cache-scripts/cache-download.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 set -eu
 
-if [ "$#" -ne 1 ]; then
+if [ "$#" -lt 1 ]; then
     echo "Usage: $0 <tar.gz_file_to_download_and_extract>"
     exit 1
 fi
 
 # Get the tar.gz file name from the argument
 TAR_FILE="$1"
+OUT_DIR="${2:-.}"
 
 function on_exit() {
   # Cleanup the temporary tar.gz file
@@ -20,6 +21,7 @@ trap on_exit EXIT
 aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "$TAR_FILE" --quiet --no-progress
 
 # Extract the cache file
-tar -xzf "$TAR_FILE"
+mkdir -p "$OUT_DIR"
+tar -xzf "$TAR_FILE" -C "$OUT_DIR"
 
-echo "Cache download and extraction complete."
+echo "Cache download and extraction of $TAR_FILE complete."

build-system/s3-cache-scripts/cache-upload.sh

@@ -24,4 +24,4 @@ trap on_exit EXIT
 # Rest of args are our binary paths
 tar -czf "$TAR_FILE" $@
 
-aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"
+aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"

build-system/scripts/can_use_ci_cache

@@ -1,3 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
-type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1
+if [ ! -z ${AWS_ACCESS_KEY_ID:-} ] && [ ! -z ${AWS_SECRET_ACCESS_KEY:-} ]; then
+  exit 0
+elif [ -f ~/.aws/credentials ]; then
+  exit 0
+else
+  exit 1
+fi

l1-contracts/.rebuild_patterns

@@ -0,0 +1 @@
+^l1-contracts/src/.*\.sol$

l1-contracts/Earthfile

@@ -1,15 +1,26 @@
 VERSION 0.8
 
-build:
+src:
     FROM ../build-images+from-registry
     WORKDIR /usr/src/l1-contracts
     COPY --dir lib src terraform test *.json *.toml *.sh .
-    #RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff
-    # "slither": "forge clean && forge build --build-info --skip '*/test/**' --force && slither . --checklist --ignore-compile --show-ignored-findings --config-file ./slither.config.json | tee slither_output.md",
-    # "slither-has-diff": "./slither_has_diff.sh"
+    DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns"
+
+lint:
+    FROM +src
     RUN solhint --config ./.solhint.json --fix "src/**/*.sol"
     RUN forge clean && forge fmt --check
-    RUN forge build
+
+build:
+    FROM +lint
+    WORKDIR /usr/src/l1-contracts
+
+    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+        --prefix="l1-contracts" \
+        --rebuild_patterns=".rebuild_patterns" \
+        --command="forge build" \
+        --build_artifacts="out"
+
     SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts
 
 test:

l1-contracts/bootstrap_cache.sh

@@ -2,9 +2,7 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving contracts from remote cache...\033[0m"
-extract_repo_if_working_copy_clean l1-contracts /usr/src/l1-contracts/out .
-
-remove_old_images l1-contracts
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns ../build-system/s3-cache-scripts/compute-content-hash.sh)
+../build-system/s3-cache-scripts/cache-download.sh l1-contracts-$HASH.tar.gz

noir-projects/Earthfile

@@ -1,69 +1,82 @@
 VERSION 0.8
 
-source:
-  FROM ../build-images+from-registry
+deps:
+  LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
 
-  # Install nargo
+  FROM ../build-images+from-registry
   COPY ../noir/+nargo/nargo /usr/bin/nargo
-  # Install world state napi
-  COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node
+  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
+  ENV BB_HASH=$bb_source_hash
+  ENV NARGO=nargo
 
+source:
+  FROM +deps
   WORKDIR /usr/src/noir-projects
 
-  # Copy source.
-  COPY package.json yarn.lock mega_honk_circuits.json .
+  COPY package.json yarn.lock .
+  RUN yarn
+
+  COPY mega_honk_circuits.json .
   COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts .
 
 build-contracts:
   ARG RAYON_NUM_THREADS
-  LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
-  FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
 
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  FROM +source
+  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
 
   # Install transpiler
   COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler
+  ENV TRANSPILER=avm-transpiler
 
-  RUN yarn
-  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler  ./bootstrap.sh
+  WORKDIR /usr/src/noir-projects/noir-contracts
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-noir-contracts" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../avm-transpiler/.rebuild_patterns ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target"
+
+  WORKDIR /usr/src/noir-projects
   SAVE ARTIFACT noir-contracts
 
 build-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
-
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN yarn
 
   ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
+  ENV PARALLEL_VK=false
+
+  WORKDIR /usr/src/noir-projects/noir-protocol-circuits
+
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-noir-protocol-circuits" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target Nargo.toml private_kernel_reset_dimensions.json crates/autogenerated"
+
+  WORKDIR /usr/src/noir-projects
   SAVE ARTIFACT noir-protocol-circuits
 
 build-mock-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
-
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN yarn
 
   ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN echo "building with num threads $RAYON_NUM_THREADS"
-  RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
+  ENV PARALLEL_VK=false
+
+  WORKDIR /usr/src/noir-projects/mock-protocol-circuits
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-mock-protocol-circuits" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target"
+
+  WORKDIR /usr/src/noir-projects
   SAVE ARTIFACT mock-protocol-circuits
 
 build:
@@ -109,7 +122,7 @@ format:
   FROM +source
 
   WORKDIR /usr/src/noir-projects/noir-protocol-circuits
-  RUN yarn && node ./scripts/generate_variants.js
+  RUN node ./scripts/generate_variants.js
   RUN nargo fmt --check
 
   WORKDIR /usr/src/noir-projects/mock-protocol-circuits

noir-projects/bootstrap_cache.sh

@@ -2,16 +2,19 @@
 set -eu
 
 cd "$(dirname "$0")"
-source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
+
+SCRIPTS_PATH=../build-system/s3-cache-scripts/
 
 echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m"
-extract_repo_if_working_copy_clean noir-projects \
-  /usr/src/noir-projects/noir-contracts/target ./noir-contracts \
-  /usr/src/noir-projects/noir-protocol-circuits/target ./noir-protocol-circuits
 
-remove_old_images noir-projects
+PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
+MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
+CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" $SCRIPTS_PATH/compute-content-hash.sh)
 
-yarn
+echo "
+noir-protocol-circuits $PROTOCOL_CIRCUITS_HASH
+mock-protocol-circuits $MOCK_CIRCUITS_HASH
+noir-contracts $CONTRACTS_HASH
+" | xargs --max-procs 0 --max-args 2 bash -c "$SCRIPTS_PATH/cache-download.sh noir-projects-\$0-\$1.tar.gz \$0"
 
-./mock-protocol-circuits/bootstrap.sh
-(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)
+yarn