feat: use s3 cache in bootstrap fast

avm-transpiler/Earthfile

@@ -8,11 +8,16 @@ source:
     WORKDIR /usr/src/avm-transpiler
 
     COPY --dir scripts src Cargo.lock Cargo.toml rust-toolchain.toml .rustfmt.toml .
+    DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns"
 
 build:
     FROM +source
     # build avm transpiler, and make sure the big build and deps folders don't hit cache
-    RUN ./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}
+    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+        --prefix="avm-transpiler" \
+        --rebuild_patterns=".rebuild_patterns" \
+        --command="./scripts/bootstrap_native.sh && rm -rf target/release/{build,deps}" \
+        --build_artifacts="target"
     SAVE ARTIFACT target/release/avm-transpiler avm-transpiler
     SAVE ARTIFACT scripts/compile_then_transpile.sh
 

avm-transpiler/bootstrap_cache.sh

@@ -5,7 +5,5 @@ cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving avm-transpiler from remote cache...\033[0m"
-extract_repo_if_working_copy_clean avm-transpiler \
-  /usr/src/avm-transpiler/target/release/avm-transpiler ./target/release/
-
-remove_old_images avm-transpiler
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh)
+cache-download.sh avm-transpiler-$HASH.tar.gz > /dev/null

barretenberg/cpp/bootstrap_cache.sh

@@ -5,13 +5,20 @@ cd "$(dirname "$0")"
 source ../../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving bb binary from remote cache...\033[0m"
-extract_repo_if_working_copy_clean barretenberg-x86_64-linux-clang \
-  /usr/src/barretenberg/cpp/build/bin ./build \
-  /usr/src/barretenberg/cpp/build-pic/lib ./build-pic
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh)
+TMP=$(mktemp -d)
 
-echo -e "\033[1mRetrieving bb.wasm from remote cache...\033[0m"
-extract_repo_if_working_copy_clean barretenberg-wasm-linux-clang \
-  /usr/src/barretenberg/cpp/build-wasm/bin ./build-wasm \
-  /usr/src/barretenberg/cpp/build-wasm-threads/bin ./build-wasm-threads
+function on_exit() {
+  rm -rf "$TMP"
+}
+trap on_exit EXIT
 
-remove_old_images barretenberg-wasm-linux-clang
+cache-download.sh barretenberg-preset-wasm-$HASH.tar.gz $TMP/build-wasm > /dev/null
+cache-download.sh barretenberg-preset-wasm-threads-v1-$HASH.tar.gz $TMP/build-wasm-threads > /dev/null
+cache-download.sh barretenberg-preset-release-$HASH.tar.gz $TMP/build-release > /dev/null
+cache-download.sh barretenberg-preset-release-world-state-$HASH.tar.gz $TMP/build-world-state > /dev/null
+
+mv -n $TMP/build-wasm/build build-wasm/
+mv -n $TMP/build-wasm-threads/build build-wasm-threads/
+mv -n $TMP/build-release/build build/
+mv -n $TMP/build-world-state/build/bin/* build/bin/

bootstrap.sh

@@ -128,7 +128,7 @@ elif [ "$CMD" = "full" ]; then
 elif [ "$CMD" = "fast" ]; then
   export USE_CACHE=1
   if ! can_use_ci_cache; then
-    echo -e "${BOLD}${YELLOW}WARNING: Either docker or aws credentials are missing. Install docker and request credentials. Note this is for internal aztec devs only.${RESET}"
+    echo -e "${BOLD}${YELLOW}WARNING: AWS credentials are missing. Note this is for internal aztec devs only.${RESET}"
     exit 1
   fi
 elif [ "$CMD" = "check" ]; then

build-system/s3-cache-scripts/cache-download.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 set -eu
 
-if [ "$#" -ne 1 ]; then
+if [ "$#" -lt 1 ]; then
     echo "Usage: $0 <tar.gz_file_to_download_and_extract>"
     exit 1
 fi
 
 # Get the tar.gz file name from the argument
 TAR_FILE="$1"
+OUT_DIR="${2:-.}"
 
 function on_exit() {
   # Cleanup the temporary tar.gz file
@@ -20,6 +21,7 @@ trap on_exit EXIT
 aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "s3://aztec-ci-artifacts/build-cache/$TAR_FILE" "$TAR_FILE" --quiet
 
 # Extract the cache file
-tar -xzf "$TAR_FILE"
+mkdir -p "$OUT_DIR"
+tar -xzf "$TAR_FILE" -C "$OUT_DIR"
 
 echo "Cache download and extraction complete."

build-system/s3-cache-scripts/cache-upload.sh

@@ -24,4 +24,4 @@ trap on_exit EXIT
 # Rest of args are our binary paths
 tar -czf "$TAR_FILE" $@
 
-aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"
+aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp "$TAR_FILE" "s3://aztec-ci-artifacts/build-cache/$NAME"

build-system/scripts/can_use_ci_cache

@@ -1,3 +1,11 @@
 #!/usr/bin/env bash
 set -euo pipefail
-type docker &> /dev/null && docker ps 2>&1 > /dev/null && [ -f ~/.aws/credentials ] && ecr_login > /dev/null || exit 1
+if [ -n $AWS_ACCESS_KEY_ID ] && [ -n $AWS_SECRET_ACCESS_KEY ]; then
+  exit 0
+elif [ -f ~/.aws/credentials ]; then
+  echo "export AWS_ACCESS_KEY_ID=\$(aws configure get default.aws_access_key_id)" >> $BASH_ENV
+  echo "export AWS_SECRET_ACCESS_KEY=\$(aws configure get default.aws_secret_access_key)" >> $BASH_ENV
+  exit 0
+else
+  exit 1
+fi

build-system/scripts/setup_env

@@ -25,7 +25,7 @@ ROOT_PATH=$(git rev-parse --show-toplevel)
 BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
 PROJECT=$(cat $ROOT_PATH/PROJECT)
 COMMIT_MESSAGE=$(git log -n 1 --pretty=format:"%s" $COMMIT_HASH)
-PATH=$PATH:$BUILD_SYSTEM_PATH/scripts
+PATH=$PATH:$BUILD_SYSTEM_PATH/scripts:$BUILD_SYSTEM_PATH/s3-cache-scripts
 
 check_env
 
@@ -81,7 +81,7 @@ echo export BUILD_SYSTEM_PATH=$BUILD_SYSTEM_PATH >> $BASH_ENV
 echo export DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1} >> $BASH_ENV
 echo export BUILDKIT_PROGRESS=plain >> $BASH_ENV
 echo export SSH_CONFIG_PATH=$BUILD_SYSTEM_PATH/remote/ssh_config >> $BASH_ENV
-echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts >> $BASH_ENV
+echo export PATH=\$PATH:$BUILD_SYSTEM_PATH/scripts:$BUILD_SYSTEM_PATH/s3-cache-scripts >> $BASH_ENV
 echo export AWS_DEFAULT_REGION=eu-west-2 >> $BASH_ENV
 echo export ECR_REGION=us-east-2 >> $BASH_ENV
 echo export AWS_ACCOUNT=278380418400 >> $BASH_ENV

l1-contracts/.rebuild_patterns

@@ -0,0 +1 @@
+^l1-contracts/src/.*\.sol$

l1-contracts/Earthfile

@@ -1,16 +1,27 @@
 VERSION 0.8
 
-build:
+src:
     FROM ../build-images+from-registry
     WORKDIR /usr/src/l1-contracts
     COPY --dir lib src terraform test *.json *.toml *.sh .
-    #RUN git init && git add . && yarn lint && yarn slither && yarn slither-has-diff
-    # "slither": "forge clean && forge build --build-info --skip '*/test/**' --force && slither . --checklist --ignore-compile --show-ignored-findings --config-file ./slither.config.json | tee slither_output.md",
-    # "slither-has-diff": "./slither_has_diff.sh"
+    DO ../build-system/s3-cache-scripts/+WRITE_CONTENT_HASH --rebuild_patterns=".rebuild_patterns"
+
+lint:
+    FROM +src
     RUN solhint --config ./.solhint.json --fix "src/**/*.sol"
     RUN forge clean && forge fmt --check
-    RUN forge build
-    SAVE ARTIFACT /usr/src/l1-contracts /usr/src/l1-contracts
+
+build:
+    FROM +lint
+    WORKDIR /usr/src/l1-contracts
+
+    DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+        --prefix="l1-contracts" \
+        --rebuild_patterns=".rebuild_patterns" \
+        --command="forge build" \
+        --build_artifacts="out"
+
+    SAVE ARTIFACT /usr/src/l1-contracts/out /usr/src/l1-contracts/out
 
 test:
     FROM +build

l1-contracts/bootstrap_cache.sh

@@ -5,6 +5,5 @@ cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving contracts from remote cache...\033[0m"
-extract_repo_if_working_copy_clean l1-contracts /usr/src/l1-contracts/out .
-
-remove_old_images l1-contracts
+HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns compute-content-hash.sh)
+cache-download.sh l1-contracts-$HASH.tar.gz > /dev/null

noir/bootstrap_cache.sh

@@ -5,9 +5,9 @@ cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m"
-extract_repo_if_working_copy_clean noir-packages /usr/src/noir/packages ./
-echo -e "\033[1mRetrieving nargo from remote cache...\033[0m"
-extract_repo_if_working_copy_clean noir /usr/src/noir/noir-repo/target/release ./noir-repo/target/
+NATIVE_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_native compute-content-hash.sh)
+cache-download.sh noir-nargo-$NATIVE_HASH.tar.gz > /dev/null
 
-remove_old_images noir-packages
-remove_old_images noir
+echo -e "\033[1mRetrieving nargo from remote cache...\033[0m"
+PACKAGES_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS=.rebuild_patterns_packages compute-content-hash.sh)
+cache-download.sh noir-packages-$PACKAGES_HASH.tar.gz

scripts/earthly-local

@@ -10,8 +10,7 @@ function start_minio() {
     # Already started
     return
   fi
-  mkdir -p ~/.minio/data
-  docker run -d -p 12000:9000 -p 12001:12001 -v ~/.minio/data:/data \
+  docker run -d -p 12000:9000 -p 12001:12001 -v minio-data:/data \
     quay.io/minio/minio server /data --console-address ":12001"
   # make our cache bucket
   AWS_ACCESS_KEY_ID="minioadmin" AWS_SECRET_ACCESS_KEY="minioadmin" aws --endpoint-url http://localhost:12000 s3 mb s3://aztec-ci-artifacts 2>/dev/null || true

yarn-project/Earthfile

@@ -45,7 +45,7 @@ build:
     COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node
     COPY ../noir/+nargo/acvm /usr/src/noir/noir-repo/target/release/acvm
     COPY --dir ../noir-projects/+build/. /usr/src/noir-projects
-    COPY ../l1-contracts/+build/usr/src/l1-contracts /usr/src/l1-contracts
+    COPY ../l1-contracts/+build/usr/src/l1-contracts/out /usr/src/l1-contracts/out
 
     WORKDIR /usr/src/yarn-project
     COPY . .