fix: cache noir projects

AztecProtocol · Oct 10, 2024 · d9e6864 · d9e6864
1 parent 7897420
commit d9e6864
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 46 deletions.
diff --git a/noir-projects/Earthfile b/noir-projects/Earthfile
@@ -1,80 +1,91 @@
 VERSION 0.8
 
-source:
-  FROM ../build-images+from-registry
+deps:
+  LOCALLY
+  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
 
-  # Install nargo
+  FROM ../build-images+from-registry
   COPY ../noir/+nargo/nargo /usr/bin/nargo
-  # Install world state napi
-  COPY ../barretenberg/cpp/+preset-release-world-state/bin/world_state_napi.node /usr/src/barretenberg/cpp/build/bin/world_state_napi.node
+  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
+      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
+  ENV BB_HASH=$bb_source_hash
+  ENV NARGO=nargo
 
+source:
+  FROM +deps
   WORKDIR /usr/src/noir-projects
 
-  # Copy source.
-  COPY package.json yarn.lock mega_honk_circuits.json .
+  COPY package.json yarn.lock .
+  RUN yarn
+
+  COPY mega_honk_circuits.json .
   COPY --dir aztec-nr noir-contracts noir-protocol-circuits mock-protocol-circuits scripts .
 
 build-contracts:
   ARG RAYON_NUM_THREADS
-  LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
-  FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
 
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
+  FROM +source
+  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
 
   # Install transpiler
   COPY ../avm-transpiler/+build/avm-transpiler /usr/bin/avm-transpiler
+  ENV TRANSPILER=avm-transpiler
 
-  RUN yarn
-  ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN cd noir-contracts && NARGO=nargo TRANSPILER=avm-transpiler  ./bootstrap.sh
-  SAVE ARTIFACT noir-contracts
+  WORKDIR /usr/src/noir-projects/noir-contracts
+
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-noir-contracts" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../avm-transpiler/.rebuild_patterns ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target"
+
+  SAVE ARTIFACT target
 
 build-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
-
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN yarn
+  WORKDIR /usr/src/noir-projects/noir-protocol-circuits
 
   ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN cd noir-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
-  SAVE ARTIFACT noir-protocol-circuits
+  ENV PARALLEL_VK=false
+
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-noir-protocol-circuits" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="yarn && ./bootstrap.sh" \
+    --build_artifacts="target"
+
+  SAVE ARTIFACT target
 
 build-mock-protocol-circuits:
   ARG RAYON_NUM_THREADS
   LOCALLY
-  LET bb_source_hash = $(cd .. && git ls-tree -r HEAD | grep 'barretenberg/cpp' | awk '{print $3}' | git hash-object --stdin)
   FROM +source
-  RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p ~/.aws && \
-      bash -c 'echo -e "[default]\naws_access_key_id=$AWS_ACCESS_KEY_ID\naws_secret_access_key=$AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials'
 
-  # Install bb
-  COPY ../barretenberg/cpp/+preset-release/bin/bb /usr/src/barretenberg/cpp/build/bin/bb
-  RUN yarn
+  WORKDIR /usr/src/noir-projects/mock-protocol-circuits
 
   ENV RAYON_NUM_THREADS=$RAYON_NUM_THREADS
-  RUN echo "building with num threads $RAYON_NUM_THREADS"
-  RUN cd mock-protocol-circuits && BB_HASH=$bb_source_hash NARGO=nargo PARALLEL_VK=false ./bootstrap.sh
-  SAVE ARTIFACT mock-protocol-circuits
+  ENV PARALLEL_VK=false
+
+  DO ../build-system/s3-cache-scripts/+WITH_CACHE \
+    --prefix="noir-projects-mock-protocol-circuits" \
+    --rebuild_patterns="../../noir/.rebuild_patterns_native ../../barretenberg/cpp/.rebuild_patterns .rebuild_patterns" \
+    --command="./bootstrap.sh" \
+    --build_artifacts="target"
+
+  SAVE ARTIFACT target
 
 build:
   FROM +source
   BUILD +build-contracts
   BUILD +build-protocol-circuits
   BUILD +build-mock-protocol-circuits
 
-  COPY +build-contracts/noir-contracts ./noir-contracts
-  COPY +build-protocol-circuits/noir-protocol-circuits ./noir-protocol-circuits
-  COPY +build-mock-protocol-circuits/mock-protocol-circuits ./mock-protocol-circuits
+  COPY --dir +build-contracts/target ./noir-contracts
+  COPY --dir +build-protocol-circuits/target ./noir-protocol-circuits
+  COPY --dir +build-mock-protocol-circuits/target ./mock-protocol-circuits
 
   SAVE ARTIFACT aztec-nr
   SAVE ARTIFACT noir-contracts
@@ -109,7 +120,7 @@ format:
   FROM +source
 
   WORKDIR /usr/src/noir-projects/noir-protocol-circuits
-  RUN yarn && node ./scripts/generate_variants.js
+  RUN node ./scripts/generate_variants.js
   RUN nargo fmt --check
 
   WORKDIR /usr/src/noir-projects/mock-protocol-circuits

diff --git a/noir-projects/bootstrap_cache.sh b/noir-projects/bootstrap_cache.sh
@@ -5,13 +5,16 @@ cd "$(dirname "$0")"
 source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null
 
 echo -e "\033[1mRetrieving noir projects from remote cache...\033[0m"
-extract_repo_if_working_copy_clean noir-projects \
-  /usr/src/noir-projects/noir-contracts/target ./noir-contracts \
-  /usr/src/noir-projects/noir-protocol-circuits/target ./noir-protocol-circuits
 
-remove_old_images noir-projects
+PROTOCOL_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./noir-protocol-circuits/.rebuild_patterns" compute-content-hash.sh)
+cache-download.sh noir-projects-noir-protocol-circuits-$PROTOCOL_CIRCUITS_HASH.tar.gz noir-protocol-circuits > /dev/null
+
+MOCK_CIRCUITS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../barretenberg/cpp/.rebuild_patterns ./mock-protocol-circuits/.rebuild_patterns" compute-content-hash.sh)
+cache-download.sh noir-projects-mock-protocol-circuits-$MOCK_CIRCUITS_HASH.tar.gz mock-protocol-circuits > /dev/null
+
+CONTRACTS_HASH=$(AZTEC_CACHE_REBUILD_PATTERNS="../noir/.rebuild_patterns_native ../avm-transpiler/.rebuild_patterns ../barretenberg/cpp/.rebuild_patterns noir-contracts/.rebuild_patterns" compute-content-hash.sh)
+cache-download.sh noir-projects-noir-contracts-$CONTRACTS_HASH.tar.gz noir-contracts > /dev/null
 
 yarn
 
-./mock-protocol-circuits/bootstrap.sh
-(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)
+(cd ./noir-protocol-circuits && yarn && node ./scripts/generate_variants.js)
diff --git a/noir-projects/mock-protocol-circuits/.rebuild_patterns b/noir-projects/mock-protocol-circuits/.rebuild_patterns
@@ -0,0 +1,3 @@
+^noir-projects/mock-protocol-circuits/.*\.(nr|toml)$
+^noir-projects/mock-protocol-circuits/bootstrap.sh$
+^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$
diff --git a/noir-projects/noir-contracts/.rebuild_patterns b/noir-projects/noir-contracts/.rebuild_patterns
@@ -0,0 +1,4 @@
+^noir-projects/noir-contracts/.*\.(nr|toml)$
+^noir-projects/noir-contracts/bootstrap.sh$
+^noir-projects/aztec-nr/.*\.(nr|toml)$
+^noir-projects/noir-protocol-circuits/crates/types/.*\.(nr|toml)$
diff --git a/noir-projects/noir-protocol-circuits/.rebuild_patterns b/noir-projects/noir-protocol-circuits/.rebuild_patterns
@@ -0,0 +1,2 @@
+^noir-projects/noir-protocol-circuits/.*\.(nr|toml)$
+^noir-projects/noir-protocol-circuits/bootstrap.sh$
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		^noir-projects/noir-protocol-circuits/.*\.(nr\|toml)$
		^noir-projects/noir-protocol-circuits/bootstrap.sh$