Merge 76e8e34 into 002b4aa

.github/workflows/ci.yml

@@ -167,19 +167,33 @@ jobs:
           concurrency_key: build-x86
       - name: "Build and upload bench aggregate file"
         working-directory: ./yarn-project/scripts
-        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate
-      - name: "Download base benchmark"
+        run: |
+          earthly-ci -P \
+            --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+            --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+            +bench-aggregate
+      - name: "Download base benchmark and package into earthly"
         if: ${{ github.event_name == 'pull_request' }}
-        run: scripts/logs/download_base_benchmark_from_s3.sh
+        run: |
+          # Download the base benchmark locally (requires AWS creds and .git history)
+          mkdir -p $BENCH_FOLDER
+          ./scripts/logs/download_base_benchmark_from_s3.sh
+          # Package it into an earthly artifact to read from bench-comment
+          earthly-ci -P ./scripts/logs+pack-base-benchmark --LOCAL_BENCH_FOLDER=./tmp/bench
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          BENCH_FOLDER: ./yarn-project/scripts/bench
+          BENCH_FOLDER: "./scripts/logs/tmp/bench"
           PULL_REQUEST: "${{ github.event.pull_request.number }}"
       - name: "Generate summary comment if pull request"
         if: ${{ github.event_name == 'pull_request' }}
         working-directory: ./yarn-project/scripts
-        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment
+        run: |
+          earthly-ci -P \
+            --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+            --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+            --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} \
+            +bench-comment
 
   noir-format:
     needs: setup

scripts/logs/Earthfile

@@ -0,0 +1,19 @@
+VERSION 0.8
+FROM node:18.19.0
+
+pack-base-benchmark:
+  # Copies the base benchmark (ie the master run) into a container and packs it as an artifact,
+  # so it can be consumed from bench-comment. Note that we need to download base-benchmark 
+  # outside of this target beforehand. We cannot run it within an Earthly container because it needs
+  # access to the .git folder, and we cannot run it with a LOCALLY statement because Earthly does 
+  # not support secrets when running locally (and we need) the AWS access keys to access S3. 
+  # We also cannot COPY the local file directly from bench-comment, since the file must be present 
+  # within the build context so we can copy it (ie within yarn-project/scripts), and that invalidates 
+  # the cache of yarn-project+build since it does a `COPY . .`, and we cannot add the bench file to
+  # earthlyignore or we would not be able to copy it from anywhere. So we need to place this target
+  # outside yarn-project altogether, since that folder should not be modified.
+  ARG LOCAL_BENCH_FOLDER
+  FROM scratch
+  LET BENCH_FOLDER=/usr/var/bench
+  COPY $LOCAL_BENCH_FOLDER $BENCH_FOLDER
+  SAVE ARTIFACT $BENCH_FOLDER bench

yarn-project/scripts/Earthfile

@@ -13,9 +13,7 @@ download-logs:
   ARG BRANCH
   FROM +build
   LET LOG_FOLDER=/usr/var/logs
-  LET BENCH_FOLDER=/usr/var/bench
   ENV LOG_FOLDER=$LOG_FOLDER
-  ENV BENCH_FOLDER=$BENCH_FOLDER
   ENV COMMIT_HASH=$COMMIT_HASH
   ENV PULL_REQUEST=$PULL_REQUEST
   ENV BRANCH=$BRANCH
@@ -44,11 +42,8 @@ bench-aggregate:
   SAVE ARTIFACT $BENCH_FOLDER bench
 
 bench-comment:
-  # Use the scripts image to run bench comment after loading the benchmark from bench-aggregate.
-  # Requires base-benchmark to be downloaded outside of this target. Note we cannot run it within
-  # an Earthly container because it needs access to the .git folder, and we cannot run it with a
-  # LOCALLY statement because Earthly does not support secrets when running locally (and we need)
-  # the AWS access keys to access S3. This step is then manually run in the ci.yml.
+  # Use the scripts image to run bench comment after loading the benchmark from bench-aggregate
+  # and the base benchmark (ie the benches from master to compare to) from pack-base-benchmark.
   ARG COMMIT_HASH
   ARG PULL_REQUEST
   ARG BRANCH
@@ -59,8 +54,8 @@ bench-comment:
   ENV PULL_REQUEST=$PULL_REQUEST
   ENV PR_NUMBER=$PULL_REQUEST
   ENV BRANCH=$BRANCH
-  COPY ./bench $BENCH_FOLDER/
-  COPY +bench-aggregate/bench $BENCH_FOLDER
+  COPY ../../scripts/logs+pack-base-benchmark/bench $BENCH_FOLDER/
+  COPY +bench-aggregate/bench $BENCH_FOLDER/
   RUN echo "Bench folder contents $(ls $BENCH_FOLDER)"
   RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN \
     [ -f $BENCH_FOLDER/benchmark.json ] \