chore: min noir build (#4812)

* move noir repo from `noir` to `noir/noir-repo`
* move aztec specific build files out of noir-repo into `noir/*`.
* only build the 3 packages we need when bootstrapping.
* build and run native tests and all package tests in ci, as separate
jobs for speed, to prevent discovering regressions when we merge back to
`noir-lang/noir`.

`full` bootstrap (not `fast`) from clean, was 10m on new mainframe.

.circleci/config.yml

@@ -124,6 +124,18 @@ jobs:
           command: create_ecr_manifest noir x86_64,arm64
           aztec_manifest_key: noir
 
+  noir-tests:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build"
+          command: cond_spot_run_build noir-tests 32
+          aztec_manifest_key: noir-tests
+
   noir-packages:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -136,6 +148,18 @@ jobs:
           command: cond_spot_run_build noir-packages 32
           aztec_manifest_key: noir-packages
 
+  noir-packages-tests:
+    docker:
+      - image: aztecprotocol/alpine-build-image
+    resource_class: small
+    steps:
+      - *checkout
+      - *setup_env
+      - run:
+          name: "Build"
+          command: cond_spot_run_build noir-packages-tests 32
+          aztec_manifest_key: noir-packages-tests
+
   noir-compile-acir-tests:
     docker:
       - image: aztecprotocol/alpine-build-image
@@ -1335,7 +1359,12 @@ workflows:
             - noir-x86_64
             - noir-arm64
           <<: *defaults
+      - noir-tests: *defaults
       - noir-packages: *defaults
+      - noir-packages-tests:
+          requires:
+            - noir-ecr-manifest
+          <<: *defaults
       - noir-compile-acir-tests:
           requires:
             - noir-ecr-manifest

.github/workflows/protocol-circuits-gate-diff.yml

@@ -72,7 +72,7 @@ jobs:
       - name: Install nargo from source with noirup
         run: noirup $toolchain
         env:
-          toolchain: --path ./noir
+          toolchain: --path ./noir/noir-repo
 
       - name: Check nargo installation
         run: nargo --version

avm-transpiler/Cargo.toml

@@ -9,8 +9,8 @@ license = "MIT OR Apache-2.0"
 
 [dependencies]
 # local
-acvm = { path = "../noir/acvm-repo/acvm" }
-noirc_driver = { path = "../noir/compiler/noirc_driver" }
+acvm = { path = "../noir/noir-repo/acvm-repo/acvm" }
+noirc_driver = { path = "../noir/noir-repo/compiler/noirc_driver" }
 
 # external
 base64 = "0.21"

barretenberg/acir_tests/Dockerfile.bb

@@ -4,7 +4,7 @@ FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests
 FROM node:18.19.0-alpine
 RUN apk update && apk add git bash curl jq coreutils
 COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
-COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
+COPY --from=noir-acir-tests /usr/src/noir/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .
 # Run every acir test through native bb build prove_then_verify flow for UltraPlonk.

barretenberg/acir_tests/Dockerfile.bb.js

@@ -3,7 +3,7 @@ FROM aztecprotocol/noir-compile-acir-tests as noir-acir-tests
 
 FROM node:18.19.0
 COPY --from=0 /usr/src/barretenberg/ts-build /usr/src/barretenberg/ts
-COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
+COPY --from=noir-acir-tests /usr/src/noir/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs
 RUN apt update && apt install -y lsof jq
 WORKDIR /usr/src/barretenberg/acir_tests
 # Build/install ts apps.

barretenberg/acir_tests/Dockerfile.bb.sol

@@ -6,7 +6,7 @@ FROM node:18.19.0-alpine
 RUN apk update && apk add git bash curl jq
 COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
 COPY --from=1 /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol /usr/src/barretenberg/sol/src/ultra/BaseUltraVerifier.sol
-COPY --from=noir-acir-tests /usr/src/noir/test_programs /usr/src/noir/test_programs
+COPY --from=noir-acir-tests /usr/src/noir/noir-repo/test_programs /usr/src/noir/noir-repo/test_programs
 COPY --from=ghcr.io/foundry-rs/foundry:latest /usr/local/bin/anvil /usr/local/bin/anvil
 WORKDIR /usr/src/barretenberg/acir_tests
 COPY . .

barretenberg/acir_tests/Dockerfile.noir_acir_tests

@@ -4,7 +4,7 @@
 # This chains off the nargo build, and creates a container with a compiled set of acir tests.
 FROM aztecprotocol/noir
 RUN apt update && apt install -y jq && rm -rf /var/lib/apt/lists/* && apt-get clean
-ENV PATH="/usr/src/noir/target/release:${PATH}"
-WORKDIR /usr/src/noir/test_programs
+ENV PATH="/usr/src/noir/noir-repo/target/release:${PATH}"
+WORKDIR /usr/src/noir/noir-repo/test_programs
 COPY . .
 RUN ./rebuild.sh

barretenberg/acir_tests/clone_test_vectors.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -eu
 
-TEST_SRC=${TEST_SRC:-../../noir/test_programs/acir_artifacts}
+TEST_SRC=${TEST_SRC:-../../noir/noir-repo/test_programs/acir_artifacts}
 
 if [ ! -d acir_tests ]; then
   cp -R $TEST_SRC acir_tests

barretenberg/acir_tests/run_acir_tests.sh

@@ -29,7 +29,7 @@ fi
 
 export BIN CRS_PATH VERBOSE BRANCH
 
-# copy the gzipped acir test data from noir/test_programs to barretenberg/acir_tests
+# copy the gzipped acir test data from noir/noir-repo/test_programs to barretenberg/acir_tests
 ./clone_test_vectors.sh
 
 cd acir_tests

boxes/Dockerfile

@@ -8,10 +8,10 @@ FROM aztecprotocol/noir-projects as noir-projects
 FROM node:18.19.0
 RUN apt update && apt install netcat-openbsd
 COPY --from=aztec /usr/src /usr/src
-COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo
+COPY --from=noir /usr/src/noir/noir-repo/target/release/nargo /usr/src/noir/noir-repo/target/release/nargo
 COPY --from=noir-projects /usr/src/noir-projects/aztec-nr /usr/src/noir-projects/aztec-nr
 WORKDIR /usr/src/boxes
-ENV AZTEC_NARGO=/usr/src/noir/target/release/nargo
+ENV AZTEC_NARGO=/usr/src/noir/noir-repo/target/release/nargo
 ENV AZTEC_CLI=/usr/src/yarn-project/cli/aztec-cli-dest
 RUN yarn && yarn build
 ENTRYPOINT ["/bin/sh", "-c"]

build-system/scripts/build

@@ -131,7 +131,7 @@ if [ "$MULTIARCH" == "buildx" ]; then
   # This is the simplest approach for build jobs that are not too intensive.
   docker buildx create --name builder --use
   docker buildx inspect --bootstrap
-  docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH --platform linux/amd64,linux/arm64 . --push
+  docker buildx build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg CONTENT_HASH=$CONTENT_HASH --build-arg COMMIT_HASH=$COMMIT_HASH --platform linux/amd64,linux/arm64 . --push
 else
   # If multiarch is set to "host", the assumption is that we're doing multiple builds on different machine architectures
   # in parallel, and that there is a another job that runs afterwards to combine them into a manifest.
@@ -140,7 +140,7 @@ else
     IMAGE_COMMIT_URI=$(calculate_image_uri $REPOSITORY host)
   fi
 
-  docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg ARG_CONTENT_HASH=$CONTENT_HASH .
+  docker build -t $IMAGE_COMMIT_URI -f $DOCKERFILE --build-arg COMMIT_TAG=$COMMIT_TAG_VERSION --build-arg CONTENT_HASH=$CONTENT_HASH --build-arg COMMIT_HASH=$COMMIT_HASH .
   echo "Pushing image: $IMAGE_COMMIT_URI"
   retry docker push $IMAGE_COMMIT_URI > /dev/null 2>&1
 fi

build-system/scripts/build_local

@@ -13,6 +13,7 @@ TARGET_PROJECT=${1:-}
 ONLY_TARGET=${ONLY_TARGET:-}
 NO_CACHE=${NO_CACHE:-}
 MANIFEST="build_manifest.yml"
+COMMIT_HASH=$(git rev-parse --verify HEAD)
 
 if [ -n "$NO_CACHE" ]; then
   ADDITIONAL_ARGS="${ADDITIONAL_ARGS:-} --no-cache"
@@ -95,15 +96,15 @@ for E in "${PROJECTS[@]}"; do
 
   if (cd $(git rev-parse --show-toplevel) && git diff-index --name-only HEAD; git ls-files --others --exclude-standard) | grep -qE "$GREP_PATTERN"; then
     echo -e "${YELLOW}Project or dependency has local modifications! Building...${RESET}"
-    docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI .
+    docker build ${ADDITIONAL_ARGS:-} --build-arg COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI .
   else
     if [ -z "$NO_CACHE" ] && docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w "$CACHE_IMAGE_URI$"; then
       echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}"
     else
       if [ -z "$NO_CACHE" ] && [ -f ~/.aws/credentials ] && ecr_login && image_exists $REPO $TAG ; then
         docker pull $CACHE_IMAGE_URI
       else
-        docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI .
+        docker build ${ADDITIONAL_ARGS:-} --build-arg COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI .
       fi
     fi
   fi

build-system/scripts/query_manifest

@@ -130,4 +130,5 @@ case "$CMD" in
     ;;
   multiarch)
     yq -r ".\"$REPO\".multiarch // false" $MANIFEST
+    ;;
 esac