feat: Sync from noir

.noir-sync-commit

@@ -1 +1 @@
-f337992de96ef656681ebfc96a30c2c9c9b82a70
+913be5b013323449963d31ab00f521f572cfcd67

noir/noir-repo/.github/critical_libraries_status/README.md

@@ -0,0 +1,20 @@
+# Critical Libraries Status
+
+This directory contains one `.failures.jsonl` file per external directory that is checked by CI.
+CI will run the external repository tests and compare the test failures against those recorded
+in these files. If there's a difference, CI will fail.
+
+This allows us to mark some tests as expected to fail if we introduce breaking changes.
+When tests are fixed on the external repository, CI will let us know that we need to remove
+the `.failures.jsonl` failures on our side.
+
+The format of the `.failures.jsonl` files is one JSON per line with a failure:
+
+```json
+{"suite":"one","name":"foo"}
+```
+
+If it's expected that an external repository doesn't compile (because a PR introduces breaking changes
+to, say, the type system) you can remove the `.failures.jsonl` file for that repository and CI
+will pass again. Once the repository compiles again, CI will let us know and require us to put
+back the `.failures.jsonl` file.

noir/noir-repo/.github/critical_libraries_status/noir-lang/ec/.actual.jsonl

@@ -0,0 +1,4 @@
+{"event":"started","name":"one","test_count":1,"type":"suite"}
+{"event":"started","name":"foo","suite":"one","type":"test"}
+{"event":"failed","exec_time":0.05356625,"name":"foo","suite":"one","type":"test"}
+{"event":"ok","failed":0,"ignored":0,"passed":1,"type":"suite"}

noir/noir-repo/.github/critical_libraries_status/noir-lang/ec/.actual.jsonl.jq

@@ -0,0 +1 @@
+{"suite":"one","name":"foo"}

noir/noir-repo/.github/scripts/check_test_results.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -eu
+
+# Usage: ./check_test_results.sh <expected.json> <actual.jsonl>
+# Compares the output of two test results of the same repository.
+# If any of the files doesn't exist or is empty, the script will consider that the test suite
+# couldn't be compiled.
+
+function process_json_lines() {
+  cat $1 | jq -c 'select(.type == "test" and .event == "failed") | {suite: .suite, name: .name}' | jq -s -c 'sort_by(.suite, .name) | .[]' > $1.jq
+}
+
+if [ -f $1 ] && [ -f $2 ]; then
+  # Both files exist, let's compare them
+  $(process_json_lines $2)
+  if ! diff $1 $2.jq; then
+    echo "Error: test failures don't match expected failures"
+    echo "Lines prefixed with '>' are new test failures (you could add them to '$1')"
+    echo "Lines prefixed with '<' are tests that were expected to fail but passed (you could remove them from '$1')"
+  fi
+elif [ -f $1 ]; then
+  # Only the expected file exists, which means the actual test couldn't be compiled.
+  echo "Error: external library tests couldn't be compiled."
+  echo "You could rename '$1' to '$1.does_not_compile' if it's expected that the external library can't be compiled."
+  exit -1
+elif [ -f $2 ]; then
+  # Only the actual file exists, which means we are expecting the external library
+  # not to compile but it did.
+  echo "Error: expected external library not to compile, but it did."
+  echo "You could create '$1' with these contents:"
+  $(process_json_lines $2)
+  cat $2.jq
+  exit -1
+else
+  # Both files don't exists, which means we are expecting the external library not
+  # to compile, and it didn't, so all is good.
+  exit 0
+fi

noir/noir-repo/.github/workflows/reports.yml

@@ -76,7 +76,7 @@ jobs:
 
       - name: Compare gates reports
         id: gates_diff
-        uses: noir-lang/noir-gates-diff@1931aaaa848a1a009363d6115293f7b7fc72bb87
+        uses: noir-lang/noir-gates-diff@84ada11295b9a1e1da7325af4e45e2db9f775175
         with:
           report: gates_report.json
           summaryQuantile: 0.9 # only display the 10% most significant circuit size diffs in the summary (defaults to 20%)
@@ -121,7 +121,7 @@ jobs:
 
       - name: Compare Brillig bytecode size reports
         id: brillig_bytecode_diff
-        uses: noir-lang/noir-gates-diff@d88f7523b013b9edd3f31c5cfddaef87a3fe1b48
+        uses: noir-lang/noir-gates-diff@84ada11295b9a1e1da7325af4e45e2db9f775175
         with:
           report: gates_report_brillig.json
           header: |
@@ -170,7 +170,7 @@ jobs:
 
       - name: Compare Brillig execution reports
         id: brillig_execution_diff
-        uses: noir-lang/noir-gates-diff@d88f7523b013b9edd3f31c5cfddaef87a3fe1b48
+        uses: noir-lang/noir-gates-diff@84ada11295b9a1e1da7325af4e45e2db9f775175
         with:
           report: gates_report_brillig_execution.json
           header: |
@@ -225,8 +225,8 @@ jobs:
           retention-days: 3
           overwrite: true
 
-  generate_compilation_report:
-    name: Compilation time
+  generate_compilation_and_execution_report:
+    name: Compilation and execution time
     needs: [build-nargo]
     runs-on: ubuntu-22.04
     permissions:
@@ -252,35 +252,48 @@ jobs:
       - name: Generate Compilation report
         working-directory: ./test_programs
         run: |
-          ./compilation_report.sh
-          cat compilation_report.json
+          ./compilation_report.sh 0 1
           mv compilation_report.json ../compilation_report.json
 
+      - name: Generate Execution report
+        working-directory: ./test_programs
+        run: |
+          ./execution_report.sh 0 1
+          mv execution_report.json ../execution_report.json
+
       - name: Upload compilation report
         uses: actions/upload-artifact@v4
         with:
           name: in_progress_compilation_report
           path: compilation_report.json
           retention-days: 3
           overwrite: true
+
+      - name: Upload execution report
+        uses: actions/upload-artifact@v4
+        with:
+          name: in_progress_execution_report
+          path: execution_report.json
+          retention-days: 3
+          overwrite: true
 
-  external_repo_compilation_report:
+  external_repo_compilation_and_execution_report:
     needs: [build-nargo]
     runs-on: ubuntu-latest
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
         include:
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: true }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root, take_average: true }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, take_average: true }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, take_average: true  }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, take_average: true }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
 
-    name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+    name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }}
     steps:
       - name: Download nargo binary
         uses: actions/download-artifact@v4
@@ -302,6 +315,13 @@ jobs:
           sparse-checkout: |
             test_programs/compilation_report.sh
           sparse-checkout-cone-mode: false
+
+      - uses: actions/checkout@v4
+        with:
+          path: scripts
+          sparse-checkout: |
+            test_programs/execution_report.sh
+          sparse-checkout-cone-mode: false
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -310,34 +330,104 @@ jobs:
           path: test-repo
           ref: ${{ matrix.project.ref }}
 
-      - name: Generate compilation report
+      - name: Generate compilation report without averages
         working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.take_average }}
         run: |
           mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
           chmod +x ./compilation_report.sh
           ./compilation_report.sh 1
+
+      - name: Generate execution report
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.is_library }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          ./execution_report.sh 1
+
+      - name: Generate compilation report with averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
+          chmod +x ./compilation_report.sh
+          ./compilation_report.sh 1 1
+
+      - name: Generate execution report without averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.is_library && !matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          ./execution_report.sh 1
+
+      - name: Generate execution report with averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.is_library && matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          ./execution_report.sh 1 1
+
+      - name: Generate compilation report with averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
+          chmod +x ./compilation_report.sh
+          ./compilation_report.sh 1 1
+
+      - name: Generate execution report without averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.is_library && !matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          ./execution_report.sh 1
+
+      - name: Generate execution report with averages
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        if: ${{ !matrix.project.is_library && matrix.project.take_average }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          ./execution_report.sh 1 1
 
       - name: Move compilation report
-        id: report
+        id: compilation_report
         shell: bash
         run: |
           PACKAGE_NAME=${{ matrix.project.path }}
           PACKAGE_NAME=$(basename $PACKAGE_NAME)
           mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
           echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
 
+      - name: Move execution report
+        id: execution_report
+        shell: bash
+        if: ${{ !matrix.project.is_library }}
+        run: |
+          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=$(basename $PACKAGE_NAME)
+          mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json
+          echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
+
       - name: Upload compilation report
         uses: actions/upload-artifact@v4
         with:
-          name: compilation_report_${{ steps.report.outputs.compilation_report_name }}
-          path: compilation_report_${{ steps.report.outputs.compilation_report_name }}.json
+          name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}
+          path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json
+          retention-days: 3
+          overwrite: true
+
+      - name: Upload execution report
+        uses: actions/upload-artifact@v4
+        with:
+          name: execution_report_${{ steps.execution_report.outputs.execution_report_name }}
+          path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json
           retention-days: 3
           overwrite: true
 
   upload_compilation_report:
     name: Upload compilation report 
-    needs: [generate_compilation_report, external_repo_compilation_report]
-    # We want this job to run even if one variation of the matrix in `external_repo_compilation_report` fails
+    needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails
     if: always() 
     runs-on: ubuntu-latest
     permissions:
@@ -364,7 +454,7 @@ jobs:
 
       - name: Parse compilation report
         id: compilation_report
-        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
+        uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4
         with:
           report: compilation_report.json
           header: |
@@ -477,7 +567,7 @@ jobs:
 
       - name: Parse memory report
         id: memory_report
-        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
+        uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4
         with:
           report: memory_report.json
           header: |
@@ -491,48 +581,47 @@ jobs:
           header: memory
           message: ${{ steps.memory_report.outputs.markdown }}
 
-  generate_compilation_report:
-    name: Compilation time
-    needs: [build-nargo]
-    runs-on: ubuntu-22.04
+  upload_execution_report:
+    name: Upload execution report 
+    needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails
+    if: always() 
+    runs-on: ubuntu-latest
     permissions:
       pull-requests: write
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download nargo binary
+      - name: Download initial execution report
         uses: actions/download-artifact@v4
         with:
-          name: nargo
-          path: ./nargo
+          name: in_progress_execution_report
 
-      - name: Set nargo on PATH
-        run: |
-          nargo_binary="${{ github.workspace }}/nargo/nargo"
-          chmod +x $nargo_binary
-          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
-          export PATH="$PATH:$(dirname $nargo_binary)"
-          nargo -V
+      - name: Download matrix execution reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: execution_report_*
+          path: ./reports
 
-      - name: Generate Compilation report
-        working-directory: ./test_programs
+      - name: Merge execution reports using jq
         run: |
-          ./compilation_report.sh
-          mv compilation_report.json ../compilation_report.json
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh execution_report
 
-      - name: Parse compilation report
-        id: compilation_report
-        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
+      - name: Parse execution report
+        id: execution_report
+        uses: noir-lang/noir-bench-report@e408e131e96c3615b4f820d7d642360fb4d6e2f4
         with:
-          report: compilation_report.json
+          report: execution_report.json
           header: |
-            # Compilation Report
-          memory_report: false
+            # Execution Report
+          execution_report: true
 
       - name: Add memory report to sticky comment
         if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
         uses: marocchino/sticky-pull-request-comment@v2
         with:
-          header: compilation
-          message: ${{ steps.compilation_report.outputs.markdown }}
+          header: execution_time
+          message: ${{ steps.execution_report.outputs.markdown }}
+