chore(ci): Execution time report

.github/workflows/reports.yml

@@ -225,8 +225,8 @@ jobs:
           retention-days: 3
           overwrite: true
 
-  generate_compilation_report:
-    name: Compilation time
+  generate_compilation_and_execution_report:
+    name: Compilation and execution time
     needs: [build-nargo]
     runs-on: ubuntu-22.04
     permissions:
@@ -253,34 +253,47 @@ jobs:
         working-directory: ./test_programs
         run: |
           ./compilation_report.sh
-          cat compilation_report.json
           mv compilation_report.json ../compilation_report.json
 
+      - name: Generate Execution report
+        working-directory: ./test_programs
+        run: |
+          ./execution_report.sh
+          mv execution_report.json ../execution_report.json
+
       - name: Upload compilation report
         uses: actions/upload-artifact@v4
         with:
           name: in_progress_compilation_report
           path: compilation_report.json
           retention-days: 3
           overwrite: true
+
+      - name: Upload execution report
+        uses: actions/upload-artifact@v4
+        with:
+          name: in_progress_execution_report
+          path: execution_report.json
+          retention-days: 3
+          overwrite: true
 
-  external_repo_compilation_report:
+  external_repo_compilation_and_execution_report:
     needs: [build-nargo]
     runs-on: ubuntu-latest
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
         include:
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, is_library: "true" }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
 
-    name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+    name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }}
     steps:
       - name: Download nargo binary
         uses: actions/download-artifact@v4
@@ -302,6 +315,13 @@ jobs:
           sparse-checkout: |
             test_programs/compilation_report.sh
           sparse-checkout-cone-mode: false
+
+      - uses: actions/checkout@v4
+        with:
+          path: scripts
+          sparse-checkout: |
+            test_programs/execution_report.sh
+          sparse-checkout-cone-mode: false
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -316,28 +336,57 @@ jobs:
           mv /home/runner/work/noir/noir/scripts/test_programs/compilation_report.sh ./compilation_report.sh
           chmod +x ./compilation_report.sh
           ./compilation_report.sh 1
+
+      - name: Generate execution report
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/execution_report.sh ./execution_report.sh
+          IS_LIBRARY=${{ matrix.project.is_library }}
+          if [[ "$IS_LIBRARY" != "true" ]]; then
+            ./execution_report.sh 1
+          fi
 
       - name: Move compilation report
-        id: report
+        id: compilation_report
         shell: bash
         run: |
           PACKAGE_NAME=${{ matrix.project.path }}
           PACKAGE_NAME=$(basename $PACKAGE_NAME)
           mv ./test-repo/${{ matrix.project.path }}/compilation_report.json ./compilation_report_$PACKAGE_NAME.json
           echo "compilation_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
 
+      - name: Move execution report
+        id: execution_report
+        shell: bash
+        run: |
+          IS_LIBRARY=${{ matrix.project.is_library }}
+          if [[ "$IS_LIBRARY" != "true" ]]; then
+            PACKAGE_NAME=${{ matrix.project.path }}
+            PACKAGE_NAME=$(basename $PACKAGE_NAME)
+            mv ./test-repo/${{ matrix.project.path }}/execution_report.json ./execution_report_$PACKAGE_NAME.json
+            echo "execution_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
+          fi
+
       - name: Upload compilation report
         uses: actions/upload-artifact@v4
         with:
-          name: compilation_report_${{ steps.report.outputs.compilation_report_name }}
-          path: compilation_report_${{ steps.report.outputs.compilation_report_name }}.json
+          name: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}
+          path: compilation_report_${{ steps.compilation_report.outputs.compilation_report_name }}.json
+          retention-days: 3
+          overwrite: true
+
+      - name: Upload execution report
+        uses: actions/upload-artifact@v4
+        with:
+          name: execution_report_${{ steps.execution_report.outputs.execution_report_name }}
+          path: execution_report_${{ steps.execution_report.outputs.execution_report_name }}.json
           retention-days: 3
           overwrite: true
 
   upload_compilation_report:
     name: Upload compilation report 
-    needs: [generate_compilation_report, external_repo_compilation_report]
-    # We want this job to run even if one variation of the matrix in `external_repo_compilation_report` fails
+    needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails
     if: always() 
     runs-on: ubuntu-latest
     permissions:
@@ -490,3 +539,48 @@ jobs:
         with:
           header: memory
           message: ${{ steps.memory_report.outputs.markdown }}
+
+  upload_execution_report:
+    name: Upload execution report 
+    needs: [generate_compilation_and_execution_report, external_repo_compilation_and_execution_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_compilation_and_execution_report` fails
+    if: always() 
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download initial execution report
+        uses: actions/download-artifact@v4
+        with:
+          name: in_progress_execution_report
+
+      - name: Download matrix execution reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: execution_report_*
+          path: ./reports
+
+      - name: Merge execution reports using jq
+        run: |
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh execution_report
+
+      - name: Parse execution report
+        id: execution_report
+        uses: noir-lang/noir-bench-report@0954121203ee55dcda5c7397b9c669c439a20922
+        with:
+          report: execution_report.json
+          header: |
+            # Execution Report
+          execution_report: true
+
+      - name: Add memory report to sticky comment
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: execution_time
+          message: ${{ steps.execution_report.outputs.markdown }}
+

.gitignore

@@ -36,6 +36,7 @@ gates_report.json
 gates_report_brillig.json
 gates_report_brillig_execution.json
 compilation_report.json
+execution_report.json
 
 # Github Actions scratch space
 # This gives a location to download artifacts into the repository in CI without making git dirty.

test_programs/execution_report.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -e
+
+current_dir=$(pwd)
+base_path="$current_dir/execution_success"
+
+# Tests to be profiled for execution report
+tests_to_profile=("sha256_regression" "regression_4709" "ram_blowup_regression")
+
+echo "{\"execution_reports\": [ " > $current_dir/execution_report.json
+
+# If there is an argument that means we want to generate a report for only the current directory
+if [ "$#" -ne 0 ]; then
+  base_path="$current_dir"
+  tests_to_profile=(".")
+fi
+
+ITER="1"
+NUM_ARTIFACTS=${#tests_to_profile[@]}
+
+for dir in ${tests_to_profile[@]}; do 
+    if [[ " ${excluded_dirs[@]} " =~ " ${dir} " ]]; then
+      continue
+    fi
+
+    if [[ ${CI-false} = "true" ]] && [[ " ${ci_excluded_dirs[@]} " =~ " ${dir} " ]]; then
+      continue
+    fi
+
+    cd $base_path/$dir
+
+    # The default package to run is the supplied list hardcoded at the top of the script
+    PACKAGE_NAME=$dir
+    # Otherwise default to the current directory as the package we want to run
+    if [ "$#" -ne 0 ]; then
+      PACKAGE_NAME=$(basename $current_dir)
+    fi
+
+    # Check whether a compilation artifact exists. 
+    # Any programs part of this benchmark should already be compiled.
+    # We want to make sure that compilation time is not included in the execution time.
+    if [ ! -e ./target/*.json ]; then
+      echo "Missing compilation artifact for $PACKAGE_NAME"
+      exit 1
+    fi
+
+    COMPILE_TIME=$((time nargo execute --silence-warnings) 2>&1 | grep real | grep -oE '[0-9]+m[0-9]+.[0-9]+s')
+    echo -e " {\n    \"artifact_name\":\"$PACKAGE_NAME\",\n    \"time\":\"$COMPILE_TIME\"" >> $current_dir/execution_report.json
+
+    if (($ITER == $NUM_ARTIFACTS)); then
+        echo "}" >> $current_dir/execution_report.json
+    else 
+        echo "}, " >> $current_dir/execution_report.json
+    fi
+
+    ITER=$(( $ITER + 1 ))
+done
+
+echo "]}" >> $current_dir/execution_report.json