Merge branch 'main' into partial_arc_program

.github/workflows/hyperfine.yml

@@ -0,0 +1,199 @@
+name: Hyperfine Benchmark
+
+on:
+  pull_request:
+    branches: [ '*' ]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build-programs:
+    strategy:
+      matrix:
+        branch: [ base, head ]
+    name: Build Cairo programs for ${{ matrix.branch }}
+    runs-on: ubuntu-22.04
+    outputs:
+      benchmark-hashes-base: ${{ steps.export-hashes.outputs.benchmark-hashes-base }}
+      benchmark-hashes-head: ${{ steps.export-hashes.outputs.benchmark-hashes-head }}
+    steps:
+      - name: Checkout base commit
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request[matrix.branch].sha }}
+
+      - name: Fetch from cache
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: base_programs/*.json
+          key: benchmarks-${{ hashFiles( 'cairo_programs/benchmarks/*.cairo' ) }}
+          restore-keys: benchmarks-
+
+      - name: Install Python
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install Cairo compiler
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: pip install -r requirements.txt
+
+      - name: Build programs
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: |
+          make -j cairo_bench_programs
+          mkdir -p ${{ matrix.branch }}_programs
+          cp cairo_programs/benchmarks/*.json ${{ matrix.branch }}_programs
+
+      - name: Export benchmark hashes
+        id: export-hashes
+        run:
+          echo "benchmark-hashes-${{ matrix.branch }}=${{ hashFiles( 'cairo_programs/benchmarks/*.cairo' ) }}" >> "$GITHUB_OUTPUT"
+
+
+  build-binaries:
+    strategy:
+      matrix:
+        branch: [ base, head ]
+    name: Build cairo-vm-cli for ${{ matrix.branch }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Populate cache
+        uses: actions/cache@v3
+        id: cache 
+        with:
+          path: bin/cairo-vm-cli-${{ matrix.branch }}
+          key: binary-${{ github.event.pull_request[matrix.branch].sha }}
+
+      - name: Install Rust
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.66.1
+
+      - name: Checkout
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request[matrix.branch].sha }}
+
+      - name: Fetch Rust cache
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        uses: Swatinem/rust-cache@v2
+
+      - name: Build binary
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: |
+          cargo b --release -p cairo-vm-cli
+          mkdir bin
+          cp target/release/cairo-vm-cli bin/cairo-vm-cli-${{ matrix.branch }}
+
+
+  run-hyperfine:
+    strategy:
+      matrix:
+        program_state: [ modified, unmodified ]
+    name: Run benchmarks for ${{ matrix.program_state }} programs
+    needs: [ build-programs, build-binaries ]
+    runs-on: ubuntu-22.04
+    steps:
+    - name: Install Hyperfine
+      uses: taiki-e/install-action@v2
+      with:
+        tool: hyperfine@1.16
+
+    - name: Fetch base binary
+      uses: actions/cache/restore@v3
+      with:
+        path: bin/cairo-vm-cli-base
+        key: binary-${{ github.event.pull_request.base.sha }}
+
+    - name: Fetch HEAD binary
+      uses: actions/cache/restore@v3
+      with:
+        path: bin/cairo-vm-cli-head
+        key: binary-${{ github.event.pull_request.head.sha }}
+
+    - name: Fetch base programs
+      uses: actions/cache/restore@v3
+      with:
+        path: base_programs/*.json
+        key: benchmarks-${{ needs.build-programs.outputs.benchmark-hashes-base }}
+
+    - name: Fetch head programs
+      uses: actions/cache/restore@v3
+      with:
+        path: head_programs/*.json
+        key: benchmarks-${{ needs.build-programs.outputs.benchmark-hashes-head }}
+
+    - name: Benchmark ${{ matrix.program_state }} programs
+      id: run-benchmarks
+      run: |
+        mkdir target_programs
+        if [ 'modified' = ${{ matrix.program_state }} ]; then
+          BINS=head
+          for f in head_programs/*.json; do
+            # Only run new or modified benchmarks
+            if !cmp -s ${f/head/base} $f; then
+              cp $f target_programs/
+            fi
+          done
+        else
+          BINS="base,head"
+          for f in base_programs/*.json; do
+            # Only run unmodified benchmarks
+            if cmp -s ${f/base/head} $f; then
+              cp $f target_programs/
+            fi
+          done
+        fi
+        find target_programs -name '*.json' -exec basename -s .json '{}' '+' | \
+        sort | xargs -P 2 -I '{program}' \
+        hyperfine -N -r 10 --export-markdown "target_programs/{program}.md" \
+        -L bin "$BINS" -n "{bin} {program}" \
+        -s "cat ./bin/cairo-vm-cli-{bin} target_programs/{program}.json" \
+        "./bin/cairo-vm-cli-{bin} --layout starknet_with_keccak \
+         --memory_file /dev/null --trace_file /dev/null target_programs/{program}.json"
+        echo "benchmark_count=$(ls target_programs/*.md | wc -l)" >> $GITHUB_OUTPUT
+
+    - name: Print tables
+      if: steps.run-benchmarks.outputs.benchmark_count != 0
+      run: |
+        {
+          echo "Benchmark Results for ${{ matrix.program_state }} programs :rocket:"
+          for f in target_programs/*.md; do
+            echo
+            cat $f
+          done
+        } | tee -a comment_body.md
+
+    - name: Find comment
+      if: ${{ steps.run-benchmarks.outputs.benchmark_count != 0 }}
+      uses: peter-evans/find-comment@v2
+      id: fc
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: Benchmark Results for ${{ matrix.program_state }} programs
+
+    - name: Create comment
+      if: steps.fc.outputs.comment-id == '' && steps.run-benchmarks.outputs.benchmark_count != 0
+      uses: peter-evans/create-or-update-comment@v3
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        body-path: comment_body.md
+
+    - name: Update comment
+      if: steps.fc.outputs.comment-id != '' && steps.run-benchmarks.outputs.benchmark_count != 0
+      uses: peter-evans/create-or-update-comment@v3
+      with:
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        body-path: comment_body.md
+        edit-mode: replace

.github/workflows/iai_pr.yml

@@ -1,9 +1,5 @@
 name: benchmark_pr 
 
-on:
-  pull_request:
-    branches: [ '*' ]
-
 jobs:
   fetch-iai-results:
     runs-on: ubuntu-22.04

.gitignore

@@ -7,3 +7,6 @@
 **/*.memory
 **/*.swp
 bench/results
+.python-version
+cairo-rs-env/*
+cairo-rs-pypy-env/*