csv

.github/workflows/eval.yml

@@ -0,0 +1,44 @@
+name: Evaluate Performance
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # Runs at 00:00 UTC every day
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: self-hosted
+    if: "! contains(toJSON(github.event.commits.*.message), '[skip-ci]')"
+    env:
+      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+
+      - name: Set up git private repo access
+        run: |
+          git config --global url."https://${{ secrets.PRIVATE_PULL_TOKEN }}@github.com/".insteadOf ssh://git@github.com
+          git config --global url."https://${{ secrets.PRIVATE_PULL_TOKEN }}@github.com".insteadOf https://github.com
+
+      - name: Install nightly toolchain
+        id: rustc-toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2024-01-25
+          override: true
+
+      - name: Install SP1 toolchain
+        run: |
+          curl -L https://sp1.succinct.xyz | bash
+          sp1up
+
+      - name: Build and Run Evaluation
+        run: |
+          ./eval.sh
+
+      - name: Upload Benchmark as Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: benchmark-results
+          path: benchmark.csv

.gitignore

@@ -13,4 +13,7 @@ pgo-data.profdata
 
 # Proofs
 **/proof-with-pis.json
-**/proof-with-io.json
+**/proof-with-io.json
+
+# Benchmark
+benchmark.csv

eval.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e
+
+declare -a programs=("fibonacci" "ssz-withdrawals" "tendermint")
+declare -a hash_functions=("poseidon" "blake3" "keccak256")
+declare -a shard_sizes=("262144" "524288" "1048576" "2097152" "4194304")
+
+root_directory=$(pwd)
+
+benchmark_path="${root_directory}/benchmark.csv"
+
+for program in "${programs[@]}"; do
+    echo "Processing program: $program"
+
+    program_directory="${root_directory}/examples/$program/program"
+    if [ ! -d "$program_directory" ]; then
+        echo "Program directory $program_directory not found!"
+        continue
+    fi
+
+    cd "$program_directory"
+
+    if ! RUSTFLAGS="-C passes=loweratomic -C link-arg=-Ttext=0x00200800 -C panic=abort" \
+        CARGO_NET_GIT_FETCH_WITH_CLI=true \
+        cargo prove build; then
+        echo "Failed to build $program, skipping..."
+        continue
+    fi
+
+    echo "Building $program done"
+    elf_path="${program_directory}/elf/riscv32im-succinct-zkvm-elf"
+
+    cd "${root_directory}/eval"
+    for hash_fn in "${hash_functions[@]}"; do
+        for shard_size in "${shard_sizes[@]}"; do
+            echo "Running $program with hash function $hash_fn and shard size $shard_size"
+            if ! CARGO_NET_GIT_FETCH_WITH_CLI=true RUSTFLAGS='-C target-cpu=native' cargo run -p eval --release -- \
+                --program $program --hashfn $hash_fn --shard-size $shard_size --benchmark-path "$benchmark_path" --elf-path "$elf_path"; then
+                echo "Error running evaluation for $program with hash function $hash_fn and shard size $shard_size"
+            fi
+        done
+    done
+
+    cd "$root_directory"
+done