diff --git a/.github/workflows/build-guest-release.yml b/.github/workflows/build-guest-release.yml index cb98599a..efcd79b4 100644 --- a/.github/workflows/build-guest-release.yml +++ b/.github/workflows/build-guest-release.yml @@ -3,22 +3,22 @@ name: Build ZKVM-Prover Guest on: push: tags: - - ** + - 'v0.[0-9]+.[0-9]+' jobs: build-guest: runs-on: ubuntu-latest env: SCROLL_ZKVM_VERSION: ${{ github.ref_name }} - DIR_OUTPUT: releases/dev + DIR_OUTPUT: releases AWS_REGION: us-west-2 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Install xxd - run: sudo apt-get update && sudo apt-get install -y vim-common + - name: Install xxd and jq + run: sudo apt-get update && sudo apt-get install -y vim-common jq - name: Install Rust nightly toolchain uses: actions-rs/toolchain@v1 @@ -41,9 +41,7 @@ jobs: run: make build-guest - name: Prepare release - run: | - # checksum for assets - find $DIR_OUTPUT -type f ! -name sha256sums.txt -exec sha256sum {} \; > $DIR_OUTPUT/sha256sums.txt + run: sh release.sh - id: install-aws-cli uses: unfor19/install-aws-cli-action@v1 @@ -61,4 +59,4 @@ jobs: - name: Upload releases to S3 run: | - aws s3 cp $DIR_OUTPUT s3://circuit-release/scroll-zkvm/${{ github.ref_name }} --recursive + aws s3 cp $DIR_OUTPUT s3://circuit-release/scroll-zkvm --recursive diff --git a/Cargo.lock b/Cargo.lock index d5f3c265..ab1435c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6612,7 +6612,6 @@ dependencies = [ name = "scroll-zkvm-build-guest" version = "0.5.0" dependencies = [ - "base64", "bitcode", "cargo_metadata 0.19.2", "dotenv", @@ -6682,13 +6681,13 @@ name = "scroll-zkvm-integration" version = "0.5.0" dependencies = [ "alloy-primitives", - "base64", "cargo_metadata 0.20.0", "chrono", "eyre", "ff 0.13.1", "glob", "halo2curves-axiom 0.7.0", + "hex", "metrics-tracing-context", "once_cell", "openvm-build", diff --git a/Cargo.toml b/Cargo.toml index d8a88434..f83d453d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,6 +77,7 @@ eyre = "0.6" halo2curves-axiom = { version = "0.5.3" } itertools = "0.14" hex-literal = { version = "0.4.1", default-features = false } +hex = "0.4" metrics = "0.23.0" metrics-util = "0.17" metrics-tracing-context = "0.16.0" diff --git a/Dockerfile b/Dockerfile index 662f6232..094a5cbd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,3 @@ RUN wget https://github.com/ethereum/solc-bin/raw/refs/heads/gh-pages/linux-amd6 COPY . . -ENTRYPOINT ["/app/build-guest-actions-entrypoint.sh"] diff --git a/README.md b/README.md index 670a5568..09bc08e5 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,16 @@ $ make test-e2e-bundle *Note*: Configure `RUST_LOG=debug` for debug logs or `RUST_LOG=none,scroll_zkvm_prover=debug` for logs specifically from the `scroll-zkvm-prover` crate. +## Release of prover circuits + +All apps of circuits are uploaded into aws s3 storage, and can be download via following urls: + +`/scroll-zkvm/releases///` + ++ Current the url for s3 storage is `https://circuit-release.s3.us-west-2.amazonaws.com` ++ The fork name can be read via [release-fork](./release-fork) file ++ The circuit app has to be accessed by specifying its proof type (chunk/batch/bundle) and the vk of the circuit. + ## Usage of Prover API ### Dependency diff --git a/build-guest-actions-entrypoint.sh b/build-guest-actions-entrypoint.sh deleted file mode 100755 index 92e92d79..00000000 --- a/build-guest-actions-entrypoint.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -mkdir -p releases -rm -rf releases/dev - -# run crates/build-guest -cargo run --release -p scroll-zkvm-build-guest diff --git a/build-guest.sh b/build-guest.sh index 6e805503..03a2b2b1 100755 --- a/build-guest.sh +++ b/build-guest.sh @@ -2,7 +2,7 @@ set -ex mkdir -p releases -rm -rf releases/dev +rm -rf releases/* [ -f "crates/build-guest/.env" ] && . crates/build-guest/.env @@ -14,8 +14,24 @@ fi # build docker image docker build --platform linux/amd64 -t build-guest:local . +# cleanup function +cleanup() { + + if [ -f ./build-guest.cid ]; then + docker rm -f $(cat ./build-guest.cid) 2>/dev/null || true + fi + rm -f ./build-guest.cid + +} + +# set trap to cleanup on exit +trap cleanup EXIT + # run docker image -docker run --cidfile ./build-guest.cid --platform linux/amd64 -e FEATURE=${FEATURE} build-guest:local +docker run --cidfile ./build-guest.cid --platform linux/amd64\ + -e BUILD_STAGES=${BUILD_STAGES}\ + build-guest:local\ + cargo run --release -p scroll-zkvm-build-guest container_id=$(cat ./build-guest.cid) if [ -n "$(echo ${BUILD_STAGES} | grep stage1)" ]; then @@ -41,12 +57,5 @@ if [ -n "$(echo ${BUILD_STAGES} | grep stage3)" ]; then done fi - # copy release files from container to local -mkdir -p releases -rm -rf releases/dev docker cp ${container_id}:/app/releases/dev releases/dev - -# remove docker container -docker rm ${container_id} -rm ./build-guest.cid \ No newline at end of file diff --git a/crates/build-guest/Cargo.toml b/crates/build-guest/Cargo.toml index 7b3bae87..3fca0e21 100644 --- a/crates/build-guest/Cargo.toml +++ b/crates/build-guest/Cargo.toml @@ -21,13 +21,13 @@ openvm-circuit = { workspace = true } tracing.workspace = true tracing-subscriber.workspace = true -base64.workspace = true bitcode.workspace = true halo2curves-axiom.workspace = true eyre.workspace = true p3-field.workspace = true serde.workspace = true hex-literal.workspace = true +hex.workspace = true serde_json.workspace = true serde_with.workspace = true snark-verifier-sdk.workspace = true @@ -35,7 +35,6 @@ toml.workspace = true cargo_metadata = "0.19.1" dotenv = "0.15.0" -hex = "0.4.3" [features] default = [] diff --git a/crates/build-guest/src/main.rs b/crates/build-guest/src/main.rs index f00ff3e1..8a696b65 100644 --- a/crates/build-guest/src/main.rs +++ b/crates/build-guest/src/main.rs @@ -291,14 +291,13 @@ fn run_stage4_dump_vk_json( if let (Some(exe), Some(leaf)) = (exe_commitments.get(circuit), leaf_commitments.get(circuit)) { - let app_vk = scroll_zkvm_types::types_agg::ProgramCommitment { + use scroll_zkvm_types::{types_agg::ProgramCommitment, utils::serialize_vk}; + let app_vk = serialize_vk::serialize(&ProgramCommitment { exe: *exe, vm: *leaf, - } - .serialize(); + }); - use base64::{Engine, prelude::BASE64_STANDARD}; - let app_vk = BASE64_STANDARD.encode(app_vk); + let app_vk = hex::encode(&app_vk); println!("{circuit}: {app_vk}"); app_vk } else { diff --git a/crates/integration/Cargo.toml b/crates/integration/Cargo.toml index 3b8cd56e..a32dca17 100644 --- a/crates/integration/Cargo.toml +++ b/crates/integration/Cargo.toml @@ -23,13 +23,13 @@ openvm-native-transpiler = { workspace = true, default-features = false } openvm-transpiler = { workspace = true, default-features = false } alloy-primitives.workspace = true -base64.workspace = true cargo_metadata.workspace = true eyre.workspace = true rayon.workspace = true serde.workspace = true vm-zstd = { workspace = true, features = ["zstd"] } +hex.workspace = true chrono = "0.4" ff = "0.13" glob = "0.3" diff --git a/crates/integration/src/testers/bundle.rs b/crates/integration/src/testers/bundle.rs index 76cb5255..2ef0b3c3 100644 --- a/crates/integration/src/testers/bundle.rs +++ b/crates/integration/src/testers/bundle.rs @@ -3,6 +3,7 @@ use scroll_zkvm_types::{ bundle::{BundleInfo, BundleWitness}, proof::ProofEnum, public_inputs::ForkName, + utils::serialize_vk, }; // Only related to hardcoded commitments. Can be refactored later. @@ -98,13 +99,12 @@ impl BundleTaskGenerator { fn calculate_bundle_witness(&self) -> eyre::Result { use scroll_zkvm_types::{ - public_inputs::MultiVersionPublicInputs, - types_agg::{AggregationInput, ProgramCommitment}, + public_inputs::MultiVersionPublicInputs, types_agg::AggregationInput, }; let fork_name = testing_hardfork(); let vk = BatchProverTester::load_prover(false)?.get_app_vk(); - let commitment = ProgramCommitment::deserialize(&vk); + let commitment = serialize_vk::deserialize(&vk); let mut batch_proofs = Vec::new(); let mut batch_infos: Vec = Vec::new(); diff --git a/crates/integration/src/utils/mod.rs b/crates/integration/src/utils/mod.rs index 9232d1f6..b66c3abf 100644 --- a/crates/integration/src/utils/mod.rs +++ b/crates/integration/src/utils/mod.rs @@ -11,8 +11,8 @@ use scroll_zkvm_types::{ bundle::{BundleInfo, BundleWitness}, chunk::{ChunkInfo, ChunkWitness}, public_inputs::{ForkName, MultiVersionPublicInputs}, - types_agg::{AggregationInput, ProgramCommitment}, - utils::{keccak256, point_eval}, + types_agg::AggregationInput, + utils::{keccak256, point_eval, serialize_vk}, }; use vm_zstd::zstd_encode; @@ -313,7 +313,7 @@ pub fn build_batch_witnesses( }; let fork_name = testing_hardfork(); - let commitment = ProgramCommitment::deserialize(prover_vk); + let commitment = serialize_vk::deserialize(prover_vk); let chunk_proofs = chunk_infos .iter() .map(|chunk_info| { @@ -367,7 +367,7 @@ fn test_build_and_parse_batch_task() -> eyre::Result<()> { let task_wit = build_batch_witnesses( &witnesses, - &ProgramCommitment::default().serialize(), + &[0u8; 64], // use a default, all zero vk Default::default(), )?; diff --git a/crates/integration/tests/bundle_circuit.rs b/crates/integration/tests/bundle_circuit.rs index e3cf7bc9..dc41514e 100644 --- a/crates/integration/tests/bundle_circuit.rs +++ b/crates/integration/tests/bundle_circuit.rs @@ -53,9 +53,7 @@ fn print_vks() -> eyre::Result<()> { ..Default::default() }; - use base64::{Engine, prelude::BASE64_STANDARD}; - let app_vk = - BASE64_STANDARD.encode(Prover::setup(config, false, None).unwrap().get_app_vk()); + let app_vk = hex::encode(Prover::setup(config, false, None).unwrap().get_app_vk()); println!("{circuit}: {app_vk}"); app_vk }); diff --git a/crates/prover/Cargo.toml b/crates/prover/Cargo.toml index 496cb55d..8a65eede 100644 --- a/crates/prover/Cargo.toml +++ b/crates/prover/Cargo.toml @@ -14,6 +14,7 @@ scroll-zkvm-types-batch.workspace = true alloy-primitives = { workspace = true, features = ["tiny-keccak"] } rkyv.workspace = true bincode_v1.workspace = true +hex.workspace = true sbv-primitives = { workspace = true } tracing.workspace = true metrics.workspace = true @@ -30,7 +31,6 @@ openvm-stark-sdk = { workspace = true, default-features = false } base64 = "0.22" git-version = "0.3.5" -hex = "0.4" munge = "=0.4.1" once_cell = "1.20" serde = "1.0" diff --git a/crates/prover/src/prover/mod.rs b/crates/prover/src/prover/mod.rs index 8e18d475..3d1ce340 100644 --- a/crates/prover/src/prover/mod.rs +++ b/crates/prover/src/prover/mod.rs @@ -10,7 +10,7 @@ use openvm_sdk::{ config::{AggConfig, SdkVmConfig}, keygen::{AggProvingKey, AppProvingKey}, }; -use scroll_zkvm_types::{proof::OpenVmEvmProof, types_agg::ProgramCommitment}; +use scroll_zkvm_types::{proof::OpenVmEvmProof, types_agg::ProgramCommitment, utils::serialize_vk}; use scroll_zkvm_verifier::verifier::{AGG_STARK_PROVING_KEY, UniversalVerifier}; use tracing::instrument; @@ -115,7 +115,7 @@ impl Prover { /// Pick up loaded app commit as "vk" in proof, to distinguish from which circuit the proof comes pub fn get_app_vk(&self) -> Vec { - self.get_app_commitment().serialize() + serialize_vk::serialize(&self.get_app_commitment()) } /// Pick up the actual vk (serialized) for evm proof, would be empty if prover @@ -253,7 +253,6 @@ impl Prover { stdin, ) .map_err(|e| Error::GenProof(e.to_string()))?; - let comm = self.get_app_commitment(); let proof = StarkProof { proofs: vec![proof.proof], public_values: proof.user_public_values, @@ -261,7 +260,7 @@ impl Prover { //vm_commitment: comm.vm, }; tracing::info!("verifing stark proof"); - UniversalVerifier::verify_stark_proof(&proof, &comm.serialize()) + UniversalVerifier::verify_stark_proof(&proof, &self.get_app_vk()) .map_err(|e| Error::VerifyProof(e.to_string()))?; tracing::info!("verifing stark proof done"); Ok(proof) diff --git a/crates/types/src/utils.rs b/crates/types/src/utils.rs index 48c04264..4e0af5be 100644 --- a/crates/types/src/utils.rs +++ b/crates/types/src/utils.rs @@ -38,6 +38,33 @@ pub mod as_base64 { } } +pub mod serialize_vk { + use types_base::aggregation::ProgramCommitment; + pub fn deserialize(commitment_bytes: &[u8]) -> ProgramCommitment { + let mut exe: [u32; 8] = [0; 8]; + for (i, bytes4) in commitment_bytes[..32].chunks(4).enumerate() { + let bytes: [u8; 4] = bytes4.try_into().unwrap(); + exe[i] = u32::from_le_bytes(bytes); + } + + let mut vm: [u32; 8] = [0; 8]; + for (i, bytes4) in commitment_bytes[32..].chunks(4).enumerate() { + let bytes: [u8; 4] = bytes4.try_into().unwrap(); + vm[i] = u32::from_le_bytes(bytes); + } + ProgramCommitment { exe, vm } + } + + pub fn serialize(commit: &ProgramCommitment) -> Vec { + commit + .exe + .iter() + .chain(commit.vm.iter()) + .flat_map(|u| u.to_le_bytes().into_iter()) + .collect() + } +} + pub mod point_eval { use super::sha256_rv32; use c_kzg; diff --git a/crates/verifier/src/verifier.rs b/crates/verifier/src/verifier.rs index 6971357d..fb2d1d14 100644 --- a/crates/verifier/src/verifier.rs +++ b/crates/verifier/src/verifier.rs @@ -1,7 +1,7 @@ use once_cell::sync::Lazy; use openvm_sdk::{Sdk, commit::CommitBytes, config::AggStarkConfig, keygen::AggStarkProvingKey}; use scroll_zkvm_types::proof::OpenVmEvmProof; -use scroll_zkvm_types::{proof::StarkProof, types_agg::ProgramCommitment}; +use scroll_zkvm_types::{proof::StarkProof, utils::serialize_vk}; use std::path::Path; /// Proving key for STARK aggregation. Primarily used to aggregate @@ -21,7 +21,7 @@ impl UniversalVerifier { } pub fn verify_stark_proof(stark_proof: &StarkProof, vk: &[u8]) -> eyre::Result<()> { - let prog_commit = ProgramCommitment::deserialize(vk); + let prog_commit = serialize_vk::deserialize(vk); /* if stark_proof.exe_commitment != prog_commit.exe { @@ -51,7 +51,7 @@ impl UniversalVerifier { } pub fn verify_evm_proof(&self, evm_proof: &OpenVmEvmProof, vk: &[u8]) -> eyre::Result<()> { - let prog_commit = ProgramCommitment::deserialize(vk); + let prog_commit = serialize_vk::deserialize(vk); if evm_proof.app_commit.app_exe_commit.to_u32_digest() != prog_commit.exe { eyre::bail!("evm: mismatch EXE commitment"); diff --git a/release-fork b/release-fork new file mode 100644 index 00000000..e2044e89 --- /dev/null +++ b/release-fork @@ -0,0 +1 @@ +feynman \ No newline at end of file diff --git a/release.sh b/release.sh index 752af673..818e201e 100644 --- a/release.sh +++ b/release.sh @@ -1,49 +1,51 @@ #!/bin/bash +set -ue -# release version -SCROLL_ZKVM_TESTRUN_DIR=`realpath .output/bundle-tests-20250722_234931/` -SCROLL_ZKVM_VERSION=0.5.2 if [ -z "${SCROLL_ZKVM_VERSION}" ]; then echo "SCROLL_ZKVM_VERSION not set" exit 1 else - DIR_OUTPUT="releases/${SCROLL_ZKVM_VERSION}" + VERIFIER_RELEASES_DIR="releases/${SCROLL_ZKVM_VERSION}" fi -# directory to read assets from -if [ -z "${SCROLL_ZKVM_TESTRUN_DIR}" ]; then - echo "SCROLL_ZKVM_TESTRUN_DIR not set" - exit 1 -else - DIR_INPUT="${SCROLL_ZKVM_TESTRUN_DIR}" +# Read FORKNAME from release-fork file +FORKNAME=$(head -n 1 release-fork) + +DEV_DIR="releases/dev" +VK_JSON="$DEV_DIR/verifier/openVmVk.json" +RELEASES_DIR="releases/$FORKNAME" + +# Output sha256 checksums +find $DEV_DIR -type f ! -name sha256sums.txt -exec sha256sum {} \; > $DEV_DIR/sha256sums.txt + +# Check if openVmVk.json exists +if [ ! -f "$VK_JSON" ]; then + echo "Error: openVmVk.json not found in $DEV_DIR/" + exit 1 fi -# create all required directories for release -mkdir -p $DIR_OUTPUT/chunk -mkdir -p $DIR_OUTPUT/batch -mkdir -p $DIR_OUTPUT/bundle -mkdir -p $DIR_OUTPUT/verifier - -# copy chunk-program related assets -cp ./crates/circuits/chunk-circuit/openvm/app.vmexe $DIR_OUTPUT/chunk/app.vmexe -cp ./crates/circuits/chunk-circuit/openvm.toml $DIR_OUTPUT/chunk/openvm.toml - -# copy batch-program related assets -cp ./crates/circuits/batch-circuit/openvm/app.vmexe $DIR_OUTPUT/batch/app.vmexe -cp ./crates/circuits/batch-circuit/openvm.toml $DIR_OUTPUT/batch/openvm.toml - -# copy bundle-program related assets -cp ./crates/circuits/bundle-circuit/openvm/app.vmexe $DIR_OUTPUT/bundle/app.vmexe -cp ./crates/circuits/bundle-circuit/openvm.toml $DIR_OUTPUT/bundle/openvm.toml -cp ./crates/circuits/bundle-circuit/openvm/verifier.bin $DIR_OUTPUT/bundle/verifier.bin -cp ./crates/circuits/bundle-circuit/openvm/verifier.sol $DIR_OUTPUT/bundle/verifier.sol -xxd -l 32 -p ./crates/circuits/bundle-circuit/digest_1 | tr -d '\n' | awk '{gsub("%", ""); print}' > $DIR_OUTPUT/bundle/digest_1.hex -xxd -l 32 -p ./crates/circuits/bundle-circuit/digest_2 | tr -d '\n' | awk '{gsub("%", ""); print}' > $DIR_OUTPUT/bundle/digest_2.hex - -# copy verifier-only assets -#cp $DIR_INPUT/bundle/root-verifier-vm-config $DIR_OUTPUT/verifier/root-verifier-vm-config -#cp $DIR_INPUT/bundle/root-verifier-committed-exe $DIR_OUTPUT/verifier/root-verifier-committed-exe -cp ./crates/circuits/bundle-circuit/openvm/verifier.bin $DIR_OUTPUT/verifier/verifier.bin - -# upload to s3 -aws --profile default s3 cp $DIR_OUTPUT s3://circuit-release/scroll-zkvm/$DIR_OUTPUT --recursive +# Read verification keys from JSON file +chunk_vk=$(jq -r '.chunk_vk' "$VK_JSON") +batch_vk=$(jq -r '.batch_vk' "$VK_JSON") +bundle_vk=$(jq -r '.bundle_vk' "$VK_JSON") + +# Create directories and copy files +mkdir -p "$RELEASES_DIR/chunk/$chunk_vk" +mkdir -p "$RELEASES_DIR/batch/$chunk_vk" +mkdir -p "$RELEASES_DIR/bundle/$chunk_vk" + +# Copy files from releases/dev to the new directories +cp -r "$DEV_DIR/chunk"/* "$RELEASES_DIR/chunk/$chunk_vk/" +cp -r "$DEV_DIR/batch"/* "$RELEASES_DIR/batch/$chunk_vk/" +cp -r "$DEV_DIR/bundle"/* "$RELEASES_DIR/bundle/$chunk_vk/" +mkdir -p $VERIFIER_RELEASES_DIR +mv $DEV_DIR/* $VERIFIER_RELEASES_DIR + +echo "Files organized for release successfully:" +echo " chunk files -> $RELEASES_DIR/chunk/$chunk_vk" +echo " batch files -> $RELEASES_DIR/batch/$chunk_vk" +echo " bundle files -> $RELEASES_DIR/bundle/$chunk_vk" +echo " verifier files -> $VERIFIER_RELEASES_DIR" +echo " recursivly upload releases directory" + +#aws --profile default s3 cp releases s3://circuit-release/scroll-zkvm --recursive