feat: merge upstream changes (#1550)

.github/workflows/pr.yml

@@ -287,4 +287,73 @@ jobs:
         run: |
           cd examples/cycle-tracking/script
           cargo add sp1-sdk --path $GITHUB_WORKSPACE/crates/sdk
-          SP1_DEV=1 RUST_LOG=info cargo run --release
+          SP1_DEV=1 RUST_LOG=info cargo run --release
+
+  toolchain-test:
+    name: "Test toolchain installation (${{ matrix.name }})"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: "Ubuntu 24.04 (x86_64)"
+            runner: "ubuntu-24.04"
+          - name: "Ubuntu 22.04 (x86_64)"
+            runner: "ubuntu-22.04"
+          - name: "Ubuntu 20.04 (x86_64)"
+            runner: "ubuntu-20.04"
+          - name: "macOS Monterey (x86_64)"
+            runner: "macos-12"
+          - name: "macOS Ventura (x86_64)"
+            runner: "macos-13"
+          - name: "macOS Sonoma (ARM64)"
+            runner: "macos-14"
+
+    runs-on: "${{ matrix.runner }}"
+    steps:
+      - name: "Checkout source code"
+        uses: "actions/checkout@v4"
+
+      - name: "Install cargo-prove"
+        run: |
+          cargo install --locked --path ./crates/cli
+
+      - name: "Install SP1 toolchain"
+        run: |
+          cargo prove install-toolchain --token ${{ secrets.GITHUB_TOKEN }}
+
+      - name: "Create SP1 project from template"
+        run: |
+          cargo prove new hello
+
+      - name: "Build SP1 project"
+        run: |
+          cd ./hello/program
+          cargo prove build
+
+  toolchain-test-ec2:
+    name: "Test toolchain installation (${{ matrix.name }})"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # AMI from `us-east-1`
+          - name: "Debian 12 (x86_64)"
+            ec2-instance: "c5.2xlarge"
+            ami: "ami-064519b8c76274859"
+            volume: "/dev/xvda"
+          - name: "Debian 12 (ARM64)"
+            ec2-instance: "c6g.2xlarge"
+            ami: "ami-0789039e34e739d67"
+            volume: "/dev/xvda"
+    uses: "./.github/workflows/toolchain-ec2.yml"
+    with:
+      image-id: "${{ matrix.ami }}"
+      instance-type: "${{ matrix.ec2-instance }}"
+      root-volume: "${{ matrix.volume }}"
+    secrets:
+      AWS_REGION: "${{ secrets.AWS_REGION }}"
+      AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+      AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+      AWS_SUBNET_ID: "${{ secrets.AWS_SUBNET_ID }}"
+      AWS_SG_ID: "${{ secrets.AWS_SG_ID }}"
+      GH_PAT: "${{ secrets.GH_PAT }}"

.github/workflows/toolchain-ec2.yml

@@ -0,0 +1,127 @@
+name: "Toolchain installation test with EC2"
+
+on:
+  workflow_call:
+    inputs:
+      image-id:
+        required: true
+        type: "string"
+      instance-type:
+        required: true
+        type: "string"
+      root-volume:
+        required: false
+        type: "string"
+        default: "/dev/sda1"
+    secrets:
+      AWS_REGION:
+        required: true
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      AWS_SUBNET_ID:
+        required: true
+      AWS_SG_ID:
+        required: true
+      GH_PAT:
+        required: true
+
+jobs:
+  start-runner:
+    name: "Start self-hosted EC2 runner"
+    runs-on: "ubuntu-latest"
+    outputs:
+      label: "${{ steps.start-ec2-runner.outputs.label }}"
+      ec2-instance-id: "${{ steps.start-ec2-runner.outputs.ec2-instance-id }}"
+
+    steps:
+      # Use an access key for an IAM user with these permissions:
+      # - ec2:RunInstances
+      # - ec2:TerminateInstances
+      # - ec2:DescribeInstances
+      # - ec2:DescribeInstanceStatus
+      - name: "Configure AWS credentials"
+        uses: "aws-actions/configure-aws-credentials@v1"
+        with:
+          aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          aws-secret-access-key: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
+          aws-region: "${{ secrets.AWS_REGION }}"
+
+      - name: "Start EC2 runner"
+        id: "start-ec2-runner"
+        uses: "xJonathanLEI/ec2-github-runner@main"
+        with:
+          mode: "start"
+          # Must use personal access token here as `GITHUB_TOKEN` does not have access to runners.
+          # Use a fine-grained token with these permissions to at least this repository:
+          # - Administration: Read and write
+          # - Contents: Read and write
+          # - Metadata: Read-only
+          # - Workflows: Read and write
+          github-token: "${{ secrets.GH_PAT }}"
+          ec2-image-id: "${{ inputs.image-id }}"
+          ec2-instance-type: "${{ inputs.instance-type }}"
+          subnet-id: "${{ secrets.AWS_SUBNET_ID }}"
+          security-group-id: "${{ secrets.AWS_SG_ID }}"
+          storage-size: 1024
+          storage-device: "${{ inputs.root-volume }}"
+
+  toolchain-test:
+    name: "Run toolchain test"
+    runs-on: "${{ needs.start-runner.outputs.label }}"
+    needs:
+      - "start-runner"
+
+    steps:
+      # Workaround for EC2 runner missing $HOME
+      - name: "Set HOME env var"
+        run: |
+          if [ -z "$HOME" ]; then
+            echo "HOME=/home/ubuntu" >> $GITHUB_ENV
+          fi
+
+      - name: "Checkout"
+        uses: "actions/checkout@v4"
+
+      - name: "Install Rust"
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: "Install build dependencies"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential pkg-config libssl-dev
+
+      - name: "Install cargo-prove"
+        run: |
+          cargo install --locked --path ./crates/cli
+
+      - name: "Install SP1 toolchain"
+        run: |
+          cargo prove install-toolchain --token ${{ secrets.GH_PAT }}
+
+  stop-runner:
+    name: "Stop self-hosted EC2 runner"
+    runs-on: "ubuntu-latest"
+    needs:
+      - "start-runner"
+      - "toolchain-test"
+    if: ${{ always() }}
+
+    steps:
+      - name: "Configure AWS credentials"
+        uses: "aws-actions/configure-aws-credentials@v1"
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: "Stop EC2 runner"
+        uses: "xJonathanLEI/ec2-github-runner@main"
+        with:
+          mode: "stop"
+          github-token: ${{ secrets.GH_PAT }}
+          label: "${{ needs.start-runner.outputs.label }}"
+          ec2-instance-id: "${{ needs.start-runner.outputs.ec2-instance-id }}"

book/writing-programs/patched-crates.md

@@ -18,8 +18,8 @@ Under the hood, we use [precompiles](./precompiles.md) to achieve tremendous per
 | curve25519-dalek    | [sp1-patches/curve25519-dalek](https://github.com/sp1-patches/curve25519-dalek)       | ed25519 verify                                      |
 | ecdsa-core          | [sp1-patches/signatures](http://github.com/sp1-patches/signatures)                    | secp256k1 verify                                    |
 | secp256k1           | [sp1-patches/rust-secp256k1](http://github.com/sp1-patches/rust-secp256k1)            | secp256k1 verify                                    |
-| ****                | substrate-bn                                                                          | [sp1-patches/bn](https://github.com/sp1-patches/bn) | BN254 |
-| substrate-bls12_381 | [sp1-patches/bls12_381](https://github.com/sp1-patches/bls12_381)                     | bls12_381                                           |
+| substrate-bn        | [sp1-patches/bn](https://github.com/sp1-patches/bn)                                   | BN254                                               |
+| substrate-bls12_381 | [sp1-patches/bls12_381](https://github.com/sp1-patches/bls12_381)                     | BLS12-381                                           |
 
 ## Using Patched Crates
 

crates/core/machine/Cargo.toml

@@ -63,7 +63,6 @@ strum = "0.26"
 web-time = "1.1.0"
 rayon-scan = "0.1.1"
 thiserror = "1.0.63"
-num-bigint = { version = "0.4.6", default-features = false }
 rand = "0.8.5"
 bytemuck = "1.16.0"
 hashbrown = { version = "0.14.5", features = ["serde", "inline-more"] }

crates/core/machine/src/io.rs

@@ -1,6 +1,3 @@
-use crate::utils::Buffer;
-use k256::sha2::{Digest, Sha256};
-use num_bigint::BigUint;
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sp1_core_executor::SP1ReduceProof;
 use sp1_stark::{baby_bear_poseidon2::BabyBearPoseidon2, StarkVerifyingKey};
@@ -15,12 +12,6 @@ pub struct SP1Stdin {
     pub proofs: Vec<(SP1ReduceProof<BabyBearPoseidon2>, StarkVerifyingKey<BabyBearPoseidon2>)>,
 }
 
-/// Public values for the prover.
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct SP1PublicValues {
-    buffer: Buffer,
-}
-
 impl SP1Stdin {
     /// Create a new `SP1Stdin`.
     pub const fn new() -> Self {
@@ -71,83 +62,6 @@ impl SP1Stdin {
     }
 }
 
-impl SP1PublicValues {
-    /// Create a new `SP1PublicValues`.
-    pub const fn new() -> Self {
-        Self { buffer: Buffer::new() }
-    }
-
-    pub fn raw(&self) -> String {
-        format!("0x{}", hex::encode(self.buffer.data.clone()))
-    }
-
-    /// Create a `SP1PublicValues` from a slice of bytes.
-    pub fn from(data: &[u8]) -> Self {
-        Self { buffer: Buffer::from(data) }
-    }
-
-    pub fn as_slice(&self) -> &[u8] {
-        self.buffer.data.as_slice()
-    }
-
-    pub fn to_vec(&self) -> Vec<u8> {
-        self.buffer.data.clone()
-    }
-
-    /// Read a value from the buffer.    
-    pub fn read<T: Serialize + DeserializeOwned>(&mut self) -> T {
-        self.buffer.read()
-    }
-
-    /// Read a slice of bytes from the buffer.
-    pub fn read_slice(&mut self, slice: &mut [u8]) {
-        self.buffer.read_slice(slice);
-    }
-
-    /// Write a value to the buffer.
-    pub fn write<T: Serialize>(&mut self, data: &T) {
-        self.buffer.write(data);
-    }
-
-    /// Write a slice of bytes to the buffer.
-    pub fn write_slice(&mut self, slice: &[u8]) {
-        self.buffer.write_slice(slice);
-    }
-
-    /// Hash the public values.
-    pub fn hash(&self) -> Vec<u8> {
-        let mut hasher = Sha256::new();
-        hasher.update(self.buffer.data.as_slice());
-        hasher.finalize().to_vec()
-    }
-
-    /// Hash the public values, mask the top 3 bits and return a BigUint. Matches the implementation
-    /// of `hashPublicValues` in the Solidity verifier.
-    ///
-    /// ```solidity
-    /// sha256(publicValues) & bytes32(uint256((1 << 253) - 1));
-    /// ```
-    pub fn hash_bn254(&self) -> BigUint {
-        // Hash the public values.
-        let mut hasher = Sha256::new();
-        hasher.update(self.buffer.data.as_slice());
-        let hash_result = hasher.finalize();
-        let mut hash = hash_result.to_vec();
-
-        // Mask the top 3 bits.
-        hash[0] &= 0b00011111;
-
-        // Return the masked hash as a BigUint.
-        BigUint::from_bytes_be(&hash)
-    }
-}
-
-impl AsRef<[u8]> for SP1PublicValues {
-    fn as_ref(&self) -> &[u8] {
-        &self.buffer.data
-    }
-}
-
 pub mod proof_serde {
     use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize};
     use sp1_stark::{MachineProof, StarkGenericConfig};
@@ -184,23 +98,3 @@ pub mod proof_serde {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_hash_public_values() {
-        let test_hex = "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef";
-        let test_bytes = hex::decode(test_hex).unwrap();
-
-        let mut public_values = SP1PublicValues::new();
-        public_values.write_slice(&test_bytes);
-        let hash = public_values.hash_bn254();
-
-        let expected_hash = "1ce987d0a7fcc2636fe87e69295ba12b1cc46c256b369ae7401c51b805ee91bd";
-        let expected_hash_biguint = BigUint::from_bytes_be(&hex::decode(expected_hash).unwrap());
-
-        assert_eq!(hash, expected_hash_biguint);
-    }
-}

crates/core/machine/src/syscall/precompiles/keccak256/air.rs

@@ -138,10 +138,11 @@ where
 #[cfg(test)]
 mod test {
     use crate::{
-        io::{SP1PublicValues, SP1Stdin},
+        io::SP1Stdin,
         riscv::RiscvAir,
         utils::{prove, setup_logger, tests::KECCAK256_ELF},
     };
+    use sp1_primitives::io::SP1PublicValues;
 
     use rand::{Rng, SeedableRng};
     use sp1_core_executor::Program;