chore: minor eval changes (#1452)

.github/workflows/pr.yml

@@ -247,7 +247,7 @@ jobs:
           ~/.sp1/bin/sp1up 
           ~/.sp1/bin/cargo-prove prove --version
 
-      - name: Build program and run script
+      - name: Run Evaluation
         run: |
           cd crates/eval
           RUSTFLAGS='-C target-cpu=native' cargo run --release -- \
@@ -260,7 +260,6 @@ jobs:
             --repo-owner "${{ github.repository_owner }}" \
             --repo-name "${{ github.event.repository.name }}" \
             --pr-number "${{ github.event.pull_request.number }}" \
-            --pr-name "${{ github.event.pull_request.title }}" \
             --branch-name "${{ github.head_ref || github.ref_name }}" \
             --commit-hash "${{ github.sha }}" \
             --author "${{ github.event.pull_request.user.login || github.actor }}"

crates/eval/Cargo.toml

@@ -21,7 +21,7 @@ csv = "1.3.0"
 serde = "1.0.204"
 bincode = "1.3.3"
 time = "0.3.26"
-slack-rust = { git = "https://github.com/dselans/slack-rust.git", branch = "main" }
+slack-rust = { package = "slack-rust-rs", version = "0.0.1" }
 tokio = { version = "1.39.0", features = ["full"] }
 reqwest = { version = "0.12.4", features = ["json"] }
 serde_json = "1.0.104"

crates/eval/src/lib.rs

@@ -6,7 +6,7 @@ use serde_json::json;
 use slack_rust::chat::post_message::{post_message, PostMessageRequest};
 use slack_rust::http_client::default_client;
 use sp1_prover::{components::SP1ProverComponents, utils::get_cycles, SP1Prover};
-use sp1_sdk::SP1Context;
+use sp1_sdk::{SP1Context, SP1Stdin};
 use sp1_stark::SP1ProverOpts;
 use std::time::{Duration, Instant};
 
@@ -59,10 +59,6 @@ struct EvalArgs {
     #[arg(long)]
     pub pr_number: Option<String>,
 
-    /// The name of the pull request.
-    #[arg(long)]
-    pub pr_name: Option<String>,
-
     /// The name of the branch.
     #[arg(long)]
     pub branch_name: Option<String>,
@@ -76,8 +72,11 @@ struct EvalArgs {
     pub author: Option<String>,
 }
 
-pub async fn evaluate_performance<C: SP1ProverComponents>() -> Result<(), Box<dyn std::error::Error>>
-{
+pub async fn evaluate_performance<C: SP1ProverComponents>(
+    opts: SP1ProverOpts,
+) -> Result<(), Box<dyn std::error::Error>> {
+    println!("opts: {:?}", opts);
+
     let args = EvalArgs::parse();
 
     // Set environment variables to configure the prover.
@@ -95,13 +94,16 @@ pub async fn evaluate_performance<C: SP1ProverComponents>() -> Result<(), Box<dy
             .collect()
     };
 
+    sp1_sdk::utils::setup_logger();
+
     // Run the evaluations on each program.
     let mut reports = Vec::new();
     for program in &programs {
         println!("Evaluating program: {}", program.name);
-        let report = run_evaluation::<C>(program.name, program.elf, program.input);
+        let (elf, stdin) = load_program(program.elf, program.input);
+        let report = run_evaluation::<C>(program.name, &elf, &stdin, opts);
         reports.push(report);
-        println!("Program: {} completed", program.name);
+        println!("Finished Program: {}", program.name);
     }
 
     // Prepare and format the results.
@@ -160,22 +162,21 @@ pub struct PerformanceReport {
 
 fn run_evaluation<C: SP1ProverComponents>(
     program_name: &str,
-    elf_path: &str,
-    input_path: &str,
+    elf: &[u8],
+    stdin: &SP1Stdin,
+    opts: SP1ProverOpts,
 ) -> PerformanceReport {
-    let (elf, stdin) = load_program(elf_path, input_path);
-    let cycles = get_cycles(&elf, &stdin);
+    let cycles = get_cycles(elf, stdin);
 
     let prover = SP1Prover::<C>::new();
-    let (pk, vk) = prover.setup(&elf);
+    let (pk, vk) = prover.setup(elf);
 
-    let opts = SP1ProverOpts::default();
     let context = SP1Context::default();
 
-    let (_, exec_duration) = time_operation(|| prover.execute(&elf, &stdin, context.clone()));
+    let (_, exec_duration) = time_operation(|| prover.execute(elf, stdin, context.clone()));
 
     let (core_proof, core_duration) =
-        time_operation(|| prover.prove_core(&pk, &stdin, opts, context).unwrap());
+        time_operation(|| prover.prove_core(&pk, stdin, opts, context).unwrap());
 
     let (_, compress_duration) =
         time_operation(|| prover.compress(&vk, core_proof, vec![], opts).unwrap());
@@ -195,9 +196,6 @@ fn run_evaluation<C: SP1ProverComponents>(
 
 fn format_results(args: &EvalArgs, results: &[PerformanceReport]) -> Vec<String> {
     let mut detail_text = String::new();
-    if let Some(pr_name) = &args.pr_name {
-        detail_text.push_str(&format!("*PR*: {}\n", pr_name));
-    }
     if let Some(branch_name) = &args.branch_name {
         detail_text.push_str(&format!("*Branch*: {}\n", branch_name));
     }
@@ -400,7 +398,6 @@ mod tests {
             repo_owner: Some("succinctlabs".to_string()),
             repo_name: Some("sp1".to_string()),
             pr_number: Some("123456".to_string()),
-            pr_name: Some("Test PR".to_string()),
             branch_name: Some("feature-branch".to_string()),
             commit_hash: Some("abcdef1234567890".to_string()),
             author: Some("John Doe".to_string()),
@@ -414,7 +411,6 @@ mod tests {
 
         assert_eq!(formatted_results.len(), 3);
         assert!(formatted_results[0].contains("SP1 Performance Test Results"));
-        assert!(formatted_results[1].contains("*PR*: Test PR"));
         assert!(formatted_results[1].contains("*Branch*: feature-branch"));
         assert!(formatted_results[1].contains("*Commit*: abcdef12"));
         assert!(formatted_results[1].contains("*Author*: John Doe"));

crates/eval/src/main.rs

@@ -1,8 +1,10 @@
 use anyhow::Result;
 use sp1_eval::evaluate_performance;
 use sp1_prover::components::DefaultProverComponents;
+use sp1_stark::SP1ProverOpts;
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    evaluate_performance::<DefaultProverComponents>().await
+    let opts = SP1ProverOpts::default();
+    evaluate_performance::<DefaultProverComponents>(opts).await
 }

crates/eval/src/program.rs

@@ -1,37 +1,37 @@
 use sp1_sdk::SP1Stdin;
-use std::fs::File;
-use std::io::Read;
 
 #[derive(Clone)]
 pub struct TesterProgram {
     pub name: &'static str,
-    pub elf: &'static str,
-    pub input: &'static str,
+    pub elf: &'static [u8],
+    pub input: &'static [u8],
 }
 
 impl TesterProgram {
-    const fn new(name: &'static str, elf: &'static str, input: &'static str) -> Self {
+    const fn new(name: &'static str, elf: &'static [u8], input: &'static [u8]) -> Self {
         Self { name, elf, input }
     }
 }
 
 pub const PROGRAMS: &[TesterProgram] = &[
-    TesterProgram::new("fibonacci", "fibonacci/elf", "fibonacci/input.bin"),
-    TesterProgram::new("ssz-withdrawals", "ssz-withdrawals/elf", "ssz-withdrawals/input.bin"),
-    TesterProgram::new("tendermint", "tendermint/elf", "tendermint/input.bin"),
+    TesterProgram::new(
+        "fibonacci",
+        include_bytes!("../programs/fibonacci/elf"),
+        include_bytes!("../programs/fibonacci/input.bin"),
+    ),
+    TesterProgram::new(
+        "ssz-withdrawals",
+        include_bytes!("../programs/ssz-withdrawals/elf"),
+        include_bytes!("../programs/ssz-withdrawals/input.bin"),
+    ),
+    TesterProgram::new(
+        "tendermint",
+        include_bytes!("../programs/tendermint/elf"),
+        include_bytes!("../programs/tendermint/input.bin"),
+    ),
 ];
 
-pub fn load_program(elf_path: &str, input_path: &str) -> (Vec<u8>, SP1Stdin) {
-    let elf_path = format!("./programs/{}", elf_path);
-    let input_path = format!("./programs/{}", input_path);
-
-    let mut elf_file = File::open(elf_path).expect("failed to open elf");
-    let mut elf = Vec::new();
-    elf_file.read_to_end(&mut elf).expect("failed to read elf");
-
-    let input_file = File::open(input_path).expect("failed to open input");
-    let stdin: SP1Stdin =
-        bincode::deserialize_from(input_file).expect("failed to deserialize input");
-
-    (elf, stdin)
+pub fn load_program(elf: &[u8], input: &[u8]) -> (Vec<u8>, SP1Stdin) {
+    let stdin: SP1Stdin = bincode::deserialize(input).expect("failed to deserialize input");
+    (elf.to_vec(), stdin)
 }