Skip to content

Commit

Permalink
perf: add benchmarks for runner construction (lambdaclass#995)
Browse files Browse the repository at this point in the history 
* perf: add benchmarks for runner construction

- IAI and Criterion benchmarks for constructing 100 `CairoRunner`
  instances from a single `Program` instance.
- IAI and Criterion benchmark for `Program` parsing as baseline.
- Remove old criterion benchmarks for whole programs, they've been
  superseded by Hyperfine and have very high cost/benefit ratio.

* perf: delete whole-program benchmarks from IAI

* ci: re-enable IAI workflow
  • Loading branch information
@Oppen @kariy
Oppen authored and kariy committed Jun 23, 2023
1 parent 83108ec commit 38bae49
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 186 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/iai_pr.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
name: benchmark_pr

on:
pull_request:
branches: [ '*' ]

jobs:
fetch-iai-results:
runs-on: ubuntu-22.04
Expand Down
90 changes: 30 additions & 60 deletions bench/criterion_benchmark.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
use std::path::Path;

use cairo_vm::{
cairo_run,
hint_processor::builtin_hint_processor::builtin_hint_processor_definition::BuiltinHintProcessor,
};
use cairo_vm::{types::program::Program, vm::runners::cairo_runner::CairoRunner};
use criterion::{black_box, criterion_group, criterion_main, Criterion};

#[cfg(feature = "with_mimalloc")]
Expand All @@ -13,61 +8,36 @@ use mimalloc::MiMalloc;
#[global_allocator]
static ALLOC: MiMalloc = MiMalloc;

const BENCH_NAMES: &[&str] = &[
"compare_arrays_200000",
"factorial_multirun",
"fibonacci_1000_multirun",
"integration_builtins",
"linear_search",
"keccak_integration_benchmark",
"secp_integration_benchmark",
"blake2s_integration_benchmark",
"dict_integration_benchmark",
"math_integration_benchmark",
"memory_integration_benchmark",
"math_cmp_and_pow_integration_benchmark",
"operations_with_data_structures_benchmarks",
"uint256_integration_benchmark",
"set_integration_benchmark",
"poseidon_integration_benchmark",
"pedersen",
];
const BENCH_PATH: &str = "cairo_programs/benchmarks/";

pub fn criterion_benchmarks(c: &mut Criterion) {
let mut hint_executor = BuiltinHintProcessor::new_empty();
let cairo_run_config = cairo_vm::cairo_run::CairoRunConfig {
layout: "all_cairo",
..cairo_vm::cairo_run::CairoRunConfig::default()
};
for benchmark_name in build_bench_strings() {
let file_content = std::fs::read(Path::new(&benchmark_name.1)).unwrap();
c.bench_function(&benchmark_name.0, |b| {
b.iter(|| {
cairo_run::cairo_run(
black_box(&file_content),
&cairo_run_config,
&mut hint_executor,
)
})
});
}
fn parse_program(c: &mut Criterion) {
//Picked the biggest one at the time of writing
let program = include_bytes!("../cairo_programs/benchmarks/keccak_integration_benchmark.json");
c.bench_function("parse program", |b| {
b.iter(|| {
_ = Program::from_bytes(black_box(program.as_slice()), black_box(Some("main")))
.unwrap();
})
});
}

fn build_bench_strings() -> Vec<(String, String)> {
let mut full_string = Vec::<(String, String)>::new();

for filename in BENCH_NAMES {
let file_no_extension = String::from(*filename);
let file_extension = String::from(".json");
let bench_path = String::from(BENCH_PATH);
let full_file_path = bench_path + &file_no_extension + &file_extension;
let cairo_call = format!("cairo_run({})", &full_file_path);
full_string.push((cairo_call, full_file_path));
}

full_string
fn build_many_runners(c: &mut Criterion) {
//Picked the biggest one at the time of writing
let program = include_bytes!("../cairo_programs/benchmarks/keccak_integration_benchmark.json");
let program = Program::from_bytes(program.as_slice(), Some("main")).unwrap();
c.bench_function("build runner", |b| {
b.iter(|| {
for _ in 0..100 {
_ = black_box(
CairoRunner::new(
black_box(&program),
black_box("starknet_with_keccak"),
black_box(false),
)
.unwrap(),
);
}
})
});
}

criterion_group!(benches, criterion_benchmarks);
criterion_main!(benches);
criterion_group!(runner, parse_program, build_many_runners);
criterion_main!(runner);
144 changes: 18 additions & 126 deletions bench/iai_benchmark.rs
View file
Original file line number Diff line number Diff line change
@@ -1,50 +1,6 @@
use std::{
fs::File,
io,
io::{BufWriter, Write},
};

use bincode::enc::write::Writer;
use iai::{black_box, main};

use cairo_vm::{
cairo_run::*,
hint_processor::builtin_hint_processor::builtin_hint_processor_definition::BuiltinHintProcessor,
};

// Copied from the CLI
struct FileWriter {
buf_writer: BufWriter<File>,
bytes_written: usize,
}

impl Writer for FileWriter {
fn write(&mut self, bytes: &[u8]) -> Result<(), bincode::error::EncodeError> {
self.buf_writer
.write_all(bytes)
.map_err(|e| bincode::error::EncodeError::Io {
inner: e,
index: self.bytes_written,
})?;

self.bytes_written += bytes.len();

Ok(())
}
}

impl FileWriter {
fn new(buf_writer: BufWriter<File>) -> Self {
Self {
buf_writer,
bytes_written: 0,
}
}

fn flush(&mut self) -> Result<(), io::Error> {
self.buf_writer.flush()
}
}
use cairo_vm::{types::program::Program, vm::runners::cairo_runner::CairoRunner};

#[cfg(feature = "with_mimalloc")]
use mimalloc::MiMalloc;
Expand All @@ -53,87 +9,23 @@ use mimalloc::MiMalloc;
#[global_allocator]
static ALLOC: MiMalloc = MiMalloc;

macro_rules! iai_bench_expand_prog {
($val: ident) => {
fn $val() {
let cairo_run_config = cairo_vm::cairo_run::CairoRunConfig {
trace_enabled: true,
layout: "all_cairo",
//FIXME: we need to distinguish the proof compiled programs
//proof_mode: true,
secure_run: Some(true),
..cairo_vm::cairo_run::CairoRunConfig::default()
};
let mut hint_executor = BuiltinHintProcessor::new_empty();

let program = include_bytes!(concat!(
"../cairo_programs/benchmarks/",
stringify!($val),
".json"
));
let (runner, mut vm) =
cairo_run(black_box(program), &cairo_run_config, &mut hint_executor)
.expect("cairo_run failed");

let trace_file = File::create("/dev/null").expect("open trace file");
let mut trace_writer = FileWriter::new(BufWriter::new(trace_file));
let relocated_trace = vm.get_relocated_trace().expect("relocation failed");
write_encoded_trace(
black_box(relocated_trace.as_slice()),
black_box(&mut trace_writer),
)
.expect("writing execution trace failed");
trace_writer.flush().expect("flush trace");

let memory_file = File::create("/dev/null").expect("open memory file");
let mut memory_writer = FileWriter::new(BufWriter::new(memory_file));
write_encoded_memory(
black_box(&runner.relocated_memory),
black_box(&mut memory_writer),
)
.expect("writing relocated memory failed");
memory_writer.flush().expect("flush memory");

vm.write_output(black_box(&mut String::new()))
.expect("writing output failed");
}
};
fn parse_program() {
//Picked the biggest one at the time of writing
let program = include_bytes!("../cairo_programs/benchmarks/keccak_integration_benchmark.json");
let program =
Program::from_bytes(black_box(program.as_slice()), black_box(Some("main"))).unwrap();
let _ = black_box(program);
}

iai_bench_expand_prog! {math_integration_benchmark}
iai_bench_expand_prog! {compare_arrays_200000}
iai_bench_expand_prog! {factorial_multirun}
iai_bench_expand_prog! {fibonacci_1000_multirun}
iai_bench_expand_prog! {integration_builtins}
iai_bench_expand_prog! {linear_search}
iai_bench_expand_prog! {keccak_integration_benchmark}
iai_bench_expand_prog! {secp_integration_benchmark}
iai_bench_expand_prog! {blake2s_integration_benchmark}
iai_bench_expand_prog! {dict_integration_benchmark}
iai_bench_expand_prog! {memory_integration_benchmark}
iai_bench_expand_prog! {math_cmp_and_pow_integration_benchmark}
iai_bench_expand_prog! {operations_with_data_structures_benchmarks}
iai_bench_expand_prog! {uint256_integration_benchmark}
iai_bench_expand_prog! {set_integration_benchmark}
iai_bench_expand_prog! {poseidon_integration_benchmark}
iai_bench_expand_prog! {pedersen}
fn build_many_runners() {
//Picked the biggest one at the time of writing
let program = include_bytes!("../cairo_programs/benchmarks/keccak_integration_benchmark.json");
let program =
Program::from_bytes(black_box(program.as_slice()), black_box(Some("main"))).unwrap();
for _ in 0..100 {
let runner = CairoRunner::new(black_box(&program), "starknet_with_keccak", false).unwrap();
let _ = black_box(runner);
}
}

main!(
math_integration_benchmark,
compare_arrays_200000,
factorial_multirun,
fibonacci_1000_multirun,
integration_builtins,
linear_search,
keccak_integration_benchmark,
secp_integration_benchmark,
blake2s_integration_benchmark,
dict_integration_benchmark,
memory_integration_benchmark,
math_cmp_and_pow_integration_benchmark,
operations_with_data_structures_benchmarks,
uint256_integration_benchmark,
set_integration_benchmark,
poseidon_integration_benchmark,
pedersen,
);
main!(parse_program, build_many_runners,);

0 comments on commit 38bae49

Please sign in to comment.