Criterion benches

Cargo.toml

@@ -29,3 +29,14 @@ default-features = false
 [dependencies.jemallocator]
 version = "0.3.2"
 features = ["disable_initial_exec_tls"]
+
+[dev-dependencies]
+criterion = "0.3"
+
+[[bench]]
+name = "merk"
+harness = false
+
+[[bench]]
+name = "ops"
+harness = false

benches/merk.rs

@@ -1,17 +1,14 @@
 #![feature(test)]
 
-extern crate test;
-
+use criterion::*;
 use merk::proofs::encode_into as encode_proof_into;
 use merk::restore::Restorer;
 use merk::test_utils::*;
 use merk::{Merk, Result};
 use rand::prelude::*;
 use std::thread;
-use test::Bencher;
 
-#[bench]
-fn get_1m_rocksdb(b: &mut Bencher) {
+fn get_1m_rocksdb(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
     let num_batches = initial_size / batch_size;
@@ -27,19 +24,20 @@ fn get_1m_rocksdb(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch_index = (i % num_batches) as usize;
-        let key_index = (i / num_batches) as usize;
+    c.bench_function("get_1m_rocksdb", |b| {
+        b.iter(|| {
+            let batch_index = (i % num_batches) as usize;
+            let key_index = (i / num_batches) as usize;
 
-        let key = &batches[batch_index][key_index].0;
-        merk.get(key).expect("get failed");
+            let key = &batches[batch_index][key_index].0;
+            merk.get(key).expect("get failed");
 
-        i = (i + 1) % initial_size;
+            i = (i + 1) % initial_size;
+        })
     });
 }
 
-#[bench]
-fn insert_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
+fn insert_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -52,15 +50,16 @@ fn insert_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = initial_size / batch_size;
-    b.iter(|| {
-        let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i += 1;
+    c.bench_function("insert_1m_2k_seq_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i += 1;
+        })
     });
 }
 
-#[bench]
-fn insert_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
+fn insert_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -73,15 +72,16 @@ fn insert_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = initial_size / batch_size;
-    b.iter(|| {
-        let batch = make_batch_rand(batch_size, i);
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i += 1;
+    c.bench_function("insert_1m_2k_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let batch = make_batch_rand(batch_size, i);
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i += 1;
+        })
     });
 }
 
-#[bench]
-fn update_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
+fn update_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -94,15 +94,16 @@ fn update_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i = (i + 1) % (initial_size / batch_size);
+    c.bench_function("update_1m_2k_seq_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i = (i + 1) % (initial_size / batch_size);
+        })
     });
 }
 
-#[bench]
-fn update_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
+fn update_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -115,15 +116,16 @@ fn update_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_rand(batch_size, i);
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i = (i + 1) % (initial_size / batch_size);
+    c.bench_function("update_1m_2k_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let batch = make_batch_rand(batch_size, i);
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i = (i + 1) % (initial_size / batch_size);
+        })
     });
 }
 
-#[bench]
-fn delete_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
+fn delete_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -136,19 +138,20 @@ fn delete_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        if i >= (initial_size / batch_size) {
-            println!("WARNING: too many bench iterations, whole tree deleted");
-            return;
-        }
-        let batch = make_del_batch_rand(batch_size, i);
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i = (i + 1) % (initial_size / batch_size);
+    c.bench_function("delete_1m_2k_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            if i >= (initial_size / batch_size) {
+                println!("WARNING: too many bench iterations, whole tree deleted");
+                return;
+            }
+            let batch = make_del_batch_rand(batch_size, i);
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i = (i + 1) % (initial_size / batch_size);
+        })
     });
 }
 
-#[bench]
-fn prove_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn prove_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
     let proof_size = 1;
@@ -162,22 +165,23 @@ fn prove_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_rand(proof_size, i);
-        let mut keys = Vec::with_capacity(batch.len());
-        for (key, _) in batch {
-            keys.push(key);
-        }
-        unsafe { merk.prove_unchecked(keys.as_slice()).expect("prove failed") };
-        i = (i + 1) % (initial_size / batch_size);
+    c.bench_function("prove_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let batch = make_batch_rand(proof_size, i);
+            let mut keys = Vec::with_capacity(batch.len());
+            for (key, _) in batch {
+                keys.push(key);
+            }
+            unsafe { merk.prove_unchecked(keys.as_slice()).expect("prove failed") };
+            i = (i + 1) % (initial_size / batch_size);
 
-        merk.commit(std::collections::LinkedList::new(), &[])
-            .unwrap();
+            merk.commit(std::collections::LinkedList::new(), &[])
+                .unwrap();
+        })
     });
 }
 
-#[bench]
-fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -191,21 +195,24 @@ fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
 
     let mut bytes = vec![];
 
-    b.iter(|| {
-        bytes.clear();
+    c.bench_function("build_trunk_chunk_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            bytes.clear();
 
-        let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap());
-        encode_proof_into(ops.iter(), &mut bytes);
+            let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap());
+            encode_proof_into(ops.iter(), &mut bytes);
 
-        merk.commit(std::collections::LinkedList::new(), &[])
-            .unwrap();
+            merk.commit(std::collections::LinkedList::new(), &[])
+                .unwrap();
+        })
     });
 
-    b.bytes = bytes.len() as u64;
+    //Need to figure out how to support this in criterion
+    //This is to show throughput
+    //b.bytes = bytes.len() as u64;
 }
 
-#[bench]
-fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let mut rng = rand::thread_rng();
 
     let initial_size = 1_000_000;
@@ -227,18 +234,18 @@ fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
         let index = rng.gen::<usize>() % chunks.len();
         chunks.chunk(index).unwrap()
     };
-
-    b.iter(|| {
-        let chunk = next();
-        total_bytes += chunk.len();
-        i += 1;
+    c.bench_function("chunkproducer_rand_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let chunk = next();
+            total_bytes += chunk.len();
+            i += 1;
+        })
     });
 
-    b.bytes = (total_bytes / i) as u64;
+    //b.bytes = (total_bytes / i) as u64;
 }
 
-#[bench]
-fn chunk_iter_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn chunk_iter_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -261,18 +268,18 @@ fn chunk_iter_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
             chunks.next().unwrap()
         }
     };
-
-    b.iter(|| {
-        let chunk = next();
-        total_bytes += chunk.unwrap().len();
-        i += 1;
+    c.bench_function("chunk_iter_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let chunk = next();
+            total_bytes += chunk.unwrap().len();
+            i += 1;
+        })
     });
 
-    b.bytes = (total_bytes / i) as u64;
+    //b.bytes = (total_bytes / i) as u64;
 }
 
-#[bench]
-fn restore_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn restore_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -297,32 +304,33 @@ fn restore_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
     let mut total_bytes = 0;
     let mut i = 0;
 
-    b.iter(|| {
-        if i % chunks.len() == 0 {
-            if i != 0 {
-                let restorer_merk = restorer.take().unwrap().finalize();
-                drop(restorer_merk);
-                std::fs::remove_dir_all(&path).unwrap();
-            }
+    c.bench_function("restore_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            if i % chunks.len() == 0 {
+                if i != 0 {
+                    let restorer_merk = restorer.take().unwrap().finalize();
+                    drop(restorer_merk);
+                    std::fs::remove_dir_all(&path).unwrap();
+                }
 
-            restorer = Some(Merk::restore(&path, merk.root_hash(), chunks.len()).unwrap());
-        }
+                restorer = Some(Merk::restore(&path, merk.root_hash(), chunks.len()).unwrap());
+            }
 
-        let restorer = restorer.as_mut().unwrap();
-        let chunk = chunks[i % chunks.len()].as_slice();
-        restorer.process_chunk(chunk).unwrap();
+            let restorer = restorer.as_mut().unwrap();
+            let chunk = chunks[i % chunks.len()].as_slice();
+            restorer.process_chunk(chunk).unwrap();
 
-        total_bytes += chunk.len();
-        i += 1;
+            total_bytes += chunk.len();
+            i += 1;
+        })
     });
 
     std::fs::remove_dir_all(&path).unwrap();
 
-    b.bytes = (total_bytes / i) as u64;
+    //b.bytes = (total_bytes / i) as u64;
 }
 
-#[bench]
-fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -335,8 +343,29 @@ fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let path = path + ".checkpoint";
-    b.iter(|| {
-        let checkpoint = merk.checkpoint(&path).unwrap();
-        checkpoint.destroy().unwrap();
+    c.bench_function("checkpoint_create_destroy_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
+            let checkpoint = merk.checkpoint(&path).unwrap();
+            checkpoint.destroy().unwrap();
+        })
     });
 }
+
+criterion_group! {
+    name = merk;
+    config = Criterion::default();
+    targets = get_1m_rocksdb,
+        insert_1m_2k_seq_rocksdb_noprune,
+        insert_1m_2k_rand_rocksdb_noprune,
+        update_1m_2k_seq_rocksdb_noprune,
+        update_1m_2k_rand_rocksdb_noprune,
+        delete_1m_2k_rand_rocksdb_noprune,
+        prove_1m_1_rand_rocksdb_noprune,
+        build_trunk_chunk_1m_1_rand_rocksdb_noprune,
+        chunkproducer_rand_1m_1_rand_rocksdb_noprune,
+        chunk_iter_1m_1_rand_rocksdb_noprune,
+        restore_1m_1_rand_rocksdb_noprune,
+        checkpoint_create_destroy_1m_1_rand_rocksdb_noprune
+}
+
+criterion_main!(merk);