From 175b55b94eb609d3b26d2ab1ba234859158b03d0 Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Fri, 28 May 2021 17:20:04 -0500
Subject: [PATCH 1/7] Add criterion to dev dependencies

---
 Cargo.toml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Cargo.toml b/Cargo.toml
index e3f3fcac..f1b5dcd2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -29,3 +29,6 @@ default-features = false
 [dependencies.jemallocator]
 version = "0.3.2"
 features = ["disable_initial_exec_tls"]
+
+[dev-dependencies]
+criterion = "0.3"

From 4170fa889c19ba26b362538b199c6f64b5d4a86a Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Fri, 28 May 2021 20:08:31 -0500
Subject: [PATCH 2/7] Remove bench harness for benches/merk.rs

---
 Cargo.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Cargo.toml b/Cargo.toml
index f1b5dcd2..c2cd923e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,3 +32,7 @@ features = ["disable_initial_exec_tls"]
 
 [dev-dependencies]
 criterion = "0.3"
+
+[[bench]]
+name = "merk"
+harness = false

From 07a4e17faecd14eb42bafda5ac5486325a2787dc Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Fri, 28 May 2021 20:09:18 -0500
Subject: [PATCH 3/7] Refactor merk benches to use criterion

---
 benches/merk.rs | 291 +++++++++++++++++++++++++++---------------------
 1 file changed, 166 insertions(+), 125 deletions(-)

diff --git a/benches/merk.rs b/benches/merk.rs
index 0cd2474a..179e7196 100644
--- a/benches/merk.rs
+++ b/benches/merk.rs
@@ -1,17 +1,14 @@
 #![feature(test)]
 
-extern crate test;
-
+use criterion::*;
 use merk::proofs::encode_into as encode_proof_into;
 use merk::restore::Restorer;
 use merk::test_utils::*;
 use merk::{Merk, Result};
 use rand::prelude::*;
 use std::thread;
-use test::Bencher;
 
-#[bench]
-fn get_1m_rocksdb(b: &mut Bencher) {
+fn get_1m_rocksdb(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
     let num_batches = initial_size / batch_size;
@@ -27,19 +24,21 @@ fn get_1m_rocksdb(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch_index = (i % num_batches) as usize;
-        let key_index = (i / num_batches) as usize;
-
-        let key = &batches[batch_index][key_index].0;
-        merk.get(key).expect("get failed");
-
-        i = (i + 1) % initial_size;
-    });
+    c.bench_function(
+        "get_1m_rocksdb",
+        |b| b.iter(|| {
+            let batch_index = (i % num_batches) as usize;
+            let key_index = (i / num_batches) as usize;
+
+            let key = &batches[batch_index][key_index].0;
+            merk.get(key).expect("get failed");
+
+            i = (i + 1) % initial_size;
+        })
+    );
 }
 
-#[bench]
-fn insert_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
+fn insert_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -52,15 +51,17 @@ fn insert_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = initial_size / batch_size;
-    b.iter(|| {
-        let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i += 1;
-    });
+    c.bench_function(
+        "insert_1m_2k_seq_rocksdb_noprune",
+        |b| b.iter(|| {
+            let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i += 1;
+        })
+    );
 }
 
-#[bench]
-fn insert_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
+fn insert_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -73,15 +74,17 @@ fn insert_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = initial_size / batch_size;
-    b.iter(|| {
-        let batch = make_batch_rand(batch_size, i);
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i += 1;
-    });
+    c.bench_function(
+        "insert_1m_2k_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            let batch = make_batch_rand(batch_size, i);
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i += 1;
+        })
+    );
 }
 
-#[bench]
-fn update_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
+fn update_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -94,15 +97,17 @@ fn update_1m_2k_seq_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i = (i + 1) % (initial_size / batch_size);
-    });
+    c.bench_function( 
+        "update_1m_2k_seq_rocksdb_noprune",
+        |b| b.iter(|| {
+            let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i = (i + 1) % (initial_size / batch_size);
+        })
+    );
 }
 
-#[bench]
-fn update_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
+fn update_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -115,15 +120,17 @@ fn update_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_rand(batch_size, i);
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i = (i + 1) % (initial_size / batch_size);
-    });
+    c.bench_function(
+        "update_1m_2k_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            let batch = make_batch_rand(batch_size, i);
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i = (i + 1) % (initial_size / batch_size);
+        })
+    );
 }
 
-#[bench]
-fn delete_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
+fn delete_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 2_000;
 
@@ -136,19 +143,21 @@ fn delete_1m_2k_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        if i >= (initial_size / batch_size) {
-            println!("WARNING: too many bench iterations, whole tree deleted");
-            return;
-        }
-        let batch = make_del_batch_rand(batch_size, i);
-        unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
-        i = (i + 1) % (initial_size / batch_size);
-    });
+    c.bench_function(
+        "delete_1m_2k_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            if i >= (initial_size / batch_size) {
+                println!("WARNING: too many bench iterations, whole tree deleted");
+                return;
+            }
+            let batch = make_del_batch_rand(batch_size, i);
+            unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
+            i = (i + 1) % (initial_size / batch_size);
+        })
+    );
 }
 
-#[bench]
-fn prove_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn prove_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
     let proof_size = 1;
@@ -162,22 +171,24 @@ fn prove_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_rand(proof_size, i);
-        let mut keys = Vec::with_capacity(batch.len());
-        for (key, _) in batch {
-            keys.push(key);
-        }
-        unsafe { merk.prove_unchecked(keys.as_slice()).expect("prove failed") };
-        i = (i + 1) % (initial_size / batch_size);
+    c.bench_function(
+        "prove_1m_1_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            let batch = make_batch_rand(proof_size, i);
+            let mut keys = Vec::with_capacity(batch.len());
+            for (key, _) in batch {
+                keys.push(key);
+            }
+            unsafe { merk.prove_unchecked(keys.as_slice()).expect("prove failed") };
+            i = (i + 1) % (initial_size / batch_size);
 
-        merk.commit(std::collections::LinkedList::new(), &[])
-            .unwrap();
-    });
+            merk.commit(std::collections::LinkedList::new(), &[])
+                .unwrap();
+        })
+    );
 }
 
-#[bench]
-fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -190,22 +201,25 @@ fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let mut bytes = vec![];
-
-    b.iter(|| {
-        bytes.clear();
-
-        let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap());
-        encode_proof_into(ops.iter(), &mut bytes);
-
-        merk.commit(std::collections::LinkedList::new(), &[])
-            .unwrap();
-    });
-
-    b.bytes = bytes.len() as u64;
+    c.bench_function(
+        "build_trunk_chunk_1m_1_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            bytes.clear();
+
+            let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap());
+            encode_proof_into(ops.iter(), &mut bytes);
+
+            merk.commit(std::collections::LinkedList::new(), &[])
+                .unwrap();
+        })
+    );
+
+    //Need to figure out how to support this in criterion
+    //This is to show throughput
+    //b.bytes = bytes.len() as u64;
 }
 
-#[bench]
-fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let mut rng = rand::thread_rng();
 
     let initial_size = 1_000_000;
@@ -227,18 +241,19 @@ fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
         let index = rng.gen::<usize>() % chunks.len();
         chunks.chunk(index).unwrap()
     };
-
-    b.iter(|| {
-        let chunk = next();
-        total_bytes += chunk.len();
-        i += 1;
-    });
-
-    b.bytes = (total_bytes / i) as u64;
+    c.bench_function(
+        "chunkproducer_rand_1m_1_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            let chunk = next();
+            total_bytes += chunk.len();
+            i += 1;
+        })
+    );
+    //criterion support for throughput in reports
+    //b.bytes = (total_bytes / i) as u64;
 }
 
-#[bench]
-fn chunk_iter_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn chunk_iter_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -261,18 +276,20 @@ fn chunk_iter_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
             chunks.next().unwrap()
         }
     };
-
-    b.iter(|| {
-        let chunk = next();
-        total_bytes += chunk.unwrap().len();
-        i += 1;
-    });
-
-    b.bytes = (total_bytes / i) as u64;
+    c.bench_function(
+        "chunk_iter_1m_1_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            let chunk = next();
+            total_bytes += chunk.unwrap().len();
+            i += 1;
+        })
+    );
+    
+    //need support for this in criterion
+    //b.bytes = (total_bytes / i) as u64;
 }
 
-#[bench]
-fn restore_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn restore_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -296,33 +313,36 @@ fn restore_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
 
     let mut total_bytes = 0;
     let mut i = 0;
-
-    b.iter(|| {
-        if i % chunks.len() == 0 {
-            if i != 0 {
-                let restorer_merk = restorer.take().unwrap().finalize();
-                drop(restorer_merk);
-                std::fs::remove_dir_all(&path).unwrap();
+    
+    c.bench_function(
+        "restore_1m_1_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            if i % chunks.len() == 0 {
+                if i != 0 {
+                    let restorer_merk = restorer.take().unwrap().finalize();
+                    drop(restorer_merk);
+                    std::fs::remove_dir_all(&path).unwrap();
+                }
+
+                restorer = Some(Merk::restore(&path, merk.root_hash(), chunks.len()).unwrap());
             }
 
-            restorer = Some(Merk::restore(&path, merk.root_hash(), chunks.len()).unwrap());
-        }
-
-        let restorer = restorer.as_mut().unwrap();
-        let chunk = chunks[i % chunks.len()].as_slice();
-        restorer.process_chunk(chunk).unwrap();
+            let restorer = restorer.as_mut().unwrap();
+            let chunk = chunks[i % chunks.len()].as_slice();
+            restorer.process_chunk(chunk).unwrap();
 
-        total_bytes += chunk.len();
-        i += 1;
-    });
+            total_bytes += chunk.len();
+            i += 1;
+        })
+    );
 
     std::fs::remove_dir_all(&path).unwrap();
 
-    b.bytes = (total_bytes / i) as u64;
+    //need criterion support for this
+    //b.bytes = (total_bytes / i) as u64;
 }
 
-#[bench]
-fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
+fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 1_000;
 
@@ -335,8 +355,29 @@ fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(b: &mut Bencher) {
     }
 
     let path = path + ".checkpoint";
-    b.iter(|| {
-        let checkpoint = merk.checkpoint(&path).unwrap();
-        checkpoint.destroy().unwrap();
-    });
+    c.bench_function(
+        "checkpoint_create_destroy_1m_1_rand_rocksdb_noprune",
+        |b| b.iter(|| {
+            let checkpoint = merk.checkpoint(&path).unwrap();
+            checkpoint.destroy().unwrap();
+        })
+    );
 }
+
+criterion_group!(
+    benches, 
+    get_1m_rocksdb, 
+    insert_1m_2k_seq_rocksdb_noprune,
+    insert_1m_2k_rand_rocksdb_noprune,
+    update_1m_2k_seq_rocksdb_noprune,
+    update_1m_2k_rand_rocksdb_noprune,
+    delete_1m_2k_rand_rocksdb_noprune,
+    prove_1m_1_rand_rocksdb_noprune,
+    build_trunk_chunk_1m_1_rand_rocksdb_noprune,
+    chunkproducer_rand_1m_1_rand_rocksdb_noprune,
+    chunk_iter_1m_1_rand_rocksdb_noprune,
+    restore_1m_1_rand_rocksdb_noprune,
+    checkpoint_create_destroy_1m_1_rand_rocksdb_noprune,
+);
+
+criterion_main!(benches);

From 09325f1d553375f32ae17c3dd2bcd842eac9fba7 Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Wed, 2 Jun 2021 16:56:17 -0500
Subject: [PATCH 4/7] Change criterion macro

---
 benches/merk.rs | 37 +++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/benches/merk.rs b/benches/merk.rs
index 179e7196..b5c16879 100644
--- a/benches/merk.rs
+++ b/benches/merk.rs
@@ -200,7 +200,9 @@ fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
         unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
     }
 
+   
     let mut bytes = vec![];
+
     c.bench_function(
         "build_trunk_chunk_1m_1_rand_rocksdb_noprune",
         |b| b.iter(|| {
@@ -249,7 +251,7 @@ fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
             i += 1;
         })
     );
-    //criterion support for throughput in reports
+    
     //b.bytes = (total_bytes / i) as u64;
 }
 
@@ -285,7 +287,6 @@ fn chunk_iter_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
         })
     );
     
-    //need support for this in criterion
     //b.bytes = (total_bytes / i) as u64;
 }
 
@@ -338,7 +339,6 @@ fn restore_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
 
     std::fs::remove_dir_all(&path).unwrap();
 
-    //need criterion support for this
     //b.bytes = (total_bytes / i) as u64;
 }
 
@@ -364,20 +364,21 @@ fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     );
 }
 
-criterion_group!(
-    benches, 
-    get_1m_rocksdb, 
-    insert_1m_2k_seq_rocksdb_noprune,
-    insert_1m_2k_rand_rocksdb_noprune,
-    update_1m_2k_seq_rocksdb_noprune,
-    update_1m_2k_rand_rocksdb_noprune,
-    delete_1m_2k_rand_rocksdb_noprune,
-    prove_1m_1_rand_rocksdb_noprune,
-    build_trunk_chunk_1m_1_rand_rocksdb_noprune,
-    chunkproducer_rand_1m_1_rand_rocksdb_noprune,
-    chunk_iter_1m_1_rand_rocksdb_noprune,
-    restore_1m_1_rand_rocksdb_noprune,
-    checkpoint_create_destroy_1m_1_rand_rocksdb_noprune,
-);
+criterion_group!{
+    name = benches;
+    config = Criterion::default();
+    targets = get_1m_rocksdb, 
+        insert_1m_2k_seq_rocksdb_noprune,
+        insert_1m_2k_rand_rocksdb_noprune,
+        update_1m_2k_seq_rocksdb_noprune,
+        update_1m_2k_rand_rocksdb_noprune,
+        delete_1m_2k_rand_rocksdb_noprune,
+        prove_1m_1_rand_rocksdb_noprune,
+        build_trunk_chunk_1m_1_rand_rocksdb_noprune,
+        chunkproducer_rand_1m_1_rand_rocksdb_noprune,
+        chunk_iter_1m_1_rand_rocksdb_noprune,
+        restore_1m_1_rand_rocksdb_noprune,
+        checkpoint_create_destroy_1m_1_rand_rocksdb_noprune
+}
 
 criterion_main!(benches);

From 363cae84f57d94b7b6cffdffe21a14053384ef3e Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Wed, 2 Jun 2021 17:36:45 -0500
Subject: [PATCH 5/7] Rename merk benches

---
 benches/merk.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benches/merk.rs b/benches/merk.rs
index b5c16879..d33bbab1 100644
--- a/benches/merk.rs
+++ b/benches/merk.rs
@@ -365,7 +365,7 @@ fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
 }
 
 criterion_group!{
-    name = benches;
+    name = merk;
     config = Criterion::default();
     targets = get_1m_rocksdb, 
         insert_1m_2k_seq_rocksdb_noprune,
@@ -381,4 +381,4 @@ criterion_group!{
         checkpoint_create_destroy_1m_1_rand_rocksdb_noprune
 }
 
-criterion_main!(benches);
+criterion_main!(merk);

From c833d0064ecc202fa0ad049c6b927d2ff687037e Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Wed, 2 Jun 2021 17:37:06 -0500
Subject: [PATCH 6/7] Refactor ops benches for criterion

---
 Cargo.toml     |  4 +++
 benches/ops.rs | 79 ++++++++++++++++++++++++++++++--------------------
 2 files changed, 52 insertions(+), 31 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index c2cd923e..437714ac 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -36,3 +36,7 @@ criterion = "0.3"
 [[bench]]
 name = "merk"
 harness = false
+
+[[bench]]
+name = "ops"
+harness = false
diff --git a/benches/ops.rs b/benches/ops.rs
index 14010f1e..c1e5db6e 100644
--- a/benches/ops.rs
+++ b/benches/ops.rs
@@ -1,67 +1,84 @@
 #![feature(test)]
 
-extern crate test;
-
+use criterion::*;
 use merk::owner::Owner;
 use merk::test_utils::*;
-use test::Bencher;
 
-#[bench]
-fn insert_1m_10k_seq_memonly(b: &mut Bencher) {
+fn insert_1m_10k_seq_memonly(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 10_000;
 
     let mut tree = Owner::new(make_tree_seq(initial_size));
 
     let mut i = initial_size / batch_size;
-    b.iter(|| {
-        let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
-        tree.own(|tree| apply_memonly_unchecked(tree, &batch));
-        i += 1;
-    });
+    c.bench_function(
+        "insert_1m_10k_seq_memonly",
+        |b| b.iter(|| {
+            let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
+            tree.own(|tree| apply_memonly_unchecked(tree, &batch));
+            i += 1;
+        })
+    );
 }
 
-#[bench]
-fn insert_1m_10k_rand_memonly(b: &mut Bencher) {
+fn insert_1m_10k_rand_memonly(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 10_000;
 
     let mut tree = Owner::new(make_tree_rand(initial_size, batch_size, 0));
 
     let mut i = initial_size / batch_size;
-    b.iter(|| {
-        let batch = make_batch_rand(batch_size, i);
-        tree.own(|tree| apply_memonly_unchecked(tree, &batch));
-        i += 1;
-    });
+    c.bench_function(
+        "insert_1m_10k_rand_memonly",
+        |b| b.iter(|| {
+            let batch = make_batch_rand(batch_size, i);
+            tree.own(|tree| apply_memonly_unchecked(tree, &batch));
+            i += 1;
+        })
+    );
 }
 
-#[bench]
-fn update_1m_10k_seq_memonly(b: &mut Bencher) {
+fn update_1m_10k_seq_memonly(c: &mut Criterion) {
     let initial_size = 1_000_000;
     let batch_size = 10_000;
 
     let mut tree = Owner::new(make_tree_seq(initial_size));
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
-        tree.own(|tree| apply_memonly_unchecked(tree, &batch));
-        i = (i + 1) % (initial_size / batch_size);
-    });
+    c.bench_function(
+        "update_1m_10k_seq_memonly",
+        |b| b.iter(|| {
+            let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
+            tree.own(|tree| apply_memonly_unchecked(tree, &batch));
+            i = (i + 1) % (initial_size / batch_size);
+        })
+    );
 }
 
-#[bench]
-fn update_1m_10k_rand_memonly(b: &mut Bencher) {
+fn update_1m_10k_rand_memonly(c: &mut Criterion) {
     let initial_size = 1_010_000;
     let batch_size = 10_000;
 
     let mut tree = Owner::new(make_tree_rand(initial_size, batch_size, 0));
 
     let mut i = 0;
-    b.iter(|| {
-        let batch = make_batch_rand(batch_size, i);
-        tree.own(|tree| apply_memonly_unchecked(tree, &batch));
-        i = (i + 1) % (initial_size / batch_size);
-    });
+    c.bench_function(
+        "update_1m_10k_rand_memonly",
+        |b| b.iter(|| {
+            let batch = make_batch_rand(batch_size, i);
+            tree.own(|tree| apply_memonly_unchecked(tree, &batch));
+            i = (i + 1) % (initial_size / batch_size);
+        })
+    );
+}
+
+criterion_group!{
+    name = ops;
+    config = Criterion::default();
+    targets =  insert_1m_10k_seq_memonly,
+        insert_1m_10k_rand_memonly,
+        update_1m_10k_seq_memonly,
+        update_1m_10k_rand_memonly,
 }
+
+criterion_main!(ops);

From 6eed08a59803f69d1f8da0dfbd1d25d6e16fb717 Mon Sep 17 00:00:00 2001
From: Charlie Little <cwlittle@utexas.edu>
Date: Thu, 3 Jun 2021 17:49:21 -0500
Subject: [PATCH 7/7] Cargo fmt

---
 benches/merk.rs | 95 +++++++++++++++++++++----------------------------
 benches/ops.rs  | 30 +++++++---------
 2 files changed, 54 insertions(+), 71 deletions(-)

diff --git a/benches/merk.rs b/benches/merk.rs
index d33bbab1..c9be61c0 100644
--- a/benches/merk.rs
+++ b/benches/merk.rs
@@ -24,9 +24,8 @@ fn get_1m_rocksdb(c: &mut Criterion) {
     }
 
     let mut i = 0;
-    c.bench_function(
-        "get_1m_rocksdb",
-        |b| b.iter(|| {
+    c.bench_function("get_1m_rocksdb", |b| {
+        b.iter(|| {
             let batch_index = (i % num_batches) as usize;
             let key_index = (i / num_batches) as usize;
 
@@ -35,7 +34,7 @@ fn get_1m_rocksdb(c: &mut Criterion) {
 
             i = (i + 1) % initial_size;
         })
-    );
+    });
 }
 
 fn insert_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
@@ -51,14 +50,13 @@ fn insert_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let mut i = initial_size / batch_size;
-    c.bench_function(
-        "insert_1m_2k_seq_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("insert_1m_2k_seq_rocksdb_noprune", |b| {
+        b.iter(|| {
             let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
             unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
             i += 1;
         })
-    );
+    });
 }
 
 fn insert_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
@@ -74,14 +72,13 @@ fn insert_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let mut i = initial_size / batch_size;
-    c.bench_function(
-        "insert_1m_2k_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("insert_1m_2k_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             let batch = make_batch_rand(batch_size, i);
             unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
             i += 1;
         })
-    );
+    });
 }
 
 fn update_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
@@ -97,14 +94,13 @@ fn update_1m_2k_seq_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let mut i = 0;
-    c.bench_function( 
-        "update_1m_2k_seq_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("update_1m_2k_seq_rocksdb_noprune", |b| {
+        b.iter(|| {
             let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
             unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
             i = (i + 1) % (initial_size / batch_size);
         })
-    );
+    });
 }
 
 fn update_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
@@ -120,14 +116,13 @@ fn update_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let mut i = 0;
-    c.bench_function(
-        "update_1m_2k_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("update_1m_2k_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             let batch = make_batch_rand(batch_size, i);
             unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
             i = (i + 1) % (initial_size / batch_size);
         })
-    );
+    });
 }
 
 fn delete_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
@@ -143,9 +138,8 @@ fn delete_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let mut i = 0;
-    c.bench_function(
-        "delete_1m_2k_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("delete_1m_2k_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             if i >= (initial_size / batch_size) {
                 println!("WARNING: too many bench iterations, whole tree deleted");
                 return;
@@ -154,7 +148,7 @@ fn delete_1m_2k_rand_rocksdb_noprune(c: &mut Criterion) {
             unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
             i = (i + 1) % (initial_size / batch_size);
         })
-    );
+    });
 }
 
 fn prove_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
@@ -171,9 +165,8 @@ fn prove_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let mut i = 0;
-    c.bench_function(
-        "prove_1m_1_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("prove_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             let batch = make_batch_rand(proof_size, i);
             let mut keys = Vec::with_capacity(batch.len());
             for (key, _) in batch {
@@ -185,7 +178,7 @@ fn prove_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
             merk.commit(std::collections::LinkedList::new(), &[])
                 .unwrap();
         })
-    );
+    });
 }
 
 fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
@@ -200,12 +193,10 @@ fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
         unsafe { merk.apply_unchecked(&batch, &[]).expect("apply failed") };
     }
 
-   
     let mut bytes = vec![];
 
-    c.bench_function(
-        "build_trunk_chunk_1m_1_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("build_trunk_chunk_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             bytes.clear();
 
             let (ops, _) = merk.walk(|walker| walker.unwrap().create_trunk_proof().unwrap());
@@ -214,7 +205,7 @@ fn build_trunk_chunk_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
             merk.commit(std::collections::LinkedList::new(), &[])
                 .unwrap();
         })
-    );
+    });
 
     //Need to figure out how to support this in criterion
     //This is to show throughput
@@ -243,15 +234,14 @@ fn chunkproducer_rand_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
         let index = rng.gen::<usize>() % chunks.len();
         chunks.chunk(index).unwrap()
     };
-    c.bench_function(
-        "chunkproducer_rand_1m_1_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("chunkproducer_rand_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             let chunk = next();
             total_bytes += chunk.len();
             i += 1;
         })
-    );
-    
+    });
+
     //b.bytes = (total_bytes / i) as u64;
 }
 
@@ -278,15 +268,14 @@ fn chunk_iter_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
             chunks.next().unwrap()
         }
     };
-    c.bench_function(
-        "chunk_iter_1m_1_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("chunk_iter_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             let chunk = next();
             total_bytes += chunk.unwrap().len();
             i += 1;
         })
-    );
-    
+    });
+
     //b.bytes = (total_bytes / i) as u64;
 }
 
@@ -314,10 +303,9 @@ fn restore_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
 
     let mut total_bytes = 0;
     let mut i = 0;
-    
-    c.bench_function(
-        "restore_1m_1_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+
+    c.bench_function("restore_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             if i % chunks.len() == 0 {
                 if i != 0 {
                     let restorer_merk = restorer.take().unwrap().finalize();
@@ -335,7 +323,7 @@ fn restore_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
             total_bytes += chunk.len();
             i += 1;
         })
-    );
+    });
 
     std::fs::remove_dir_all(&path).unwrap();
 
@@ -355,19 +343,18 @@ fn checkpoint_create_destroy_1m_1_rand_rocksdb_noprune(c: &mut Criterion) {
     }
 
     let path = path + ".checkpoint";
-    c.bench_function(
-        "checkpoint_create_destroy_1m_1_rand_rocksdb_noprune",
-        |b| b.iter(|| {
+    c.bench_function("checkpoint_create_destroy_1m_1_rand_rocksdb_noprune", |b| {
+        b.iter(|| {
             let checkpoint = merk.checkpoint(&path).unwrap();
             checkpoint.destroy().unwrap();
         })
-    );
+    });
 }
 
-criterion_group!{
+criterion_group! {
     name = merk;
     config = Criterion::default();
-    targets = get_1m_rocksdb, 
+    targets = get_1m_rocksdb,
         insert_1m_2k_seq_rocksdb_noprune,
         insert_1m_2k_rand_rocksdb_noprune,
         update_1m_2k_seq_rocksdb_noprune,
diff --git a/benches/ops.rs b/benches/ops.rs
index c1e5db6e..2d75458a 100644
--- a/benches/ops.rs
+++ b/benches/ops.rs
@@ -11,14 +11,13 @@ fn insert_1m_10k_seq_memonly(c: &mut Criterion) {
     let mut tree = Owner::new(make_tree_seq(initial_size));
 
     let mut i = initial_size / batch_size;
-    c.bench_function(
-        "insert_1m_10k_seq_memonly",
-        |b| b.iter(|| {
+    c.bench_function("insert_1m_10k_seq_memonly", |b| {
+        b.iter(|| {
             let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
             tree.own(|tree| apply_memonly_unchecked(tree, &batch));
             i += 1;
         })
-    );
+    });
 }
 
 fn insert_1m_10k_rand_memonly(c: &mut Criterion) {
@@ -28,14 +27,13 @@ fn insert_1m_10k_rand_memonly(c: &mut Criterion) {
     let mut tree = Owner::new(make_tree_rand(initial_size, batch_size, 0));
 
     let mut i = initial_size / batch_size;
-    c.bench_function(
-        "insert_1m_10k_rand_memonly",
-        |b| b.iter(|| {
+    c.bench_function("insert_1m_10k_rand_memonly", |b| {
+        b.iter(|| {
             let batch = make_batch_rand(batch_size, i);
             tree.own(|tree| apply_memonly_unchecked(tree, &batch));
             i += 1;
         })
-    );
+    });
 }
 
 fn update_1m_10k_seq_memonly(c: &mut Criterion) {
@@ -45,14 +43,13 @@ fn update_1m_10k_seq_memonly(c: &mut Criterion) {
     let mut tree = Owner::new(make_tree_seq(initial_size));
 
     let mut i = 0;
-    c.bench_function(
-        "update_1m_10k_seq_memonly",
-        |b| b.iter(|| {
+    c.bench_function("update_1m_10k_seq_memonly", |b| {
+        b.iter(|| {
             let batch = make_batch_seq((i * batch_size)..((i + 1) * batch_size));
             tree.own(|tree| apply_memonly_unchecked(tree, &batch));
             i = (i + 1) % (initial_size / batch_size);
         })
-    );
+    });
 }
 
 fn update_1m_10k_rand_memonly(c: &mut Criterion) {
@@ -62,17 +59,16 @@ fn update_1m_10k_rand_memonly(c: &mut Criterion) {
     let mut tree = Owner::new(make_tree_rand(initial_size, batch_size, 0));
 
     let mut i = 0;
-    c.bench_function(
-        "update_1m_10k_rand_memonly",
-        |b| b.iter(|| {
+    c.bench_function("update_1m_10k_rand_memonly", |b| {
+        b.iter(|| {
             let batch = make_batch_rand(batch_size, i);
             tree.own(|tree| apply_memonly_unchecked(tree, &batch));
             i = (i + 1) % (initial_size / batch_size);
         })
-    );
+    });
 }
 
-criterion_group!{
+criterion_group! {
     name = ops;
     config = Criterion::default();
     targets =  insert_1m_10k_seq_memonly,