Bench for sql scan

Cargo.toml

@@ -190,7 +190,7 @@ wasmtime = { version = "7", default-features = false, features = ["cranelift"] }
 # We use the "ondemand" feature to allow connecting after the start,
 # and reconnecting, from the tracy client to the database.
 # TODO(George): Need to be able to remove "broadcast" in some build configurations.
-tracing-tracy = { version = "0.10.2", features = [
+tracing-tracy = { version = "0.10.3", features = [
   "enable",
   "system-tracing",
   "context-switch-tracing",

crates/bench/benches/generic.rs

@@ -3,6 +3,7 @@ use criterion::{
     measurement::{Measurement, WallTime},
     Bencher, BenchmarkGroup, Criterion,
 };
+use spacetimedb::db::datastore::traits::TableSchema;
 use spacetimedb_bench::{
     database::BenchDatabase,
     schemas::{create_sequential, BenchTable, IndexStrategy, Location, Person, RandomTable, BENCH_PKEY_INDEX},
@@ -69,13 +70,16 @@ fn table_suite<DB: BenchDatabase, T: BenchTable + RandomTable>(g: &mut Group, db
     for (index_strategy, table_id, table_params) in &tables {
         if *index_strategy == IndexStrategy::Unique {
             iterate::<DB, T>(g, table_params, db, table_id, 100)?;
+            sql_select::<DB, T>(g, table_params, db, table_id, 100)?;
 
             if table_params.contains("person") {
                 // perform "find" benchmarks
+                sql_find::<DB, T>(g, db, table_id, index_strategy, BENCH_PKEY_INDEX, 1000, 100)?;
                 find::<DB, T>(g, db, table_id, index_strategy, BENCH_PKEY_INDEX, 1000, 100)?;
             }
         } else {
             // perform "filter" benchmarks
+            sql_where::<DB, T>(g, db, table_id, index_strategy, 1, 1000, 100)?;
             filter::<DB, T>(g, db, table_id, index_strategy, 1, 1000, 100)?;
         }
     }
@@ -122,7 +126,7 @@ fn bench_harness<
 
 #[inline(never)]
 fn empty<DB: BenchDatabase>(g: &mut Group, db: &mut DB) -> ResultBench<()> {
-    let id = format!("empty");
+    let id = "empty".to_string();
     g.bench_function(&id, |b| {
         bench_harness(
             b,
@@ -159,11 +163,11 @@ fn insert_1<DB: BenchDatabase, T: BenchTable + RandomTable>(
                 let mut data = data.clone();
                 db.clear_table(table_id)?;
                 let row = data.pop().unwrap();
-                db.insert_bulk(&table_id, data)?;
+                db.insert_bulk(table_id, data)?;
                 Ok(row)
             },
             |db, row| {
-                db.insert(&table_id, row)?;
+                db.insert(table_id, row)?;
                 Ok(())
             },
         )
@@ -196,12 +200,46 @@ fn insert_bulk<DB: BenchDatabase, T: BenchTable + RandomTable>(
                 db.clear_table(table_id)?;
                 let to_insert = data.split_off(load as usize);
                 if !data.is_empty() {
-                    db.insert_bulk(&table_id, data)?;
+                    db.insert_bulk(table_id, data)?;
                 }
                 Ok(to_insert)
             },
             |db, to_insert| {
-                db.insert_bulk(&table_id, to_insert)?;
+                db.insert_bulk(table_id, to_insert)?;
+                Ok(())
+            },
+        )
+    });
+    db.clear_table(table_id)?;
+    Ok(())
+}
+
+#[inline(never)]
+fn sql_select<DB: BenchDatabase, T: BenchTable + RandomTable>(
+    g: &mut Group,
+    table_params: &str,
+    db: &mut DB,
+    table_id: &DB::TableId,
+    count: u32,
+) -> ResultBench<()> {
+    let id = format!("sql_select/{table_params}/count={count}");
+    let data = create_sequential::<T>(0xdeadbeef, count, 1000);
+
+    db.insert_bulk(table_id, data)?;
+
+    // Each iteration performs a single transaction,
+    // though it iterates across many rows.
+    g.throughput(criterion::Throughput::Elements(1));
+
+    let table = db.get_table::<T>(table_id)?;
+
+    g.bench_function(&id, |b| {
+        bench_harness(
+            b,
+            db,
+            |_| Ok(()),
+            |db, _| {
+                db.sql_select(&table)?;
                 Ok(())
             },
         )
@@ -244,15 +282,17 @@ fn iterate<DB: BenchDatabase, T: BenchTable + RandomTable>(
 
 /// Implements both "filter" and "find" benchmarks.
 #[inline(never)]
-fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
+#[allow(clippy::too_many_arguments)]
+fn _filter_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
     g: &mut Group,
     db: &mut DB,
+    bench_name: &str,
     table_id: &DB::TableId,
     index_strategy: &IndexStrategy,
     column_index: u32,
     load: u32,
     buckets: u32,
-) -> ResultBench<()> {
+) -> ResultBench<(String, TableSchema, Vec<T>)> {
     let filter_column_type = match &T::product_type().elements[column_index as usize].algebraic_type {
         AlgebraicType::Builtin(BuiltinType::String) => "string",
         AlgebraicType::Builtin(BuiltinType::U32) => "u32",
@@ -265,15 +305,33 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
         IndexStrategy::NonUnique => "non_indexed",
         _ => unimplemented!(),
     };
-    let id = format!("filter/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");
+    let id = format!("{bench_name}/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");
 
     let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);
 
-    db.insert_bulk(&table_id, data.clone())?;
+    db.insert_bulk(table_id, data.clone())?;
 
     // Each iteration performs a single transaction.
     g.throughput(criterion::Throughput::Elements(1));
 
+    let table = db.get_table::<T>(table_id)?;
+
+    Ok((id, table, data))
+}
+
+#[inline(never)]
+fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
+    g: &mut Group,
+    db: &mut DB,
+    table_id: &DB::TableId,
+    index_strategy: &IndexStrategy,
+    column_index: u32,
+    load: u32,
+    buckets: u32,
+) -> ResultBench<()> {
+    let (id, table, data) =
+        _filter_setup::<DB, T>(g, db, "filter", table_id, index_strategy, column_index, load, buckets)?;
+
     // We loop through all buckets found in the sample data.
     // This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
     // Note that all databases have EXACTLY the same sample data.
@@ -290,7 +348,7 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
                 Ok(value)
             },
             |db, value| {
-                db.filter::<T>(&table_id, column_index, value)?;
+                db.sql_where::<T>(&table, column_index, value)?;
                 Ok(())
             },
         )
@@ -299,31 +357,132 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
     Ok(())
 }
 
-/// Implements both "filter" and "find" benchmarks.
 #[inline(never)]
-fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
+fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(
     g: &mut Group,
     db: &mut DB,
     table_id: &DB::TableId,
     index_strategy: &IndexStrategy,
-    column_id: u32,
+    column_index: u32,
     load: u32,
     buckets: u32,
 ) -> ResultBench<()> {
+    let (id, table, data) = _filter_setup::<DB, T>(
+        g,
+        db,
+        "sql_where",
+        table_id,
+        index_strategy,
+        column_index,
+        load,
+        buckets,
+    )?;
+
+    // We loop through all buckets found in the sample data.
+    // This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
+    // Note that all databases have EXACTLY the same sample data.
+    let mut i = 0;
+
+    g.bench_function(&id, |b| {
+        bench_harness(
+            b,
+            db,
+            |_| {
+                // pick something to look for
+                let value = data[i].clone().into_product_value().elements[column_index as usize].clone();
+                i = (i + 1) % load as usize;
+                Ok(value)
+            },
+            |db, value| {
+                db.sql_where::<T>(&table, column_index, value)?;
+                Ok(())
+            },
+        )
+    });
+    db.clear_table(table_id)?;
+    Ok(())
+}
+
+/// Implements both "filter" and "find" benchmarks.
+#[inline(never)]
+fn _find_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
+    g: &mut Group,
+    db: &mut DB,
+    bench_name: &str,
+    table_id: &DB::TableId,
+    index_strategy: &IndexStrategy,
+    load: u32,
+    buckets: u32,
+) -> ResultBench<(String, TableSchema, Vec<T>)> {
     assert_eq!(
         *index_strategy,
         IndexStrategy::Unique,
         "find benchmarks require unique key"
     );
-    let id = format!("find_unique/u32/load={load}");
+    let id = format!("{bench_name}/u32/load={load}");
 
     let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);
 
-    db.insert_bulk(&table_id, data.clone())?;
+    db.insert_bulk(table_id, data.clone())?;
 
     // Each iteration performs a single transaction.
     g.throughput(criterion::Throughput::Elements(1));
 
+    let table = db.get_table::<T>(table_id)?;
+
+    Ok((id, table, data))
+}
+
+#[inline(never)]
+fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
+    g: &mut Group,
+    db: &mut DB,
+    table_id: &DB::TableId,
+    index_strategy: &IndexStrategy,
+    column_id: u32,
+    load: u32,
+    buckets: u32,
+) -> ResultBench<()> {
+    let (id, table, data) = _find_setup::<DB, T>(g, db, "find_unique", table_id, index_strategy, load, buckets)?;
+
+    // We loop through all buckets found in the sample data.
+    // This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
+    // Note that all benchmarks use exactly the same sample data.
+    let mut i = 0;
+
+    g.bench_function(&id, |b| {
+        bench_harness(
+            b,
+            db,
+            |_| {
+                let value = data[i].clone().into_product_value().elements[column_id as usize].clone();
+                i = (i + 1) % load as usize;
+                Ok(value)
+            },
+            |db, value| {
+                db.filter::<T>(&table, column_id, value)?;
+                Ok(())
+            },
+        )
+    });
+    db.clear_table(table_id)?;
+    Ok(())
+}
+
+/// Implements both "filter" and "find" benchmarks.
+#[inline(never)]
+fn sql_find<DB: BenchDatabase, T: BenchTable + RandomTable>(
+    g: &mut Group,
+    db: &mut DB,
+    table_id: &DB::TableId,
+    index_strategy: &IndexStrategy,
+    column_id: u32,
+    load: u32,
+    buckets: u32,
+) -> ResultBench<()> {
+    let (id, table, data) =
+        _find_setup::<DB, T>(g, db, "sql_where_find_unique", table_id, index_strategy, load, buckets)?;
+
     // We loop through all buckets found in the sample data.
     // This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
     // Note that all benchmarks use exactly the same sample data.
@@ -339,7 +498,7 @@ fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
                 Ok(value)
             },
             |db, value| {
-                db.filter::<T>(&table_id, column_id, value)?;
+                db.sql_where::<T>(&table, column_id, value)?;
                 Ok(())
             },
         )

crates/bench/src/database.rs

@@ -1,3 +1,4 @@
+use spacetimedb::db::datastore::traits::TableSchema;
 use spacetimedb_lib::AlgebraicValue;
 
 use crate::schemas::{BenchTable, IndexStrategy};
@@ -19,6 +20,9 @@ pub trait BenchDatabase: Sized {
 
     fn create_table<T: BenchTable>(&mut self, table_style: IndexStrategy) -> ResultBench<Self::TableId>;
 
+    /// Return table metadata so we can remove this from the hot path
+    fn get_table<T: BenchTable>(&mut self, table_id: &Self::TableId) -> ResultBench<TableSchema>;
+
     /// Should not drop the table, only delete all the rows.
     fn clear_table(&mut self, table_id: &Self::TableId) -> ResultBench<()>;
 
@@ -41,7 +45,20 @@ pub trait BenchDatabase: Sized {
     /// Filter the table on the specified column index for the specified value.
     fn filter<T: BenchTable>(
         &mut self,
-        table_id: &Self::TableId,
+        table: &TableSchema,
+        column_index: u32,
+        value: AlgebraicValue,
+    ) -> ResultBench<()>;
+
+    /// Perform a `SELECT * FROM table`
+    /// Note: this can be non-generic because none of the implementations use the relevant generic argument.
+    fn sql_select(&mut self, table: &TableSchema) -> ResultBench<()>;
+
+    /// Perform a `SELECT * FROM table WHERE column = value`
+    /// Note: this can be non-generic because none of the implementations use the relevant generic argument.
+    fn sql_where<T: BenchTable>(
+        &mut self,
+        table: &TableSchema,
         column_index: u32,
         value: AlgebraicValue,
     ) -> ResultBench<()>;

crates/bench/src/lib.rs

@@ -1,3 +1,7 @@
+use crate::schemas::BenchTable;
+use spacetimedb::db::datastore::traits::{ColumnSchema, TableSchema};
+use spacetimedb_lib::auth::{StAccess, StTableType};
+
 pub mod database;
 pub mod schemas;
 pub mod spacetime_module;
@@ -6,6 +10,30 @@ pub mod sqlite;
 
 pub type ResultBench<T> = Result<T, anyhow::Error>;
 
+pub(crate) fn create_schema<T: BenchTable>(table_name: &str) -> TableSchema {
+    let columns = T::product_type()
+        .elements
+        .into_iter()
+        .enumerate()
+        .map(|(pos, col)| ColumnSchema {
+            table_id: 0,
+            col_id: pos as u32,
+            col_name: col.name.unwrap(),
+            col_type: col.algebraic_type,
+            is_autoinc: false,
+        });
+
+    TableSchema {
+        table_id: 0,
+        table_name: table_name.to_string(),
+        indexes: vec![],
+        columns: columns.collect(),
+        constraints: vec![],
+        table_type: StTableType::System,
+        table_access: StAccess::Public,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::{