Skip to content

Commit

Permalink
Factour out the request for table schema, and remove some duplicate l…
Browse files Browse the repository at this point in the history 
…ogic
  • Loading branch information
@mamcx
mamcx committed Oct 5, 2023
1 parent 0a4fe85 commit 8d4da7c
Show file tree
Hide file tree
Showing 9 changed files with 153 additions and 96 deletions.
110 changes: 64 additions & 46 deletions crates/bench/benches/generic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use criterion::{
measurement::{Measurement, WallTime},
Bencher, BenchmarkGroup, Criterion,
};
use spacetimedb::db::datastore::traits::TableSchema;
use spacetimedb_bench::{
database::BenchDatabase,
schemas::{create_sequential, BenchTable, IndexStrategy, Location, Person, RandomTable, BENCH_PKEY_INDEX},
Expand Down Expand Up @@ -230,13 +231,15 @@ fn sql_select<DB: BenchDatabase, T: BenchTable + RandomTable>(
// though it iterates across many rows.
g.throughput(criterion::Throughput::Elements(1));

let table = db.get_table::<T>(table_id)?;

g.bench_function(&id, |b| {
bench_harness(
b,
db,
|_| Ok(()),
|db, _| {
db.sql_select(table_id)?;
db.sql_select(&table)?;
Ok(())
},
)
Expand Down Expand Up @@ -279,15 +282,17 @@ fn iterate<DB: BenchDatabase, T: BenchTable + RandomTable>(

/// Implements both "filter" and "find" benchmarks.
#[inline(never)]
fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
#[allow(clippy::too_many_arguments)]
fn _filter_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
bench_name: &str,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_index: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
) -> ResultBench<(String, TableSchema, Vec<T>)> {
let filter_column_type = match &T::product_type().elements[column_index as usize].algebraic_type {
AlgebraicType::Builtin(BuiltinType::String) => "string",
AlgebraicType::Builtin(BuiltinType::U32) => "u32",
Expand All @@ -300,7 +305,7 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
IndexStrategy::NonUnique => "non_indexed",
_ => unimplemented!(),
};
let id = format!("filter/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");
let id = format!("{bench_name}/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

Expand All @@ -309,6 +314,24 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));

let table = db.get_table::<T>(table_id)?;

Ok((id, table, data))
}

#[inline(never)]
fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_index: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
let (id, table, data) =
_filter_setup::<DB, T>(g, db, "filter", table_id, index_strategy, column_index, load, buckets)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all databases have EXACTLY the same sample data.
Expand All @@ -325,7 +348,7 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(value)
},
|db, value| {
db.filter::<T>(table_id, column_index, value)?;
db.sql_where::<T>(&table, column_index, value)?;
Ok(())
},
)
Expand All @@ -334,7 +357,6 @@ fn filter<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(())
}

/// Implements both "filter" and "find" benchmarks.
#[inline(never)]
fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
Expand All @@ -345,26 +367,16 @@ fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(
load: u32,
buckets: u32,
) -> ResultBench<()> {
let filter_column_type = match &T::product_type().elements[column_index as usize].algebraic_type {
AlgebraicType::Builtin(BuiltinType::String) => "string",
AlgebraicType::Builtin(BuiltinType::U32) => "u32",
AlgebraicType::Builtin(BuiltinType::U64) => "u64",
_ => unimplemented!(),
};
let mean_result_count = load / buckets;
let indexed = match index_strategy {
IndexStrategy::MultiIndex => "indexed",
IndexStrategy::NonUnique => "non_indexed",
_ => unimplemented!(),
};
let id = format!("sql_where/{filter_column_type}/{indexed}/load={load}/count={mean_result_count}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

db.insert_bulk(table_id, data.clone())?;

// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));
let (id, table, data) = _filter_setup::<DB, T>(
g,
db,
"sql_where",
table_id,
index_strategy,
column_index,
load,
buckets,
)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
Expand All @@ -382,7 +394,7 @@ fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(value)
},
|db, value| {
db.sql_where::<T>(table_id, column_index, value)?;
db.sql_where::<T>(&table, column_index, value)?;
Ok(())
},
)
Expand All @@ -393,21 +405,21 @@ fn sql_where<DB: BenchDatabase, T: BenchTable + RandomTable>(

/// Implements both "filter" and "find" benchmarks.
#[inline(never)]
fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
fn _find_setup<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
bench_name: &str,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_id: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
) -> ResultBench<(String, TableSchema, Vec<T>)> {
assert_eq!(
*index_strategy,
IndexStrategy::Unique,
"find benchmarks require unique key"
);
let id = format!("find_unique/u32/load={load}");
let id = format!("{bench_name}/u32/load={load}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

Expand All @@ -416,6 +428,23 @@ fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));

let table = db.get_table::<T>(table_id)?;

Ok((id, table, data))
}

#[inline(never)]
fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
g: &mut Group,
db: &mut DB,
table_id: &DB::TableId,
index_strategy: &IndexStrategy,
column_id: u32,
load: u32,
buckets: u32,
) -> ResultBench<()> {
let (id, table, data) = _find_setup::<DB, T>(g, db, "find_unique", table_id, index_strategy, load, buckets)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
// Note that all benchmarks use exactly the same sample data.
Expand All @@ -431,7 +460,7 @@ fn find<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(value)
},
|db, value| {
db.filter::<T>(table_id, column_id, value)?;
db.filter::<T>(&table, column_id, value)?;
Ok(())
},
)
Expand All @@ -451,19 +480,8 @@ fn sql_find<DB: BenchDatabase, T: BenchTable + RandomTable>(
load: u32,
buckets: u32,
) -> ResultBench<()> {
assert_eq!(
*index_strategy,
IndexStrategy::Unique,
"find benchmarks require unique key"
);
let id = format!("sql_where_find_unique/u32/load={load}");

let data = create_sequential::<T>(0xdeadbeef, load, buckets as u64);

db.insert_bulk(table_id, data.clone())?;

// Each iteration performs a single transaction.
g.throughput(criterion::Throughput::Elements(1));
let (id, table, data) =
_find_setup::<DB, T>(g, db, "sql_where_find_unique", table_id, index_strategy, load, buckets)?;

// We loop through all buckets found in the sample data.
// This mildly increases variance on the benchmark, but makes "mean_result_count" more accurate.
Expand All @@ -480,7 +498,7 @@ fn sql_find<DB: BenchDatabase, T: BenchTable + RandomTable>(
Ok(value)
},
|db, value| {
db.sql_where::<T>(table_id, column_id, value)?;
db.sql_where::<T>(&table, column_id, value)?;
Ok(())
},
)
Expand Down
10 changes: 7 additions & 3 deletions crates/bench/src/database.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use spacetimedb::db::datastore::traits::TableSchema;
use spacetimedb_lib::AlgebraicValue;

use crate::schemas::{BenchTable, IndexStrategy};
Expand All @@ -19,6 +20,9 @@ pub trait BenchDatabase: Sized {

fn create_table<T: BenchTable>(&mut self, table_style: IndexStrategy) -> ResultBench<Self::TableId>;

/// Return table metadata so we can remove this from the hot path
fn get_table<T: BenchTable>(&mut self, table_id: &Self::TableId) -> ResultBench<TableSchema>;

/// Should not drop the table, only delete all the rows.
fn clear_table(&mut self, table_id: &Self::TableId) -> ResultBench<()>;

Expand All @@ -41,20 +45,20 @@ pub trait BenchDatabase: Sized {
/// Filter the table on the specified column index for the specified value.
fn filter<T: BenchTable>(
&mut self,
table_id: &Self::TableId,
table: &TableSchema,
column_index: u32,
value: AlgebraicValue,
) -> ResultBench<()>;

/// Perform a `SELECT * FROM table`
/// Note: this can be non-generic because none of the implementations use the relevant generic argument.
fn sql_select(&mut self, table_id: &Self::TableId) -> ResultBench<()>;
fn sql_select(&mut self, table: &TableSchema) -> ResultBench<()>;

/// Perform a `SELECT * FROM table WHERE column = value`
/// Note: this can be non-generic because none of the implementations use the relevant generic argument.
fn sql_where<T: BenchTable>(
&mut self,
table_id: &Self::TableId,
table: &TableSchema,
column_index: u32,
value: AlgebraicValue,
) -> ResultBench<()>;
Expand Down
28 changes: 28 additions & 0 deletions crates/bench/src/lib.rs
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
use crate::schemas::BenchTable;
use spacetimedb::db::datastore::traits::{ColumnSchema, TableSchema};
use spacetimedb_lib::auth::{StAccess, StTableType};

pub mod database;
pub mod schemas;
pub mod spacetime_module;
Expand All @@ -6,6 +10,30 @@ pub mod sqlite;

pub type ResultBench<T> = Result<T, anyhow::Error>;

pub(crate) fn create_schema<T: BenchTable>(table_name: &str) -> TableSchema {
let columns = T::product_type()
.elements
.into_iter()
.enumerate()
.map(|(pos, col)| ColumnSchema {
table_id: 0,
col_id: pos as u32,
col_name: col.name.unwrap(),
col_type: col.algebraic_type,
is_autoinc: false,
});

TableSchema {
table_id: 0,
table_name: table_name.to_string(),
indexes: vec![],
columns: columns.collect(),
constraints: vec![],
table_type: StTableType::System,
table_access: StAccess::Public,
}
}

#[cfg(test)]
mod tests {
use crate::{
Expand Down
29 changes: 15 additions & 14 deletions crates/bench/src/spacetime_module.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
use spacetimedb::db::datastore::traits::TableSchema;
use spacetimedb::db::{Config, FsyncPolicy, Storage};
use spacetimedb_lib::sats::BuiltinValue;
use spacetimedb_lib::{sats::ArrayValue, AlgebraicValue, ProductValue};
use spacetimedb_testing::modules::{start_runtime, CompiledModule, ModuleHandle};
use tokio::runtime::Runtime;

use crate::{
create_schema,
database::BenchDatabase,
schemas::{snake_case_table_name, table_name, BenchTable},
ResultBench,
Expand Down Expand Up @@ -83,6 +85,10 @@ impl BenchDatabase for SpacetimeModule {
})
}

fn get_table<T: BenchTable>(&mut self, table_id: &Self::TableId) -> ResultBench<TableSchema> {
Ok(create_schema::<T>(&table_id.pascal_case))
}

fn clear_table(&mut self, table_id: &Self::TableId) -> ResultBench<()> {
let SpacetimeModule { runtime, module } = self;
let module = module.as_mut().unwrap();
Expand Down Expand Up @@ -174,16 +180,16 @@ impl BenchDatabase for SpacetimeModule {

fn filter<T: BenchTable>(
&mut self,
table_id: &Self::TableId,
table: &TableSchema,
column_index: u32,
value: AlgebraicValue,
) -> ResultBench<()> {
let SpacetimeModule { runtime, module } = self;
let module = module.as_mut().unwrap();

let product_type = T::product_type();
let column_name = product_type.elements[column_index as usize].name.as_ref().unwrap();
let reducer_name = format!("filter_{}_by_{}", table_id.snake_case, column_name);
//let product_type = T::product_type();
let column_name = &table.columns[column_index as usize].col_name;
let reducer_name = format!("filter_{}_by_{}", table.table_name, column_name);

runtime.block_on(async move {
module
Expand All @@ -193,11 +199,10 @@ impl BenchDatabase for SpacetimeModule {
})
}

fn sql_select(&mut self, table_id: &Self::TableId) -> ResultBench<()> {
fn sql_select(&mut self, table: &TableSchema) -> ResultBench<()> {
let SpacetimeModule { runtime, module } = self;
let module = module.as_mut().unwrap();
let table_name = &table_id.pascal_case;
let sql = format!("SELECT * FROM {table_name}");
let sql = format!("SELECT * FROM {}", table.table_name);
let id = module.client.id.identity;
runtime.block_on(async move {
module.client.module.one_off_query(id, sql).await?;
Expand All @@ -207,15 +212,11 @@ impl BenchDatabase for SpacetimeModule {

fn sql_where<T: BenchTable>(
&mut self,
table_id: &Self::TableId,
table: &TableSchema,
column_index: u32,
value: AlgebraicValue,
) -> ResultBench<()> {
let column = T::product_type()
.elements
.swap_remove(column_index as usize)
.name
.unwrap();
let column = &table.columns[column_index as usize].col_name;

let value = match value.as_builtin().unwrap() {
BuiltinValue::U32(x) => x.to_string(),
Expand All @@ -228,7 +229,7 @@ impl BenchDatabase for SpacetimeModule {

let SpacetimeModule { runtime, module } = self;
let module = module.as_mut().unwrap();
let table_name = &table_id.pascal_case;
let table_name = &table.table_name;
let sql_query = format!("SELECT * FROM {table_name} WHERE {column} = {value}");

let id = module.client.id.identity;
Expand Down
Loading

0 comments on commit 8d4da7c

Please sign in to comment.