Run and test benchmarks against C# as well (#1965)

Co-authored-by: Zeke Foppa <196249+bfops@users.noreply.github.com>

.github/workflows/benchmarks.yml

@@ -76,7 +76,7 @@ jobs:
           # summary PR" step). otherwise, we can use a fully shallow checkout
           fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
 
-      - name: Install stable toolchain
+      - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -85,6 +85,13 @@ jobs:
           target: wasm32-unknown-unknown
           override: true
 
+      - name: Install .NET toolchain
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: "8.x"
+        env:
+          DOTNET_INSTALL_DIR: ~/.dotnet
+
       - name: Build
         working-directory: crates/bench/
         run: |
@@ -114,7 +121,7 @@ jobs:
           rm -rf .spacetime
           cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
           # sticker price benchmark
-          cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" 'stdb_module/disk/update_bulk'
+          cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" 'stdb_module/.*/disk/update_bulk'
           cargo bench --bench special -- --save-baseline "$BASELINE_NAME"
           cargo run --bin summarize pack "$BASELINE_NAME"
           popd
@@ -192,7 +199,7 @@ jobs:
       - name: Install valgrind & iai-callgrind-runner
         run: |
           apt-get update
-          apt-get install -y valgrind protobuf-compiler bash sudo curl gh 
+          apt-get install -y valgrind protobuf-compiler bash sudo curl gh
           cargo install --git https://github.com/clockworklabs/iai-callgrind.git --branch main iai-callgrind-runner
           git config --global --add safe.directory /__w/SpacetimeDB/SpacetimeDB
 
@@ -343,4 +350,3 @@ jobs:
           echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
           echo "This is necessary because we are running as root inside a docker image"
           chmod -R a+rw .
-

crates/bench/Cargo.toml

@@ -54,6 +54,7 @@ regex.workspace = true
 rusqlite.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+serial_test.workspace = true
 tempdir.workspace = true
 tokio.workspace = true
 tracing-subscriber.workspace = true

crates/bench/benches/generic.rs

@@ -12,6 +12,7 @@ use spacetimedb_bench::{
 };
 use spacetimedb_lib::sats::AlgebraicType;
 use spacetimedb_primitives::ColId;
+use spacetimedb_testing::modules::{Csharp, Rust};
 
 #[global_allocator]
 static GLOBAL: MiMalloc = MiMalloc;
@@ -23,11 +24,13 @@ lazy_static! {
 fn criterion_benchmark(c: &mut Criterion) {
     bench_suite::<sqlite::SQLite>(c, true).unwrap();
     bench_suite::<spacetime_raw::SpacetimeRaw>(c, true).unwrap();
-    bench_suite::<spacetime_module::SpacetimeModule>(c, true).unwrap();
+    bench_suite::<spacetime_module::SpacetimeModule<Rust>>(c, true).unwrap();
+    bench_suite::<spacetime_module::SpacetimeModule<Csharp>>(c, true).unwrap();
 
     bench_suite::<sqlite::SQLite>(c, false).unwrap();
     bench_suite::<spacetime_raw::SpacetimeRaw>(c, false).unwrap();
-    bench_suite::<spacetime_module::SpacetimeModule>(c, false).unwrap();
+    bench_suite::<spacetime_module::SpacetimeModule<Rust>>(c, false).unwrap();
+    bench_suite::<spacetime_module::SpacetimeModule<Csharp>>(c, false).unwrap();
 }
 
 #[inline(never)]

crates/bench/benches/special.rs

@@ -1,5 +1,5 @@
 use criterion::async_executor::AsyncExecutor;
-use criterion::{criterion_group, criterion_main, Criterion};
+use criterion::{criterion_group, criterion_main, Criterion, SamplingMode};
 use mimalloc::MiMalloc;
 use spacetimedb_bench::{
     database::BenchDatabase,
@@ -9,6 +9,7 @@ use spacetimedb_bench::{
 use spacetimedb_lib::sats::{self, bsatn};
 use spacetimedb_lib::{bsatn::ToBsatn as _, ProductValue};
 use spacetimedb_schema::schema::TableSchema;
+use spacetimedb_testing::modules::{Csharp, ModuleLanguage, Rust};
 use std::sync::Arc;
 use std::sync::OnceLock;
 
@@ -20,14 +21,19 @@ fn criterion_benchmark(c: &mut Criterion) {
     serialize_benchmarks::<u32_u64_u64>(c);
     serialize_benchmarks::<u64_u64_u32>(c);
 
-    let db = SpacetimeModule::build(true).unwrap();
+    custom_benchmarks::<Rust>(c);
+    custom_benchmarks::<Csharp>(c);
+}
+
+fn custom_benchmarks<L: ModuleLanguage>(c: &mut Criterion) {
+    let db = SpacetimeModule::<L>::build(true).unwrap();
 
     custom_module_benchmarks(&db, c);
     custom_db_benchmarks(&db, c);
 }
 
-fn custom_module_benchmarks(m: &SpacetimeModule, c: &mut Criterion) {
-    let mut group = c.benchmark_group("special/stdb_module");
+fn custom_module_benchmarks<L: ModuleLanguage>(m: &SpacetimeModule<L>, c: &mut Criterion) {
+    let mut group = c.benchmark_group(format!("special/{}", SpacetimeModule::<L>::name()));
 
     let args = sats::product!["0".repeat(65536).into_boxed_str()];
     group.bench_function("large_arguments/64KiB", |b| {
@@ -44,10 +50,11 @@ fn custom_module_benchmarks(m: &SpacetimeModule, c: &mut Criterion) {
     }
 }
 
-fn custom_db_benchmarks(m: &SpacetimeModule, c: &mut Criterion) {
-    let mut group = c.benchmark_group("special/db_game");
+fn custom_db_benchmarks<L: ModuleLanguage>(m: &SpacetimeModule<L>, c: &mut Criterion) {
+    let mut group = c.benchmark_group(format!("special/db_game/{}", L::NAME));
     // This bench take long, so adjust for it
     group.sample_size(10);
+    group.sampling_mode(SamplingMode::Flat);
 
     let init_db: OnceLock<()> = OnceLock::new();
     for n in [10, 100] {
@@ -69,14 +76,14 @@ fn custom_db_benchmarks(m: &SpacetimeModule, c: &mut Criterion) {
     }
 
     let init_db: OnceLock<()> = OnceLock::new();
-    for n in [500, 5_000] {
+    for n in [10, 100] {
         let args = sats::product![n];
         group.bench_function(format!("ia_loop/load={n}"), |b| {
             // Initialize outside the benchmark so the db is seed once, to avoid `unique` constraints violations
             init_db.get_or_init(|| {
                 m.block_on(async {
                     m.module
-                        .call_reducer_binary("init_game_ia_loop", &sats::product![5_000])
+                        .call_reducer_binary("init_game_ia_loop", &sats::product![500])
                         .await
                         .unwrap();
                 })

crates/bench/src/database.rs

@@ -10,7 +10,7 @@ use crate::ResultBench;
 ///
 /// Not all benchmarks have to go through this trait.
 pub trait BenchDatabase: Sized {
-    fn name() -> &'static str;
+    fn name() -> String;
 
     type TableId: Clone + 'static;
 

crates/bench/src/lib.rs

@@ -16,6 +16,8 @@ mod tests {
         sqlite::SQLite,
         ResultBench,
     };
+    use serial_test::serial;
+    use spacetimedb_testing::modules::{Csharp, Rust};
     use std::{io, path::Path, sync::Once};
     use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
 
@@ -101,30 +103,37 @@ mod tests {
         Ok(())
     }
 
+    fn test_basic_invariants<DB: BenchDatabase>() -> ResultBench<()> {
+        basic_invariants::<DB, u32_u64_str>(IndexStrategy::Unique0, true)?;
+        basic_invariants::<DB, u32_u64_u64>(IndexStrategy::Unique0, true)?;
+        basic_invariants::<DB, u32_u64_str>(IndexStrategy::BTreeEachColumn, true)?;
+        basic_invariants::<DB, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true)?;
+        Ok(())
+    }
+
     #[test]
-    fn test_basic_invariants_sqlite() {
-        basic_invariants::<SQLite, u32_u64_str>(IndexStrategy::Unique0, true).unwrap();
-        basic_invariants::<SQLite, u32_u64_u64>(IndexStrategy::Unique0, true).unwrap();
-        basic_invariants::<SQLite, u32_u64_str>(IndexStrategy::BTreeEachColumn, true).unwrap();
-        basic_invariants::<SQLite, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true).unwrap();
+    fn test_basic_invariants_sqlite() -> ResultBench<()> {
+        test_basic_invariants::<SQLite>()
     }
 
     #[test]
-    fn test_basic_invariants_spacetime_raw() {
-        basic_invariants::<SpacetimeRaw, u32_u64_str>(IndexStrategy::Unique0, true).unwrap();
-        basic_invariants::<SpacetimeRaw, u32_u64_u64>(IndexStrategy::Unique0, true).unwrap();
-        basic_invariants::<SpacetimeRaw, u32_u64_str>(IndexStrategy::BTreeEachColumn, true).unwrap();
-        basic_invariants::<SpacetimeRaw, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true).unwrap();
+    fn test_basic_invariants_spacetime_raw() -> ResultBench<()> {
+        test_basic_invariants::<SpacetimeRaw>()
+    }
+
+    // note: there can only be one #[test] invoking spacetime module stuff.
+    // #[test]s run concurrently and they fight over lockfiles.
+    // so, run the sub-tests here in sequence.
+
+    #[test]
+    #[serial]
+    fn test_basic_invariants_spacetime_module_rust() -> ResultBench<()> {
+        test_basic_invariants::<SpacetimeModule<Rust>>()
     }
 
     #[test]
-    fn test_basic_invariants_spacetime_module() {
-        // note: there can only be one #[test] invoking spacetime module stuff.
-        // #[test]s run concurrently and they fight over lockfiles.
-        // so, run the sub-tests here in sequence.
-        basic_invariants::<SpacetimeModule, u32_u64_str>(IndexStrategy::Unique0, true).unwrap();
-        basic_invariants::<SpacetimeModule, u32_u64_u64>(IndexStrategy::Unique0, true).unwrap();
-        basic_invariants::<SpacetimeModule, u32_u64_str>(IndexStrategy::BTreeEachColumn, true).unwrap();
-        basic_invariants::<SpacetimeModule, u32_u64_u64>(IndexStrategy::BTreeEachColumn, true).unwrap();
+    #[serial]
+    fn test_basic_invariants_spacetime_module_csharp() -> ResultBench<()> {
+        test_basic_invariants::<SpacetimeModule<Csharp>>()
     }
 }

crates/bench/src/spacetime_module.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{marker::PhantomData, path::Path};
 
 use spacetimedb::db::{Config, Storage};
 use spacetimedb_lib::{
@@ -7,7 +7,7 @@ use spacetimedb_lib::{
 };
 use spacetimedb_paths::RootDir;
 use spacetimedb_primitives::ColId;
-use spacetimedb_testing::modules::{start_runtime, CompilationMode, CompiledModule, LoggerRecord, ModuleHandle};
+use spacetimedb_testing::modules::{start_runtime, LoggerRecord, ModuleHandle, ModuleLanguage};
 use tokio::runtime::Runtime;
 
 use crate::{
@@ -17,23 +17,6 @@ use crate::{
 };
 use criterion::async_executor::AsyncExecutor;
 
-lazy_static::lazy_static! {
-    pub static ref BENCHMARKS_MODULE: CompiledModule = {
-        if std::env::var_os("STDB_BENCH_CS").is_some() {
-            CompiledModule::compile("benchmarks-cs", CompilationMode::Release)
-        } else {
-            // Temporarily add CARGO_TARGET_DIR override to avoid conflicts with main target dir.
-            // Otherwise for some reason Cargo will mark all dependencies with build scripts as
-            // fresh - but only if running benchmarks (if modules are built in release mode).
-            // See https://github.com/clockworklabs/SpacetimeDB/issues/401.
-            std::env::set_var("CARGO_TARGET_DIR", concat!(env!("CARGO_MANIFEST_DIR"), "/target"));
-            let module = CompiledModule::compile("benchmarks", CompilationMode::Release);
-            std::env::remove_var("CARGO_TARGET_DIR");
-            module
-        }
-    };
-}
-
 /// A benchmark backend that invokes a spacetime module.
 ///
 /// This is tightly tied to the file `modules/benchmarks/src/lib.rs`;
@@ -42,26 +25,27 @@ lazy_static::lazy_static! {
 ///
 /// See the doc comment there for information on the formatting expected for
 /// table and reducer names.
-pub struct SpacetimeModule {
+pub struct SpacetimeModule<L> {
     // Module must be dropped BEFORE runtime otherwise there is a deadlock!
     // In Rust, struct fields are guaranteed to drop in declaration order, so don't reorder this field.
     pub module: ModuleHandle,
     runtime: Runtime,
+    lang: PhantomData<L>,
 }
 
 // Note: we use block_on for the methods here. It adds about 70ns of overhead.
 // This isn't currently a problem. Overhead to call an empty reducer is currently 20_000 ns.
 
-impl AsyncExecutor for &SpacetimeModule {
+impl<L> AsyncExecutor for &SpacetimeModule<L> {
     fn block_on<T>(&self, future: impl std::future::Future<Output = T>) -> T {
         self.runtime.block_on(future)
     }
 }
 
 // It's easier to do it this way because async traits are a mess.
-impl BenchDatabase for SpacetimeModule {
-    fn name() -> &'static str {
-        "stdb_module"
+impl<L: ModuleLanguage> BenchDatabase for SpacetimeModule<L> {
+    fn name() -> String {
+        format!("stdb_module/{}", L::NAME)
     }
 
     type TableId = TableId;
@@ -81,7 +65,7 @@ impl BenchDatabase for SpacetimeModule {
             // It's fine that we're constructing this path ad-hoc, as it's just
             // a path location for tests, not part of our stable directory structure.
             let path = RootDir(Path::new(env!("CARGO_MANIFEST_DIR")).join(".spacetime"));
-            BENCHMARKS_MODULE.load_module(config, Some(&path)).await
+            L::get_module().load_module(config, Some(&path)).await
         });
 
         for table in module.client.module.info.module_def.tables() {
@@ -90,7 +74,11 @@ impl BenchDatabase for SpacetimeModule {
         for reducer in module.client.module.info.module_def.reducers() {
             log::trace!("SPACETIME_MODULE: LOADED REDUCER: {:?}", reducer);
         }
-        Ok(SpacetimeModule { runtime, module })
+        Ok(SpacetimeModule {
+            runtime,
+            module,
+            lang: PhantomData,
+        })
     }
 
     fn create_table<T: BenchTable>(
@@ -105,7 +93,7 @@ impl BenchDatabase for SpacetimeModule {
     }
 
     fn clear_table(&mut self, table_id: &Self::TableId) -> ResultBench<()> {
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
         runtime.block_on(async move {
             // FIXME: this doesn't work. delete is unimplemented!!
             /*
@@ -122,7 +110,7 @@ impl BenchDatabase for SpacetimeModule {
     // message in the log.
     // This implementation will not work if other people are concurrently interacting with our module.
     fn count_table(&mut self, table_id: &Self::TableId) -> ResultBench<u32> {
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
 
         let count = runtime.block_on(async move {
             let name = format!("count_{}", table_id.snake_case);
@@ -140,7 +128,7 @@ impl BenchDatabase for SpacetimeModule {
     }
 
     fn empty_transaction(&mut self) -> ResultBench<()> {
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
 
         runtime.block_on(async move {
             module.call_reducer_binary("empty", &[].into()).await?;
@@ -151,7 +139,7 @@ impl BenchDatabase for SpacetimeModule {
     fn insert_bulk<T: BenchTable>(&mut self, table_id: &Self::TableId, rows: Vec<T>) -> ResultBench<()> {
         let rows = rows.into_iter().map(|row| row.into_product_value()).collect();
         let args = product![ArrayValue::Product(rows)];
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
         let reducer_name = format!("insert_bulk_{}", table_id.snake_case);
 
         runtime.block_on(async move {
@@ -162,7 +150,7 @@ impl BenchDatabase for SpacetimeModule {
 
     fn update_bulk<T: BenchTable>(&mut self, table_id: &Self::TableId, row_count: u32) -> ResultBench<()> {
         let args = product![row_count];
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
         let reducer_name = format!("update_bulk_{}", table_id.snake_case);
 
         runtime.block_on(async move {
@@ -172,7 +160,7 @@ impl BenchDatabase for SpacetimeModule {
     }
 
     fn iterate(&mut self, table_id: &Self::TableId) -> ResultBench<()> {
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
         let reducer_name = format!("iterate_{}", table_id.snake_case);
 
         runtime.block_on(async move {
@@ -187,7 +175,7 @@ impl BenchDatabase for SpacetimeModule {
         col_id: impl Into<ColId>,
         value: AlgebraicValue,
     ) -> ResultBench<()> {
-        let SpacetimeModule { runtime, module } = self;
+        let SpacetimeModule { runtime, module, .. } = self;
 
         let product_type = T::product_type();
         let column_name = product_type.elements[col_id.into().idx()].name.as_ref().unwrap();

crates/bench/src/spacetime_raw.rs

@@ -21,8 +21,8 @@ pub struct SpacetimeRaw {
 }
 
 impl BenchDatabase for SpacetimeRaw {
-    fn name() -> &'static str {
-        "stdb_raw"
+    fn name() -> String {
+        "stdb_raw".to_owned()
     }
     type TableId = TableId;