From 2b9ba2fed8b50de94a61bfa186054062addbf862 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Mon, 8 Sep 2025 17:48:55 +0200 Subject: [PATCH 01/39] v8: add and use `log_traceback` --- crates/core/src/host/v8/error.rs | 14 +++++++++++++- crates/core/src/host/v8/mod.rs | 5 ++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/core/src/host/v8/error.rs b/crates/core/src/host/v8/error.rs index 1021f9c5f11..8365823ef3f 100644 --- a/crates/core/src/host/v8/error.rs +++ b/crates/core/src/host/v8/error.rs @@ -126,7 +126,9 @@ pub(super) struct JsError { impl fmt::Display for JsError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { writeln!(f, "js error {}", self.msg)?; - writeln!(f, "{}", self.trace)?; + if !f.alternate() { + writeln!(f, "{}", self.trace)?; + } Ok(()) } } @@ -249,6 +251,16 @@ impl JsError { } } +pub(super) fn log_traceback(func_type: &str, func: &str, e: &anyhow::Error) { + log::info!("{func_type} \"{func}\" runtime error: {e:#}"); + if let Some(js_err) = e.downcast_ref::() { + log::info!("js error {}", js_err.msg); + for (index, frame) in js_err.trace.frames.iter().enumerate() { + log::info!(" Frame #{index}: {frame}"); + } + } +} + /// Run `body` within a try-catch context and capture any JS exception thrown as a [`JsError`]. pub(super) fn catch_exception<'scope, T>( scope: &mut HandleScope<'scope>, diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 92e1fb0c896..9e3c0cf429e 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -12,7 +12,7 @@ use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica use anyhow::anyhow; use core::time::Duration; use de::deserialize_js; -use error::{catch_exception, exception_already_thrown, ExcResult, Throwable}; +use error::{catch_exception, exception_already_thrown, log_traceback, ExcResult, Throwable}; use from_value::cast; use key_cache::get_or_create_key_cache; use ser::serialize_to_js; @@ -157,8 +157,7 @@ impl ModuleInstance for JsInstance { &self.replica_ctx.clone(), tx, params, - // TODO(centril): logging. - |_ty, _fun, _err| {}, + log_traceback, |tx, op, _budget| { let call_result = call_call_reducer_from_op(scope, op); // TODO(centril): energy metrering. From 724e3c4162d7d41b679bf4bc3cfe40003cdc7321 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 9 Sep 2025 13:07:39 +0200 Subject: [PATCH 02/39] V8: measure call-reducer timings & memory usage --- crates/core/src/host/v8/mod.rs | 35 ++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 9e3c0cf429e..03dc087efc7 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -10,7 +10,7 @@ use crate::host::wasm_common::module_host_actor::{ use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; use anyhow::anyhow; -use core::time::Duration; +use std::time::Instant; use de::deserialize_js; use error::{catch_exception, exception_already_thrown, log_traceback, ExcResult, Throwable}; use from_value::cast; @@ -147,35 +147,46 @@ impl ModuleInstance for JsInstance { } fn call_reducer(&mut self, tx: Option, params: CallReducerParams) -> super::ReducerCallResult { - // TODO(centril): snapshots, module->host calls - let mut isolate = Isolate::new(<_>::default()); - let scope = &mut HandleScope::new(&mut isolate); - let context = Context::new(scope, ContextOptions::default()); - let scope = &mut ContextScope::new(scope, context); - self.common.call_reducer_with_tx( &self.replica_ctx.clone(), tx, params, log_traceback, |tx, op, _budget| { - let call_result = call_call_reducer_from_op(scope, op); + // TODO(centril): snapshots, module->host calls + // Setup V8 scope. + let mut isolate: v8::OwnedIsolate = Isolate::new(<_>::default()); + let mut scope_1 = HandleScope::new(&mut isolate); + let context = Context::new(&mut scope_1, ContextOptions::default()); + let mut scope_2 = ContextScope::new(&mut scope_1, context); + + // Call the reducer. + let start = Instant::now(); + let call_result = call_call_reducer_from_op(&mut scope_2, op); + let total_duration = start.elapsed(); + // TODO(centril): energy metrering. let energy = EnergyStats { used: EnergyQuanta::ZERO, wasmtime_fuel_used: 0, remaining: ReducerBudget::ZERO, }; - // TODO(centril): timings. let timings = ExecutionTimings { - total_duration: Duration::ZERO, + total_duration, + // TODO(centril): call times. wasm_instance_env_call_times: CallTimes::new(), }; + + // Fetch the currently used heap size in V8. + // The used size is ostensibly fairer than the total size. + drop(scope_2); + drop(scope_1); + let memory_allocation = isolate.get_heap_statistics().used_heap_size(); + let exec_result = ExecuteResult { energy, timings, - // TODO(centril): memory allocation. - memory_allocation: 0, + memory_allocation, call_result, }; (tx, exec_result) From f5da920a8fce7bb81c10f6866c9e4bb841103fdf Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 9 Sep 2025 14:28:40 +0200 Subject: [PATCH 03/39] wasm: refactor energy; v8: energy, timings, and timeout --- crates/core/src/host/v8/mod.rs | 61 ++++++++++++++++--- .../src/host/wasm_common/module_host_actor.rs | 24 +++++--- .../core/src/host/wasmtime/wasmtime_module.rs | 13 +--- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 03dc087efc7..0451bb4ebbf 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -10,20 +10,23 @@ use crate::host::wasm_common::module_host_actor::{ use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; use anyhow::anyhow; -use std::time::Instant; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::time::Duration; use de::deserialize_js; use error::{catch_exception, exception_already_thrown, log_traceback, ExcResult, Throwable}; use from_value::cast; use key_cache::get_or_create_key_cache; use ser::serialize_to_js; -use spacetimedb_client_api_messages::energy::{EnergyQuanta, ReducerBudget}; +use spacetimedb_client_api_messages::energy::ReducerBudget; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::traits::Program; use spacetimedb_lib::RawModuleDef; use spacetimedb_lib::{ConnectionId, Identity}; use spacetimedb_schema::auto_migrate::MigrationPolicy; use std::sync::{Arc, LazyLock}; -use v8::{Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, Local, Value}; +use std::thread; +use std::time::Instant; +use v8::{Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, Value}; mod de; mod error; @@ -152,25 +155,28 @@ impl ModuleInstance for JsInstance { tx, params, log_traceback, - |tx, op, _budget| { + |tx, op, budget| { // TODO(centril): snapshots, module->host calls // Setup V8 scope. let mut isolate: v8::OwnedIsolate = Isolate::new(<_>::default()); + let isolate_handle = isolate.thread_safe_handle(); let mut scope_1 = HandleScope::new(&mut isolate); let context = Context::new(&mut scope_1, ContextOptions::default()); let mut scope_2 = ContextScope::new(&mut scope_1, context); + let timeout_thread_cancel_flag = run_reducer_timeout(isolate_handle, budget); + // Call the reducer. let start = Instant::now(); let call_result = call_call_reducer_from_op(&mut scope_2, op); let total_duration = start.elapsed(); + // Cancel the execution timeout in `run_reducer_timeout`. + timeout_thread_cancel_flag.store(true, Ordering::Relaxed); - // TODO(centril): energy metrering. - let energy = EnergyStats { - used: EnergyQuanta::ZERO, - wasmtime_fuel_used: 0, - remaining: ReducerBudget::ZERO, - }; + // Handle energy and timings. + let used = duration_to_budget(total_duration); + let remaining = budget - used; + let energy = EnergyStats { budget, remaining }; let timings = ExecutionTimings { total_duration, // TODO(centril): call times. @@ -195,6 +201,41 @@ impl ModuleInstance for JsInstance { } } +fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> Arc { + let execution_done_flag = Arc::new(AtomicBool::new(false)); + let execution_done_flag2 = execution_done_flag.clone(); + let timeout = budget_to_duration(budget); + + // TODO(centril): Using an OS thread is a bit heavy handed...? + thread::spawn(move || { + // Sleep until the timeout. + thread::sleep(timeout); + + if execution_done_flag2.load(Ordering::Relaxed) { + // The reducer completed successfully. + return; + } + + // Reducer is still running. + // Terminate V8 execution. + isolate_handle.terminate_execution(); + }); + + execution_done_flag +} + +fn budget_to_duration(_budget: ReducerBudget) -> Duration { + // TODO(centril): This is fake logic that allows a maximum timeout. + // Replace with sensible math. + Duration::MAX +} + +fn duration_to_budget(_duration: Duration) -> ReducerBudget { + // TODO(centril): This is fake logic that allows minimum energy usage. + // Replace with sensible math. + ReducerBudget::ZERO +} + /// Returns the global property `key`. fn get_global_property<'scope>( scope: &mut HandleScope<'scope>, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index c93b81226d2..689c1692ff3 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -8,7 +8,7 @@ use tracing::span::EnteredSpan; use super::instrumentation::CallTimes; use crate::client::ClientConnectionSender; use crate::database_logger; -use crate::energy::{EnergyMonitor, EnergyQuanta, ReducerBudget, ReducerFingerprint}; +use crate::energy::{EnergyMonitor, ReducerBudget, ReducerFingerprint}; use crate::host::instance_env::InstanceEnv; use crate::host::module_common::{build_common_module_from_raw, ModuleCommon}; use crate::host::module_host::{ @@ -60,11 +60,17 @@ pub trait WasmInstance: Send + Sync + 'static { } pub struct EnergyStats { - pub used: EnergyQuanta, - pub wasmtime_fuel_used: u64, + pub budget: ReducerBudget, pub remaining: ReducerBudget, } +impl EnergyStats { + /// Returns the used energy amount. + fn used(&self) -> ReducerBudget { + (self.budget.get() - self.remaining.get()).into() + } +} + pub struct ExecutionTimings { pub total_duration: Duration, pub wasm_instance_env_call_times: CallTimes, @@ -412,14 +418,16 @@ impl InstanceCommon { call_result, } = result; + let energy_used = energy.used(); + let energy_quanta_used = energy_used.into(); vm_metrics.report( - energy.wasmtime_fuel_used, + energy_used.get(), timings.total_duration, &timings.wasm_instance_env_call_times, ); self.energy_monitor - .record_reducer(&energy_fingerprint, energy.used, timings.total_duration); + .record_reducer(&energy_fingerprint, energy_quanta_used, timings.total_duration); if self.allocated_memory != memory_allocation { self.metric_wasm_memory_bytes.set(memory_allocation as i64); self.allocated_memory = memory_allocation; @@ -427,7 +435,7 @@ impl InstanceCommon { reducer_span .record("timings.total_duration", tracing::field::debug(timings.total_duration)) - .record("energy.used", tracing::field::debug(energy.used)); + .record("energy.used", tracing::field::debug(energy_used)); maybe_log_long_running_reducer(reducer_name, timings.total_duration); reducer_span.exit(); @@ -486,7 +494,7 @@ impl InstanceCommon { args, }, status, - energy_quanta_used: energy.used, + energy_quanta_used, host_execution_duration: timings.total_duration, request_id, timer, @@ -495,7 +503,7 @@ impl InstanceCommon { ReducerCallResult { outcome: ReducerOutcome::from(&event.status), - energy_used: energy.used, + energy_used: energy_quanta_used, execution_duration: timings.total_duration, } } diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 4374b708eb3..119b7782ee2 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -192,11 +192,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { #[tracing::instrument(level = "trace", skip_all)] fn call_reducer(&mut self, op: ReducerOp<'_>, budget: ReducerBudget) -> module_host_actor::ExecuteResult { let store = &mut self.store; - // note that ReducerBudget being a u64 is load-bearing here - although we convert budget right back into - // EnergyQuanta at the end of this function, from_energy_quanta clamps it to a u64 range. - // otherwise, we'd return something like `used: i128::MAX - u64::MAX`, which is inaccurate. + // Set the fuel budget in WASM. set_store_fuel(store, budget.into()); - let original_fuel = get_store_fuel(store); store.set_epoch_deadline(EPOCH_TICKS_PER_SECOND); // Prepare sender identity and connection ID, as LITTLE-ENDIAN byte arrays. @@ -231,14 +228,10 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let call_result = call_result.map(|code| handle_error_sink_code(code, error)); + // Compute fuel and heap usage. let remaining_fuel = get_store_fuel(store); - let remaining: ReducerBudget = remaining_fuel.into(); - let energy = module_host_actor::EnergyStats { - used: (budget - remaining).into(), - wasmtime_fuel_used: original_fuel.0 - remaining_fuel.0, - remaining, - }; + let energy = module_host_actor::EnergyStats { budget, remaining }; let memory_allocation = store.data().get_mem().memory.data_size(&store); module_host_actor::ExecuteResult { From 0e0e80653d76f35d27d891ad8c3f1be21f59f3d7 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 9 Sep 2025 18:45:38 +0200 Subject: [PATCH 04/39] v8: expose in client-api & cli & standalone --- crates/cli/src/subcommands/publish.rs | 16 ++++++++++++++++ crates/client-api/Cargo.toml | 3 +++ crates/client-api/src/routes/database.rs | 17 ++++++++++++++++- crates/core/src/host/v8/error.rs | 2 +- crates/core/src/host/v8/mod.rs | 14 +++++--------- crates/core/src/host/v8/ser.rs | 2 +- crates/core/src/messages/control_db.rs | 5 ++++- crates/standalone/Cargo.toml | 1 + 8 files changed, 47 insertions(+), 13 deletions(-) diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 4524f8c8081..e903f4a2a23 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -48,6 +48,15 @@ pub fn cli() -> clap::Command { .conflicts_with("build_options") .help("The system path (absolute or relative) to the compiled wasm binary we should publish, instead of building the project."), ) + // TODO(v8): needs better UX but good enough for a demo... + .arg( + Arg::new("javascript") + .long("javascript") + .action(SetTrue) + .requires("wasm_file") + .hide(true) + .help("UNSTABLE: interpret `--bin-path` as a JS module"), + ) .arg( Arg::new("num_replicas") .value_parser(clap::value_parser!(u8)) @@ -90,6 +99,8 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E let force = args.get_flag("force"); let anon_identity = args.get_flag("anon_identity"); let wasm_file = args.get_one::("wasm_file"); + // TODO(v8): needs better UX but good enough for a demo... + let wasm_file_is_really_js = args.get_flag("javascript"); let database_host = config.get_host_url(server)?; let build_options = args.get_one::("build_options").unwrap(); let num_replicas = args.get_one::("num_replicas"); @@ -116,6 +127,11 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E }; let program_bytes = fs::read(path_to_wasm)?; + // TODO(v8): needs better UX but good enough for a demo... + if wasm_file_is_really_js { + builder.query(&[("host_type", "Js")]) + } + let server_address = { let url = Url::parse(&database_host)?; url.host_str().unwrap_or("").to_string() diff --git a/crates/client-api/Cargo.toml b/crates/client-api/Cargo.toml index 13c6bb816da..be32c90e9c3 100644 --- a/crates/client-api/Cargo.toml +++ b/crates/client-api/Cargo.toml @@ -64,3 +64,6 @@ toml.workspace = true [lints] workspace = true + +[features] +unstable = [] diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index ccab05d1280..ea056f57b2d 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -500,6 +500,8 @@ pub struct PublishDatabaseQueryParams { token: Option, #[serde(default)] policy: MigrationPolicy, + #[serde(default)] + host_type: HostType, } use spacetimedb_client_api_messages::http::SqlStmtResult; @@ -537,10 +539,23 @@ pub async fn publish( num_replicas, token, policy, + host_type, }): Query, Extension(auth): Extension, body: Bytes, ) -> axum::response::Result> { + // Feature gate V8 modules. + // The host must've been compiled with the `unstable` feature. + // TODO(v8): ungate this when V8 is ready to ship. + #[cfg(not(feature = "unstable"))] + if host_type == HostType::Js { + return Err(( + StatusCode::BAD_REQUEST, + "JS host type requires a host with unstable features", + ) + .into()); + } + // You should not be able to publish to a database that you do not own // so, unless you are the owner, this will fail. @@ -645,7 +660,7 @@ pub async fn publish( database_identity, program_bytes: body.into(), num_replicas, - host_type: HostType::Wasm, + host_type, }, policy, ) diff --git a/crates/core/src/host/v8/error.rs b/crates/core/src/host/v8/error.rs index 8365823ef3f..bfcbc0715ce 100644 --- a/crates/core/src/host/v8/error.rs +++ b/crates/core/src/host/v8/error.rs @@ -206,7 +206,7 @@ impl fmt::Display for JsStackTraceFrame { // This isn't exactly the same format as chrome uses, // but it's close enough for now. - // TODO(centril): make it more like chrome in the future. + // TODO(v8): make it more like chrome in the future. f.write_fmt(format_args!( "at {} ({}:{}:{})", fn_name, script_name, &self.line, &self.column diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 0451bb4ebbf..b4a871b4a10 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -83,10 +83,6 @@ impl V8RuntimeInner { mcc.program.hash, ); - if true { - return Err::(anyhow!("v8_todo")); - } - let desc = todo!(); // Validate and create a common module rom the raw definition. let common = build_common_module_from_raw(mcc, desc)?; @@ -156,7 +152,7 @@ impl ModuleInstance for JsInstance { params, log_traceback, |tx, op, budget| { - // TODO(centril): snapshots, module->host calls + // TODO(v8): snapshots, module->host calls // Setup V8 scope. let mut isolate: v8::OwnedIsolate = Isolate::new(<_>::default()); let isolate_handle = isolate.thread_safe_handle(); @@ -179,7 +175,7 @@ impl ModuleInstance for JsInstance { let energy = EnergyStats { budget, remaining }; let timings = ExecutionTimings { total_duration, - // TODO(centril): call times. + // TODO(v8): call times. wasm_instance_env_call_times: CallTimes::new(), }; @@ -206,7 +202,7 @@ fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> let execution_done_flag2 = execution_done_flag.clone(); let timeout = budget_to_duration(budget); - // TODO(centril): Using an OS thread is a bit heavy handed...? + // TODO(v8): Using an OS thread is a bit heavy handed...? thread::spawn(move || { // Sleep until the timeout. thread::sleep(timeout); @@ -225,13 +221,13 @@ fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> } fn budget_to_duration(_budget: ReducerBudget) -> Duration { - // TODO(centril): This is fake logic that allows a maximum timeout. + // TODO(v8): This is fake logic that allows a maximum timeout. // Replace with sensible math. Duration::MAX } fn duration_to_budget(_duration: Duration) -> ReducerBudget { - // TODO(centril): This is fake logic that allows minimum energy usage. + // TODO(v8): This is fake logic that allows minimum energy usage. // Replace with sensible math. ReducerBudget::ZERO } diff --git a/crates/core/src/host/v8/ser.rs b/crates/core/src/host/v8/ser.rs index 62f6130ba08..2965df81aa1 100644 --- a/crates/core/src/host/v8/ser.rs +++ b/crates/core/src/host/v8/ser.rs @@ -150,7 +150,7 @@ impl<'this, 'scope> ser::Serializer for Serializer<'this, 'scope> { } fn serialize_named_product(self, _len: usize) -> Result { - // TODO(noa): this can be more efficient if we tell it the names ahead of time + // TODO(v8, noa): this can be more efficient if we tell it the names ahead of time let object = Object::new(self.scope); Ok(SerializeNamedProduct { inner: self, diff --git a/crates/core/src/messages/control_db.rs b/crates/core/src/messages/control_db.rs index 8299875e339..8f92f350324 100644 --- a/crates/core/src/messages/control_db.rs +++ b/crates/core/src/messages/control_db.rs @@ -75,9 +75,12 @@ pub struct NodeStatus { /// SEE: pub state: String, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default, Serialize, Deserialize, serde::Deserialize, +)] #[repr(i32)] pub enum HostType { + #[default] Wasm = 0, Js = 1, } diff --git a/crates/standalone/Cargo.toml b/crates/standalone/Cargo.toml index e719721721d..17b8d547062 100644 --- a/crates/standalone/Cargo.toml +++ b/crates/standalone/Cargo.toml @@ -19,6 +19,7 @@ required-features = [] # Features required to build this target (N/A for lib) [features] # Perfmaps for profiling modules perfmap = ["spacetimedb-core/perfmap"] +unstable = ["spacetimedb-client-api/unstable"] [dependencies] spacetimedb-client-api-messages.workspace = true From 8dcee06b2936cc7cd5ed7b625abe1b86a695a544 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 9 Sep 2025 18:47:52 +0200 Subject: [PATCH 05/39] v8: document some functions --- crates/core/src/host/v8/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index b4a871b4a10..eac502f59ac 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -197,6 +197,8 @@ impl ModuleInstance for JsInstance { } } +/// Spawns a thread that will terminate reducer execution +/// when `budget` has been used up. fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> Arc { let execution_done_flag = Arc::new(AtomicBool::new(false)); let execution_done_flag2 = execution_done_flag.clone(); @@ -220,12 +222,14 @@ fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> execution_done_flag } +/// Converts a [`ReducerBudget`] to a [`Duration`]. fn budget_to_duration(_budget: ReducerBudget) -> Duration { // TODO(v8): This is fake logic that allows a maximum timeout. // Replace with sensible math. Duration::MAX } +/// Converts a [`Duration`] to a [`ReducerBudget`]. fn duration_to_budget(_duration: Duration) -> ReducerBudget { // TODO(v8): This is fake logic that allows minimum energy usage. // Replace with sensible math. From 8c455a322d2f978c3f1ce5ef3a7238c9aa8c72ca Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 10 Sep 2025 18:25:31 +0200 Subject: [PATCH 06/39] v8: leave some todos --- crates/core/src/host/v8/mod.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index eac502f59ac..03958cf424a 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -9,7 +9,6 @@ use crate::host::wasm_common::module_host_actor::{ }; use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; -use anyhow::anyhow; use core::sync::atomic::{AtomicBool, Ordering}; use core::time::Duration; use de::deserialize_js; @@ -83,6 +82,10 @@ impl V8RuntimeInner { mcc.program.hash, ); + // TODO(v8): determine min required ABI by module and check that it's supported? + + // TODO(v8): validate function signatures like in WASM? Is that possible with V8? + let desc = todo!(); // Validate and create a common module rom the raw definition. let common = build_common_module_from_raw(mcc, desc)?; @@ -120,6 +123,17 @@ impl Module for JsModule { } fn create_instance(&self) -> Self::Instance { + // TODO(v8): consider some equivalent to `epoch_deadline_callback` + // where we report `Js has been running for ...`. + + // TODO(v8): timeout things like `extract_description`. + + // TODO(v8): do we care about preinits / setup or are they unnecessary? + + // TODO(v8): create `InstanceEnv`. + + // TODO(v8): extract description. + todo!() } } From bcc77310ac8835f3fc3fc02db62aa0068f2b3bfe Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 10 Sep 2025 19:52:56 +0200 Subject: [PATCH 07/39] wip --- crates/cli/src/subcommands/publish.rs | 2 +- crates/core/src/host/module_common.rs | 22 +++- crates/core/src/host/v8/mod.rs | 120 ++++++++++++++---- .../src/host/wasm_common/module_host_actor.rs | 24 ++-- .../core/src/host/wasmtime/wasmtime_module.rs | 24 +--- 5 files changed, 134 insertions(+), 58 deletions(-) diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index e903f4a2a23..3a399bacde0 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -129,7 +129,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E // TODO(v8): needs better UX but good enough for a demo... if wasm_file_is_really_js { - builder.query(&[("host_type", "Js")]) + builder = builder.query(&[("host_type", "Js")]); } let server_address = { diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index e93b68c1172..06f3da0eadf 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -4,8 +4,7 @@ use crate::{ energy::EnergyMonitor, host::{ - module_host::{DynModule, ModuleInfo}, - Scheduler, + module_host::{DynModule, ModuleInfo}, wasm_common::{module_host_actor::DescribeError, DESCRIBE_MODULE_DUNDER}, Scheduler }, module_host_context::ModuleCreationContext, replica_context::ReplicaContext, @@ -88,3 +87,22 @@ impl DynModule for ModuleCommon { &self.scheduler } } + +/// Runs the describer of modules in `run` and does some logging around it. +pub(crate) fn run_describer( + log_traceback: impl FnOnce(&str, &str, &anyhow::Error), + run: impl FnOnce() -> anyhow::Result, +) -> Result { + let describer_func_name = DESCRIBE_MODULE_DUNDER; + let start = std::time::Instant::now(); + log::trace!("Start describer \"{describer_func_name}\"..."); + + let result = run(); + + let duration = start.elapsed(); + log::trace!("Describer \"{}\" ran: {} us", describer_func_name, duration.as_micros()); + + result + .inspect_err(|err| log_traceback("describer", describer_func_name, err)) + .map_err(DescribeError::RuntimeError) +} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 03958cf424a..eed140f2d30 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -3,12 +3,15 @@ use super::module_common::{build_common_module_from_raw, ModuleCommon}; use super::module_host::{CallReducerParams, DynModule, Module, ModuleInfo, ModuleInstance, ModuleRuntime}; use super::UpdateDatabaseResult; +use crate::host::instance_env::InstanceEnv; +use crate::host::module_common::run_describer; use crate::host::wasm_common::instrumentation::CallTimes; use crate::host::wasm_common::module_host_actor::{ - EnergyStats, ExecuteResult, ExecutionTimings, InstanceCommon, ReducerOp, + DescribeError, EnergyStats, ExecuteResult, ExecutionTimings, InstanceCommon, ReducerOp, }; use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; +use core::str; use core::sync::atomic::{AtomicBool, Ordering}; use core::time::Duration; use de::deserialize_js; @@ -25,7 +28,9 @@ use spacetimedb_schema::auto_migrate::MigrationPolicy; use std::sync::{Arc, LazyLock}; use std::thread; use std::time::Instant; -use v8::{Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, Value}; +use v8::{ + Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, OwnedIsolate, Value, +}; mod de; mod error; @@ -86,17 +91,23 @@ impl V8RuntimeInner { // TODO(v8): validate function signatures like in WASM? Is that possible with V8? - let desc = todo!(); + // Convert program to a string. + let program: Arc = str::from_utf8(&mcc.program.bytes)?.into(); + + // Run the program as a script and extract the raw module def. + let desc = extract_description(&program)?; + // Validate and create a common module rom the raw definition. let common = build_common_module_from_raw(mcc, desc)?; - Ok(JsModule { common }) + Ok(JsModule { common, program }) } } #[derive(Clone)] struct JsModule { common: ModuleCommon, + program: Arc, } impl DynModule for JsModule { @@ -124,23 +135,36 @@ impl Module for JsModule { fn create_instance(&self) -> Self::Instance { // TODO(v8): consider some equivalent to `epoch_deadline_callback` - // where we report `Js has been running for ...`. - - // TODO(v8): timeout things like `extract_description`. + // where we report `Js has been running for ...`s. // TODO(v8): do we care about preinits / setup or are they unnecessary? - // TODO(v8): create `InstanceEnv`. + let common = &self.common; + let instance_env = InstanceEnv::new(common.replica_ctx().clone(), common.scheduler().clone()); + let instance = JsInstanceEnv { instance_env }; + + // NOTE(centril): We don't need to do `extract_description` here + // as unlike WASM, we have to recreate the isolate every time. - // TODO(v8): extract description. + let common = InstanceCommon::new(common); + let program = self.program.clone(); - todo!() + JsInstance { + common, + instance, + program, + } } } +struct JsInstanceEnv { + instance_env: InstanceEnv, +} + struct JsInstance { common: InstanceCommon, - replica_ctx: Arc, + instance: JsInstanceEnv, + program: Arc, } impl ModuleInstance for JsInstance { @@ -154,34 +178,35 @@ impl ModuleInstance for JsInstance { old_module_info: Arc, policy: MigrationPolicy, ) -> anyhow::Result { - let replica_ctx = &self.replica_ctx; + let replica_ctx = &self.instance.instance_env.replica_ctx; self.common .update_database(replica_ctx, program, old_module_info, policy) } fn call_reducer(&mut self, tx: Option, params: CallReducerParams) -> super::ReducerCallResult { self.common.call_reducer_with_tx( - &self.replica_ctx.clone(), + &self.instance.instance_env.replica_ctx.clone(), tx, params, log_traceback, |tx, op, budget| { // TODO(v8): snapshots, module->host calls - // Setup V8 scope. - let mut isolate: v8::OwnedIsolate = Isolate::new(<_>::default()); - let isolate_handle = isolate.thread_safe_handle(); - let mut scope_1 = HandleScope::new(&mut isolate); - let context = Context::new(&mut scope_1, ContextOptions::default()); - let mut scope_2 = ContextScope::new(&mut scope_1, context); + // Call the reducer. + let (mut isolate, (tx, call_result, total_duration)) = + with_script(&self.program, budget, |scope, _| { + let start = Instant::now(); - let timeout_thread_cancel_flag = run_reducer_timeout(isolate_handle, budget); + let (tx, call_result) = self + .instance + .instance_env + .tx + .clone() + .set(tx, || call_call_reducer_from_op(scope, op)); - // Call the reducer. - let start = Instant::now(); - let call_result = call_call_reducer_from_op(&mut scope_2, op); - let total_duration = start.elapsed(); - // Cancel the execution timeout in `run_reducer_timeout`. - timeout_thread_cancel_flag.store(true, Ordering::Relaxed); + let total_duration = start.elapsed(); + + (tx, call_result, total_duration) + }); // Handle energy and timings. let used = duration_to_budget(total_duration); @@ -195,8 +220,6 @@ impl ModuleInstance for JsInstance { // Fetch the currently used heap size in V8. // The used size is ostensibly fairer than the total size. - drop(scope_2); - drop(scope_1); let memory_allocation = isolate.get_heap_statistics().used_heap_size(); let exec_result = ExecuteResult { @@ -211,6 +234,39 @@ impl ModuleInstance for JsInstance { } } +fn with_script( + code: &str, + budget: ReducerBudget, + logic: impl for<'scope> FnOnce(&mut HandleScope<'scope>, Local<'scope, Value>) -> R, +) -> (OwnedIsolate, R) { + with_scope(budget, |scope| { + let code = v8::String::new(scope, code).unwrap(); + let script_val = v8::Script::compile(scope, code, None).unwrap().run(scope).unwrap(); + logic(scope, script_val) + }) +} + +/// Sets up an isolate and run `logic` with a [`HandleScope`]. +pub(crate) fn with_scope(budget: ReducerBudget, logic: impl FnOnce(&mut HandleScope<'_>) -> R) -> (OwnedIsolate, R) { + let mut isolate: OwnedIsolate = Isolate::new(<_>::default()); + isolate.set_capture_stack_trace_for_uncaught_exceptions(true, 1024); + let isolate_handle = isolate.thread_safe_handle(); + let mut scope_1 = HandleScope::new(&mut isolate); + let context = Context::new(&mut scope_1, ContextOptions::default()); + let mut scope_2 = ContextScope::new(&mut scope_1, context); + + let timeout_thread_cancel_flag = run_reducer_timeout(isolate_handle, budget); + + let ret = logic(&mut scope_2); + drop(scope_2); + drop(scope_1); + + // Cancel the execution timeout in `run_reducer_timeout`. + timeout_thread_cancel_flag.store(true, Ordering::Relaxed); + + (isolate, ret) +} + /// Spawns a thread that will terminate reducer execution /// when `budget` has been used up. fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> Arc { @@ -321,6 +377,14 @@ fn call_call_reducer( .map_err(Into::into) } +/// Extracts the raw module def by running `__describe_module__` in `program`. +fn extract_description(program: &str) -> Result { + let (_, ret) = with_script(program, ReducerBudget::DEFAULT_BUDGET, |scope, _| { + run_describer(log_traceback, || call_describe_module(scope)) + }); + ret +} + // Calls the `__describe_module__` function on the global proxy object to extract a [`RawModuleDef`]. fn call_describe_module(scope: &mut HandleScope<'_>) -> anyhow::Result { // Get a cached version of the `__describe_module__` property. diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 689c1692ff3..e327b0b95b0 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -166,16 +166,7 @@ impl WasmModuleHostActor { impl WasmModuleHostActor { fn make_from_instance(&self, instance: T::Instance) -> WasmModuleInstance { - let common = InstanceCommon { - info: self.common.info(), - energy_monitor: self.common.energy_monitor(), - // will be updated on the first reducer call - allocated_memory: 0, - metric_wasm_memory_bytes: WORKER_METRICS - .wasm_memory_bytes - .with_label_values(self.common.database_identity()), - trapped: false, - }; + let common = InstanceCommon::new(&self.common); WasmModuleInstance { instance, common } } } @@ -279,6 +270,19 @@ pub(crate) struct InstanceCommon { } impl InstanceCommon { + pub(crate) fn new(module: &ModuleCommon) -> Self { + Self { + info: module.info(), + energy_monitor: module.energy_monitor(), + // Will be updated on the first reducer call. + allocated_memory: 0, + metric_wasm_memory_bytes: WORKER_METRICS + .wasm_memory_bytes + .with_label_values(module.database_identity()), + trapped: false, + } + } + #[tracing::instrument(level = "trace", skip_all)] pub(crate) fn update_database( &mut self, diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 119b7782ee2..7496b6ca854 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -4,6 +4,7 @@ use super::wasm_instance_env::WasmInstanceEnv; use super::{Mem, WasmtimeFuel, EPOCH_TICKS_PER_SECOND}; use crate::energy::ReducerBudget; use crate::host::instance_env::InstanceEnv; +use crate::host::module_common::run_describer; use crate::host::wasm_common::module_host_actor::{DescribeError, InitializationError}; use crate::host::wasm_common::*; use crate::util::string_from_utf8_lossy_owned; @@ -158,29 +159,18 @@ pub struct WasmtimeInstance { impl module_host_actor::WasmInstance for WasmtimeInstance { fn extract_descriptions(&mut self) -> Result, DescribeError> { let describer_func_name = DESCRIBE_MODULE_DUNDER; - let store = &mut self.store; - let describer = self.instance.get_func(&mut *store, describer_func_name).unwrap(); - let describer = describer - .typed::(&mut *store) + let describer = self + .instance + .get_typed_func::(&mut self.store, describer_func_name) .map_err(|_| DescribeError::Signature)?; - let sink = store.data_mut().setup_standard_bytes_sink(); - - let start = std::time::Instant::now(); - log::trace!("Start describer \"{describer_func_name}\"..."); - - let result = describer.call(&mut *store, sink); - - let duration = start.elapsed(); - log::trace!("Describer \"{}\" ran: {} us", describer_func_name, duration.as_micros()); + let sink = self.store.data_mut().setup_standard_bytes_sink(); - result - .inspect_err(|err| log_traceback("describer", describer_func_name, err)) - .map_err(DescribeError::RuntimeError)?; + run_describer(log_traceback, || describer.call(&mut self.store, sink))?; // Fetch the bsatn returned by the describer call. - let bytes = store.data_mut().take_standard_bytes_sink(); + let bytes = self.store.data_mut().take_standard_bytes_sink(); Ok(bytes) } From f3d00cad31855e158ae747d7e00ae225a9cd8f07 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Thu, 11 Sep 2025 12:24:21 +0200 Subject: [PATCH 08/39] extract InstanceEnv::console_timer_end --- crates/core/src/host/instance_env.rs | 17 ++++++++++++++++ .../src/host/wasmtime/wasm_instance_env.rs | 20 ++++--------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 4d6863a7f27..b655dc5b13a 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -2,6 +2,7 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; use crate::database_logger::{BacktraceProvider, LogLevel, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; +use crate::host::wasm_common::TimingSpan; use crate::replica_context::ReplicaContext; use core::mem; use parking_lot::{Mutex, MutexGuard}; @@ -189,6 +190,22 @@ impl InstanceEnv { ); } + /// End a console timer by logging the span at INFO level. + pub fn console_timer_end(&self, span: &TimingSpan, function: Option<&str>, bt: &dyn BacktraceProvider) -> RtResult { + let elapsed = span.start.elapsed(); + let message = format!("Timing span {:?}: {:?}", &span.name, elapsed); + + let record = Record { + ts: chrono::Utc::now(), + target: None, + filename: None, + line_number: None, + function, + message: &message, + }; + self.console_log(LogLevel::Info, &record, bt); + } + /// Project `cols` in `row_ref` encoded in BSATN to `buffer` /// and return the full length of the BSATN. /// diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index d2aabaf656e..700883ae4c5 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -1333,23 +1333,11 @@ impl WasmInstanceEnv { return Ok(errno::NO_SUCH_CONSOLE_TIMER.get().into()); }; - let elapsed = span.start.elapsed(); - let message = format!("Timing span {:?}: {:?}", &span.name, elapsed); let function = caller.data().log_record_function(); - - let record = Record { - ts: chrono::Utc::now(), - target: None, - filename: None, - line_number: None, - function, - message: &message, - }; - caller.data().instance_env.console_log( - crate::database_logger::LogLevel::Info, - &record, - &caller.as_context(), - ); + caller + .data() + .instance_env + .console_timer_end(&span, function, &caller.as_context()); Ok(0) }) } From 397807dd56c980465e095f196dd991f648e4f36b Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 12 Sep 2025 14:29:38 +0200 Subject: [PATCH 09/39] wip --- crates/core/src/host/instance_env.rs | 7 +- crates/core/src/host/v8/mod.rs | 204 +++++++++++++----- crates/core/src/host/wasm_common.rs | 2 +- crates/core/src/host/wasmtime/mod.rs | 27 ++- .../src/host/wasmtime/wasm_instance_env.rs | 2 +- 5 files changed, 178 insertions(+), 64 deletions(-) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index b655dc5b13a..99b2e7cf58d 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -8,7 +8,7 @@ use core::mem; use parking_lot::{Mutex, MutexGuard}; use smallvec::SmallVec; use spacetimedb_datastore::locking_tx_datastore::MutTxId; -use spacetimedb_lib::Timestamp; +use spacetimedb_lib::{Identity, Timestamp}; use spacetimedb_primitives::{ColId, ColList, IndexId, TableId}; use spacetimedb_sats::{ bsatn::{self, ToBsatn}, @@ -171,6 +171,11 @@ impl InstanceEnv { } } + /// Returns the database's identity. + pub fn database_identity(&self) -> &Identity { + &self.replica_ctx.database.database_identity + } + /// Signal to this `InstanceEnv` that a reducer call is beginning. pub fn start_reducer(&mut self, ts: Timestamp) { self.start_time = ts; diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index eed140f2d30..ab77efd6eac 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -9,11 +9,13 @@ use crate::host::wasm_common::instrumentation::CallTimes; use crate::host::wasm_common::module_host_actor::{ DescribeError, EnergyStats, ExecuteResult, ExecutionTimings, InstanceCommon, ReducerOp, }; +use crate::host::wasmtime::{epoch_ticker, ticks_in_duration, EPOCH_TICKS_PER_SECOND}; use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; -use core::str; +use core::ffi::c_void; use core::sync::atomic::{AtomicBool, Ordering}; use core::time::Duration; +use core::{ptr, str}; use de::deserialize_js; use error::{catch_exception, exception_already_thrown, log_traceback, ExcResult, Throwable}; use from_value::cast; @@ -22,11 +24,9 @@ use ser::serialize_to_js; use spacetimedb_client_api_messages::energy::ReducerBudget; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::traits::Program; -use spacetimedb_lib::RawModuleDef; -use spacetimedb_lib::{ConnectionId, Identity}; use spacetimedb_schema::auto_migrate::MigrationPolicy; +use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; use std::sync::{Arc, LazyLock}; -use std::thread; use std::time::Instant; use v8::{ Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, OwnedIsolate, Value, @@ -134,14 +134,16 @@ impl Module for JsModule { } fn create_instance(&self) -> Self::Instance { - // TODO(v8): consider some equivalent to `epoch_deadline_callback` - // where we report `Js has been running for ...`s. - // TODO(v8): do we care about preinits / setup or are they unnecessary? let common = &self.common; let instance_env = InstanceEnv::new(common.replica_ctx().clone(), common.scheduler().clone()); - let instance = JsInstanceEnv { instance_env }; + let instance = Some(JsInstanceEnv { + instance_env, + reducer_start: Instant::now(), + call_times: CallTimes::new(), + reducer_name: String::from(""), + }); // NOTE(centril): We don't need to do `extract_description` here // as unlike WASM, we have to recreate the isolate every time. @@ -157,13 +159,72 @@ impl Module for JsModule { } } +const EXPECT_ENV: &str = "there should be a `JsInstanceEnv`"; + +fn env_on_isolate(isolate: &mut Isolate) -> &mut JsInstanceEnv { + isolate.get_slot_mut().expect(EXPECT_ENV) +} + +fn env_on_instance(inst: &mut JsInstance) -> &mut JsInstanceEnv { + inst.instance.as_mut().expect(EXPECT_ENV) +} + struct JsInstanceEnv { instance_env: InstanceEnv, + + /// The point in time the last reducer call started at. + reducer_start: Instant, + + /// Track time spent in all wasm instance env calls (aka syscall time). + /// + /// Each function, like `insert`, will add the `Duration` spent in it + /// to this tracker. + call_times: CallTimes, + + /// The last, including current, reducer to be executed by this environment. + reducer_name: String, +} + +impl JsInstanceEnv { + /// Signal to this `WasmInstanceEnv` that a reducer call is beginning. + /// + /// Returns the handle used by reducers to read from `args` + /// as well as the handle used to write the error message, if any. + pub fn start_reducer(&mut self, name: &str, ts: Timestamp) { + self.reducer_start = Instant::now(); + name.clone_into(&mut self.reducer_name); + self.instance_env.start_reducer(ts); + } + + /// Returns the name of the most recent reducer to be run in this environment. + pub fn reducer_name(&self) -> &str { + &self.reducer_name + } + + /// Returns the name of the most recent reducer to be run in this environment. + pub fn reducer_start(&self) -> Instant { + self.reducer_start + } + + /// Signal to this `WasmInstanceEnv` that a reducer call is over. + /// This resets all of the state associated to a single reducer call, + /// and returns instrumentation records. + pub fn finish_reducer(&mut self) -> ExecutionTimings { + let total_duration = self.reducer_start.elapsed(); + + // Taking the call times record also resets timings to 0s for the next call. + let wasm_instance_env_call_times = self.call_times.take(); + + ExecutionTimings { + total_duration, + wasm_instance_env_call_times, + } + } } struct JsInstance { common: InstanceCommon, - instance: JsInstanceEnv, + instance: Option, program: Arc, } @@ -178,45 +239,48 @@ impl ModuleInstance for JsInstance { old_module_info: Arc, policy: MigrationPolicy, ) -> anyhow::Result { - let replica_ctx = &self.instance.instance_env.replica_ctx; + let replica_ctx = &env_on_instance(self).instance_env.replica_ctx.clone(); self.common .update_database(replica_ctx, program, old_module_info, policy) } fn call_reducer(&mut self, tx: Option, params: CallReducerParams) -> super::ReducerCallResult { - self.common.call_reducer_with_tx( - &self.instance.instance_env.replica_ctx.clone(), - tx, - params, - log_traceback, - |tx, op, budget| { + let replica_ctx = env_on_instance(self).instance_env.replica_ctx.clone(); + + self.common + .call_reducer_with_tx(&replica_ctx, tx, params, log_traceback, |tx, op, budget| { + let callback_every = EPOCH_TICKS_PER_SECOND; + extern "C" fn callback(isolate: &mut Isolate, _: *mut c_void) { + let env = env_on_isolate(isolate); + let database = env.instance_env.replica_ctx.database_identity; + let reducer = env.reducer_name(); + let dur = env.reducer_start().elapsed(); + tracing::warn!(reducer, ?database, "Wasm has been running for {dur:?}"); + } + + // Prepare the isolate with the env. + let mut isolate = Isolate::new(<_>::default()); + isolate.set_slot(self.instance.take().expect(EXPECT_ENV)); + // TODO(v8): snapshots, module->host calls // Call the reducer. - let (mut isolate, (tx, call_result, total_duration)) = - with_script(&self.program, budget, |scope, _| { - let start = Instant::now(); - - let (tx, call_result) = self - .instance + env_on_isolate(&mut isolate).instance_env.start_reducer(op.timestamp); + let (mut isolate, (tx, call_result)) = + with_script(isolate, &self.program, callback_every, callback, budget, |scope, _| { + let (tx, call_result) = env_on_isolate(scope) .instance_env .tx .clone() .set(tx, || call_call_reducer_from_op(scope, op)); - - let total_duration = start.elapsed(); - - (tx, call_result, total_duration) + (tx, call_result) }); + let timings = env_on_isolate(&mut isolate).finish_reducer(); + self.instance = isolate.remove_slot(); - // Handle energy and timings. - let used = duration_to_budget(total_duration); + // Derive energy stats. + let used = duration_to_budget(timings.total_duration); let remaining = budget - used; let energy = EnergyStats { budget, remaining }; - let timings = ExecutionTimings { - total_duration, - // TODO(v8): call times. - wasm_instance_env_call_times: CallTimes::new(), - }; // Fetch the currently used heap size in V8. // The used size is ostensibly fairer than the total size. @@ -229,17 +293,19 @@ impl ModuleInstance for JsInstance { call_result, }; (tx, exec_result) - }, - ) + }) } } fn with_script( + isolate: OwnedIsolate, code: &str, + callback_every: u64, + callback: IsolateCallback, budget: ReducerBudget, logic: impl for<'scope> FnOnce(&mut HandleScope<'scope>, Local<'scope, Value>) -> R, ) -> (OwnedIsolate, R) { - with_scope(budget, |scope| { + with_scope(isolate, callback_every, callback, budget, |scope| { let code = v8::String::new(scope, code).unwrap(); let script_val = v8::Script::compile(scope, code, None).unwrap().run(scope).unwrap(); logic(scope, script_val) @@ -247,15 +313,20 @@ fn with_script( } /// Sets up an isolate and run `logic` with a [`HandleScope`]. -pub(crate) fn with_scope(budget: ReducerBudget, logic: impl FnOnce(&mut HandleScope<'_>) -> R) -> (OwnedIsolate, R) { - let mut isolate: OwnedIsolate = Isolate::new(<_>::default()); +pub(crate) fn with_scope( + mut isolate: OwnedIsolate, + callback_every: u64, + callback: IsolateCallback, + budget: ReducerBudget, + logic: impl FnOnce(&mut HandleScope<'_>) -> R, +) -> (OwnedIsolate, R) { isolate.set_capture_stack_trace_for_uncaught_exceptions(true, 1024); let isolate_handle = isolate.thread_safe_handle(); let mut scope_1 = HandleScope::new(&mut isolate); let context = Context::new(&mut scope_1, ContextOptions::default()); let mut scope_2 = ContextScope::new(&mut scope_1, context); - let timeout_thread_cancel_flag = run_reducer_timeout(isolate_handle, budget); + let timeout_thread_cancel_flag = run_reducer_timeout(callback_every, callback, budget, isolate_handle); let ret = logic(&mut scope_2); drop(scope_2); @@ -267,26 +338,44 @@ pub(crate) fn with_scope(budget: ReducerBudget, logic: impl FnOnce(&mut Handl (isolate, ret) } +type IsolateCallback = extern "C" fn(&mut Isolate, *mut c_void); + /// Spawns a thread that will terminate reducer execution /// when `budget` has been used up. -fn run_reducer_timeout(isolate_handle: IsolateHandle, budget: ReducerBudget) -> Arc { +/// +/// Every `callback_every` ticks, `callback` is called. +fn run_reducer_timeout( + callback_every: u64, + callback: IsolateCallback, + budget: ReducerBudget, + isolate_handle: IsolateHandle, +) -> Arc { let execution_done_flag = Arc::new(AtomicBool::new(false)); let execution_done_flag2 = execution_done_flag.clone(); let timeout = budget_to_duration(budget); + let max_ticks = ticks_in_duration(timeout); - // TODO(v8): Using an OS thread is a bit heavy handed...? - thread::spawn(move || { - // Sleep until the timeout. - thread::sleep(timeout); - + let mut num_ticks = 0; + epoch_ticker(move || { + // Check if execution completed. if execution_done_flag2.load(Ordering::Relaxed) { - // The reducer completed successfully. - return; + return None; } - // Reducer is still running. - // Terminate V8 execution. - isolate_handle.terminate_execution(); + // We've reached the number of ticks to call `callback`. + if num_ticks % callback_every == 0 && isolate_handle.request_interrupt(callback, ptr::null_mut()) { + return None; + } + + if num_ticks == max_ticks { + // Execution still ongoing while budget has been exhausted. + // Terminate V8 execution. + // This implements "gas" for v8. + isolate_handle.terminate_execution(); + } + + num_ticks += 1; + Some(()) }); execution_done_flag @@ -379,9 +468,18 @@ fn call_call_reducer( /// Extracts the raw module def by running `__describe_module__` in `program`. fn extract_description(program: &str) -> Result { - let (_, ret) = with_script(program, ReducerBudget::DEFAULT_BUDGET, |scope, _| { - run_describer(log_traceback, || call_describe_module(scope)) - }); + let budget = ReducerBudget::DEFAULT_BUDGET; + let callback_every = EPOCH_TICKS_PER_SECOND; + extern "C" fn callback(_: &mut Isolate, _: *mut c_void) {} + + let (_, ret) = with_script( + Isolate::new(<_>::default()), + program, + callback_every, + callback, + budget, + |scope, _| run_describer(log_traceback, || call_describe_module(scope)), + ); ret } diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index f2b29310ff9..3393e57a579 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -320,7 +320,7 @@ impl ResourceSlab { decl_index!(RowIterIdx => std::vec::IntoIter>); pub(super) type RowIters = ResourceSlab; -pub(super) struct TimingSpan { +pub(crate) struct TimingSpan { pub start: Instant, pub name: String, } diff --git a/crates/core/src/host/wasmtime/mod.rs b/crates/core/src/host/wasmtime/mod.rs index 5a61d5e23ab..f43390cd958 100644 --- a/crates/core/src/host/wasmtime/mod.rs +++ b/crates/core/src/host/wasmtime/mod.rs @@ -27,7 +27,21 @@ pub struct WasmtimeRuntime { const EPOCH_TICK_LENGTH: Duration = Duration::from_millis(10); -const EPOCH_TICKS_PER_SECOND: u64 = Duration::from_secs(1).div_duration_f64(EPOCH_TICK_LENGTH) as u64; +pub(crate) const EPOCH_TICKS_PER_SECOND: u64 = ticks_in_duration(Duration::from_secs(1)); + +pub(crate) const fn ticks_in_duration(duration: Duration) -> u64 { + duration.div_duration_f64(EPOCH_TICK_LENGTH) as u64 +} + +pub(crate) fn epoch_ticker(mut on_tick: impl 'static + Send + FnMut() -> Option<()>) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(EPOCH_TICK_LENGTH); + loop { + interval.tick().await; + let Some(()) = on_tick() else { return; }; + } + }); +} impl WasmtimeRuntime { pub fn new(data_dir: Option<&ServerDataDir>) -> Self { @@ -53,13 +67,10 @@ impl WasmtimeRuntime { let engine = Engine::new(&config).unwrap(); let weak_engine = engine.weak(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(EPOCH_TICK_LENGTH); - loop { - interval.tick().await; - let Some(engine) = weak_engine.upgrade() else { break }; - engine.increment_epoch(); - } + epoch_ticker(move || { + let engine = weak_engine.upgrade()?; + engine.increment_epoch(); + Some(()) }); let mut linker = Box::new(Linker::new(&engine)); diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 700883ae4c5..61f3f0d61df 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -1354,7 +1354,7 @@ impl WasmInstanceEnv { // as we want to possibly trap, but not to return an error code. Self::with_span(caller, AbiCall::Identity, |caller| { let (mem, env) = Self::mem_env(caller); - let identity = env.instance_env.replica_ctx.database.database_identity; + let identity = env.instance_env.database_identity(); // We're implicitly casting `out_ptr` to `WasmPtr` here. // (Both types are actually `u32`.) // This works because `Identity::write_to` does not require an aligned pointer, From ff65ff2470577b3fc428caafa161b93865fd1022 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 12 Sep 2025 19:18:14 +0200 Subject: [PATCH 10/39] wip --- crates/core/src/host/instance_env.rs | 28 ++ crates/core/src/host/module_common.rs | 4 +- crates/core/src/host/v8/de.rs | 2 +- crates/core/src/host/v8/error.rs | 4 +- crates/core/src/host/v8/mod.rs | 369 +++++++++++++++++- crates/core/src/host/wasmtime/mod.rs | 4 +- .../src/host/wasmtime/wasm_instance_env.rs | 22 +- 7 files changed, 405 insertions(+), 28 deletions(-) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 99b2e7cf58d..5dc7973fd16 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -19,6 +19,7 @@ use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::table::RowRef; use std::ops::DerefMut; use std::sync::Arc; +use std::vec::IntoIter; #[derive(Clone)] pub struct InstanceEnv { @@ -491,6 +492,33 @@ impl InstanceEnv { Ok(chunks) } + + pub fn fill_buffer_from_iter( + iter: &mut IntoIter>, + mut buffer: &mut [u8], + chunk_pool: &mut ChunkPool, + ) -> usize { + let mut written = 0; + // Fill the buffer as much as possible. + while let Some(chunk) = iter.as_slice().first() { + let Some((buf_chunk, rest)) = buffer.split_at_mut_checked(chunk.len()) else { + // Cannot fit chunk into the buffer, + // either because we already filled it too much, + // or because it is too small. + break; + }; + buf_chunk.copy_from_slice(chunk); + written += chunk.len(); + buffer = rest; + + // Advance the iterator, as we used a chunk. + // SAFETY: We peeked one `chunk`, so there must be one at least. + let chunk = unsafe { iter.next().unwrap_unchecked() }; + chunk_pool.put(chunk); + } + + written + } } impl TxSlot { diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index 06f3da0eadf..d5826ff54c5 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -4,7 +4,9 @@ use crate::{ energy::EnergyMonitor, host::{ - module_host::{DynModule, ModuleInfo}, wasm_common::{module_host_actor::DescribeError, DESCRIBE_MODULE_DUNDER}, Scheduler + module_host::{DynModule, ModuleInfo}, + wasm_common::{module_host_actor::DescribeError, DESCRIBE_MODULE_DUNDER}, + Scheduler, }, module_host_context::ModuleCreationContext, replica_context::ReplicaContext, diff --git a/crates/core/src/host/v8/de.rs b/crates/core/src/host/v8/de.rs index 3103cf14095..7828d5bdf65 100644 --- a/crates/core/src/host/v8/de.rs +++ b/crates/core/src/host/v8/de.rs @@ -92,7 +92,7 @@ impl de::Error for Error<'_> { } /// Returns a scratch buffer to fill when deserializing strings. -fn scratch_buf() -> [MaybeUninit; N] { +pub(crate) fn scratch_buf() -> [MaybeUninit; N] { [const { MaybeUninit::uninit() }; N] } diff --git a/crates/core/src/host/v8/error.rs b/crates/core/src/host/v8/error.rs index bfcbc0715ce..9491d326ce8 100644 --- a/crates/core/src/host/v8/error.rs +++ b/crates/core/src/host/v8/error.rs @@ -59,12 +59,12 @@ impl<'scope, M: IntoJsString> IntoException<'scope> for RangeError { } #[derive(Debug)] -pub(super) struct ExceptionThrown { +pub(crate) struct ExceptionThrown { _priv: (), } /// A result where the error indicates that an exception has already been thrown in V8. -pub(super) type ExcResult = Result; +pub(crate) type ExcResult = Result; /// Indicates that the JS side had thrown an exception. pub(super) fn exception_already_thrown() -> ExceptionThrown { diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index ab77efd6eac..62a9979de56 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -3,15 +3,19 @@ use super::module_common::{build_common_module_from_raw, ModuleCommon}; use super::module_host::{CallReducerParams, DynModule, Module, ModuleInfo, ModuleInstance, ModuleRuntime}; use super::UpdateDatabaseResult; -use crate::host::instance_env::InstanceEnv; +use crate::database_logger::{BacktraceFrame, BacktraceProvider, ModuleBacktrace, Record}; +use crate::host::instance_env::{ChunkPool, InstanceEnv}; use crate::host::module_common::run_describer; +use crate::host::v8::de::{scratch_buf, v8_interned_string}; use crate::host::wasm_common::instrumentation::CallTimes; use crate::host::wasm_common::module_host_actor::{ DescribeError, EnergyStats, ExecuteResult, ExecutionTimings, InstanceCommon, ReducerOp, }; +use crate::host::wasm_common::{RowIterIdx, RowIters, TimingSpan, TimingSpanIdx, TimingSpanSet}; use crate::host::wasmtime::{epoch_ticker, ticks_in_duration, EPOCH_TICKS_PER_SECOND}; use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; +use anyhow::Context as _; use core::ffi::c_void; use core::sync::atomic::{AtomicBool, Ordering}; use core::time::Duration; @@ -26,10 +30,13 @@ use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::traits::Program; use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; +use spacetimedb_primitives::{ColId, IndexId, TableId}; +use spacetimedb_sats::Serialize; use std::sync::{Arc, LazyLock}; use std::time::Instant; use v8::{ - Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, OwnedIsolate, Value, + Context, ContextOptions, ContextScope, Function, FunctionCallbackArguments, HandleScope, Isolate, IsolateHandle, + Local, Object, OwnedIsolate, Value, }; mod de; @@ -142,7 +149,10 @@ impl Module for JsModule { instance_env, reducer_start: Instant::now(), call_times: CallTimes::new(), + iters: Default::default(), reducer_name: String::from(""), + chunk_pool: <_>::default(), + timing_spans: <_>::default(), }); // NOTE(centril): We don't need to do `extract_description` here @@ -172,6 +182,12 @@ fn env_on_instance(inst: &mut JsInstance) -> &mut JsInstanceEnv { struct JsInstanceEnv { instance_env: InstanceEnv, + /// The slab of `BufferIters` created for this instance. + iters: RowIters, + + /// Track time spent in module-defined spans. + timing_spans: TimingSpanSet, + /// The point in time the last reducer call started at. reducer_start: Instant, @@ -183,6 +199,10 @@ struct JsInstanceEnv { /// The last, including current, reducer to be executed by this environment. reducer_name: String, + + /// A pool of unused allocated chunks that can be reused. + // TODO(Centril): consider using this pool for `console_timer_start` and `bytes_sink_write`. + chunk_pool: ChunkPool, } impl JsInstanceEnv { @@ -308,6 +328,9 @@ fn with_script( with_scope(isolate, callback_every, callback, budget, |scope| { let code = v8::String::new(scope, code).unwrap(); let script_val = v8::Script::compile(scope, code, None).unwrap().run(scope).unwrap(); + + register_host_funs(scope); + logic(scope, script_val) }) } @@ -395,14 +418,16 @@ fn duration_to_budget(_duration: Duration) -> ReducerBudget { ReducerBudget::ZERO } +fn global<'scope>(scope: &mut HandleScope<'scope>) -> Local<'scope, Object> { + scope.get_current_context().global(scope) +} + /// Returns the global property `key`. fn get_global_property<'scope>( scope: &mut HandleScope<'scope>, key: Local<'scope, v8::String>, ) -> ExcResult> { - scope - .get_current_context() - .global(scope) + global(scope) .get(scope, key.into()) .ok_or_else(exception_already_thrown) } @@ -505,6 +530,340 @@ fn call_describe_module(scope: &mut HandleScope<'_>) -> anyhow::Result(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let name: &str = deserialize_js(scope, args.get(0))?; + let id = env_on_isolate(scope).instance_env.table_id_from_name(name).unwrap(); + let ret = serialize_to_js(scope, &id)?; + Ok(ret) +} + +fn index_id_from_name<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let name: &str = deserialize_js(scope, args.get(0))?; + let id = env_on_isolate(scope).instance_env.index_id_from_name(name).unwrap(); + let ret = serialize_to_js(scope, &id)?; + Ok(ret) +} + +fn datastore_table_row_count<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let count = env_on_isolate(scope) + .instance_env + .datastore_table_row_count(table_id) + .unwrap(); + serialize_to_js(scope, &count) +} + +fn datastore_table_scan_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + + let env = env_on_isolate(scope); + // Collect the iterator chunks. + let chunks = env + .instance_env + .datastore_table_scan_bsatn_chunks(&mut env.chunk_pool, table_id) + .unwrap(); + + // Register the iterator and get back the index to write to `out`. + // Calls to the iterator are done through dynamic dispatch. + let idx = env.iters.insert(chunks.into_iter()); + + let ret = serialize_to_js(scope, &idx.0)?; + Ok(ret) +} + +fn convert_u32_to_col_id(col_id: u32) -> anyhow::Result { + let col_id: u16 = col_id.try_into().context("ABI violation, a `ColId` must be a `u16`")?; + Ok(col_id.into()) +} + +fn datastore_index_scan_range_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let index_id: IndexId = deserialize_js(scope, args.get(0))?; + + let prefix_elems: u32 = deserialize_js(scope, args.get(2))?; + let prefix_elems = convert_u32_to_col_id(prefix_elems).unwrap(); + + let prefix: &[u8] = if prefix_elems.idx() == 0 { + &[] + } else { + deserialize_js(scope, args.get(1))? + }; + + let rstart: &[u8] = deserialize_js(scope, args.get(3))?; + let rend: &[u8] = deserialize_js(scope, args.get(4))?; + + let env = env_on_isolate(scope); + + // Find the relevant rows. + let chunks = env + .instance_env + .datastore_index_scan_range_bsatn_chunks(&mut env.chunk_pool, index_id, prefix, prefix_elems, rstart, rend) + .unwrap(); + + // Insert the encoded + concatenated rows into a new buffer and return its id. + let idx = env.iters.insert(chunks.into_iter()); + + let ret = serialize_to_js(scope, &idx.0)?; + Ok(ret) +} + +fn row_iter_bsatn_advance<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; + let row_iter_idx = RowIterIdx(row_iter_idx); + let buffer_max_len: u32 = deserialize_js(scope, args.get(1))?; + + // Retrieve the iterator by `row_iter_idx`, or error. + let env = env_on_isolate(scope); + let iter = env.iters.get_mut(row_iter_idx).unwrap(); + + // Allocate a buffer with `buffer_max_len` capacity. + let mut buffer = vec![0; buffer_max_len as usize]; + // Fill the buffer as much as possible. + let written = InstanceEnv::fill_buffer_from_iter(iter, &mut buffer, &mut env.chunk_pool); + buffer.truncate(written); + + let ret = match (written, iter.as_slice().first()) { + // Nothing was written and the iterator is not exhausted. + (0, Some(_chunk)) => { + unimplemented!() + } + // The iterator is exhausted, destroy it, and tell the caller. + (_, None) => { + env.iters.take(row_iter_idx); + serialize_to_js(scope, &AdvanceRet { flag: -1, buffer })? + } + // Something was written, but the iterator is not exhausted. + (_, Some(_)) => serialize_to_js(scope, &AdvanceRet { flag: 0, buffer })?, + }; + Ok(ret) +} + +#[derive(Serialize)] +struct AdvanceRet { + buffer: Vec, + flag: i32, +} + +fn row_iter_bsatn_close<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; + let row_iter_idx = RowIterIdx(row_iter_idx); + + // Retrieve the iterator by `row_iter_idx`, or error. + let env = env_on_isolate(scope); + + // Retrieve the iterator by `row_iter_idx`, or error. + Ok(match env.iters.take(row_iter_idx) { + None => unimplemented!(), + // TODO(Centril): consider putting these into a pool for reuse. + Some(_) => serialize_to_js(scope, &0u32)?, + }) +} + +fn datastore_insert_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let mut row: Vec = deserialize_js(scope, args.get(1))?; + + // Insert the row into the DB and write back the generated column values. + let env: &mut JsInstanceEnv = env_on_isolate(scope); + let row_len = env.instance_env.insert(table_id, &mut row).unwrap(); + row.truncate(row_len); + + serialize_to_js(scope, &row) +} + +fn datastore_update_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let index_id: IndexId = deserialize_js(scope, args.get(1))?; + let mut row: Vec = deserialize_js(scope, args.get(2))?; + + // Insert the row into the DB and write back the generated column values. + let env: &mut JsInstanceEnv = env_on_isolate(scope); + let row_len = env.instance_env.update(table_id, index_id, &mut row).unwrap(); + row.truncate(row_len); + + serialize_to_js(scope, &row) +} + +fn datastore_delete_by_index_scan_range_bsatn<'s>( + scope: &mut HandleScope<'s>, + args: FunctionCallbackArguments<'s>, +) -> FnRet<'s> { + let index_id: IndexId = deserialize_js(scope, args.get(0))?; + + let prefix_elems: u32 = deserialize_js(scope, args.get(2))?; + let prefix_elems = convert_u32_to_col_id(prefix_elems).unwrap(); + + let prefix: &[u8] = if prefix_elems.idx() == 0 { + &[] + } else { + deserialize_js(scope, args.get(1))? + }; + + let rstart: &[u8] = deserialize_js(scope, args.get(3))?; + let rend: &[u8] = deserialize_js(scope, args.get(4))?; + + let env = env_on_isolate(scope); + + // Delete the relevant rows. + let num = env + .instance_env + .datastore_delete_by_index_scan_range_bsatn(index_id, prefix, prefix_elems, rstart, rend) + .unwrap(); + + serialize_to_js(scope, &num) +} + +fn datastore_delete_all_by_eq_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let relation: &[u8] = deserialize_js(scope, args.get(1))?; + + let env = env_on_isolate(scope); + let num = env + .instance_env + .datastore_delete_all_by_eq_bsatn(table_id, relation) + .unwrap(); + + serialize_to_js(scope, &num) +} + +fn volatile_nonatomic_schedule_immediate<'s>( + scope: &mut HandleScope<'s>, + args: FunctionCallbackArguments<'s>, +) -> FnRet<'s> { + let name: String = deserialize_js(scope, args.get(0))?; + let args: Vec = deserialize_js(scope, args.get(1))?; + + let env = env_on_isolate(scope); + env.instance_env + .scheduler + .volatile_nonatomic_schedule_immediate(name, crate::host::ReducerArgs::Bsatn(args.into())); + + Ok(v8::undefined(scope).into()) +} + +fn console_log<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let level: u32 = deserialize_js(scope, args.get(0))?; + + let msg = args.get(1).cast::(); + let mut buf = scratch_buf::<128>(); + let msg = msg.to_rust_cow_lossy(scope, &mut buf); + let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) + .ok_or_else(exception_already_thrown)? + .get_frame(scope, 1) + .ok_or_else(exception_already_thrown)?; + let mut buf = scratch_buf::<32>(); + let filename = frame + .get_script_name(scope) + .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); + let record = Record { + // TODO: figure out whether to use walltime now or logical reducer now (env.reducer_start) + ts: chrono::Utc::now(), + target: None, + filename: filename.as_deref(), + line_number: Some(frame.get_line_number() as u32), + message: &msg, + }; + + let env = env_on_isolate(scope); + env.instance_env.console_log((level as u8).into(), &record, &Noop); + + Ok(v8::undefined(scope).into()) +} + +struct Noop; +impl BacktraceProvider for Noop { + fn capture(&self) -> Box { + Box::new(Noop) + } +} +impl ModuleBacktrace for Noop { + fn frames(&self) -> Vec> { + Vec::new() + } +} + +fn console_timer_start<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let name = args.get(0).cast::(); + let mut buf = scratch_buf::<128>(); + let name = name.to_rust_cow_lossy(scope, &mut buf).into_owned(); + + let env = env_on_isolate(scope); + let span_id = env.timing_spans.insert(TimingSpan::new(name)).0; + serialize_to_js(scope, &span_id) +} + +fn console_timer_end<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let span_id: u32 = deserialize_js(scope, args.get(0))?; + + let env = env_on_isolate(scope); + let span = env.timing_spans.take(TimingSpanIdx(span_id)).unwrap(); + env.instance_env.console_timer_end(&span, &Noop); + + serialize_to_js(scope, &0u32) +} + +fn identity<'s>(scope: &mut HandleScope<'s>, _: FunctionCallbackArguments<'s>) -> FnRet<'s> { + let env = env_on_isolate(scope); + let identity = *env.instance_env.database_identity(); + serialize_to_js(scope, &identity) +} + +fn register_host_funs(scope: &mut HandleScope<'_>) { + register_host_fun(scope, "table_id_from_name", table_id_from_name); + register_host_fun(scope, "index_id_from_name", index_id_from_name); + register_host_fun(scope, "datastore_table_row_count", datastore_table_row_count); + register_host_fun(scope, "datastore_table_scan_bsatn", datastore_table_scan_bsatn); + register_host_fun( + scope, + "datastore_index_scan_range_bsatn", + datastore_index_scan_range_bsatn, + ); + register_host_fun(scope, "row_iter_bsatn_advance", row_iter_bsatn_advance); + register_host_fun(scope, "row_iter_bsatn_close", row_iter_bsatn_close); + register_host_fun(scope, "datastore_insert_bsatn", datastore_insert_bsatn); + register_host_fun(scope, "datastore_update_bsatn", datastore_update_bsatn); + register_host_fun( + scope, + "datastore_delete_by_index_scan_range_bsatn", + datastore_delete_by_index_scan_range_bsatn, + ); + register_host_fun( + scope, + "datastore_delete_all_by_eq_bsatn", + datastore_delete_all_by_eq_bsatn, + ); + register_host_fun( + scope, + "volatile_nonatomic_schedule_immediate", + volatile_nonatomic_schedule_immediate, + ); + register_host_fun(scope, "console_log", console_log); + register_host_fun(scope, "console_timer_start", console_timer_start); + register_host_fun(scope, "console_timer_end", console_timer_end); + register_host_fun(scope, "identity", identity); +} + +type FnRet<'s> = ExcResult>; + +fn register_host_fun( + scope: &mut HandleScope<'_>, + name: &str, + fun: impl for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, +) { + let name = v8_interned_string(scope, name).into(); + let fun = Function::new(scope, &adapt_fun(fun)).unwrap().into(); + global(scope).set(scope, name, fun).unwrap(); +} + +fn adapt_fun( + fun: impl for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, +) -> impl for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>, v8::ReturnValue) { + move |scope, args, mut rv| { + if let Ok(value) = fun(scope, args) { + rv.set(value); + } + } +} + #[cfg(test)] mod test { use super::*; diff --git a/crates/core/src/host/wasmtime/mod.rs b/crates/core/src/host/wasmtime/mod.rs index f43390cd958..06d80f85da1 100644 --- a/crates/core/src/host/wasmtime/mod.rs +++ b/crates/core/src/host/wasmtime/mod.rs @@ -38,7 +38,9 @@ pub(crate) fn epoch_ticker(mut on_tick: impl 'static + Send + FnMut() -> Option< let mut interval = tokio::time::interval(EPOCH_TICK_LENGTH); loop { interval.tick().await; - let Some(()) = on_tick() else { return; }; + let Some(()) = on_tick() else { + return; + }; } }); } diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 61f3f0d61df..10d4065d999 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -123,6 +123,7 @@ pub(super) struct WasmInstanceEnv { /// The last, including current, reducer to be executed by this environment. reducer_name: String, + /// A pool of unused allocated chunks that can be reused. // TODO(Centril): consider using this pool for `console_timer_start` and `bytes_sink_write`. chunk_pool: ChunkPool, @@ -697,27 +698,12 @@ impl WasmInstanceEnv { // Read `buffer_len`, i.e., the capacity of `buffer` pointed to by `buffer_ptr`. let buffer_len = u32::read_from(mem, buffer_len_ptr)?; let write_buffer_len = |mem, len| u32::try_from(len).unwrap().write_to(mem, buffer_len_ptr); + // Get a mutable view to the `buffer`. - let mut buffer = mem.deref_slice_mut(buffer_ptr, buffer_len)?; + let buffer = mem.deref_slice_mut(buffer_ptr, buffer_len)?; - let mut written = 0; // Fill the buffer as much as possible. - while let Some(chunk) = iter.as_slice().first() { - let Some((buf_chunk, rest)) = buffer.split_at_mut_checked(chunk.len()) else { - // Cannot fit chunk into the buffer, - // either because we already filled it too much, - // or because it is too small. - break; - }; - buf_chunk.copy_from_slice(chunk); - written += chunk.len(); - buffer = rest; - - // Advance the iterator, as we used a chunk. - // SAFETY: We peeked one `chunk`, so there must be one at least. - let chunk = unsafe { iter.next().unwrap_unchecked() }; - env.chunk_pool.put(chunk); - } + let written = InstanceEnv::fill_buffer_from_iter(iter, buffer, &mut env.chunk_pool); let ret = match (written, iter.as_slice().first()) { // Nothing was written and the iterator is not exhausted. From 58720b3eb59495a3d01cc236ecb443de9097b567 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 12 Sep 2025 23:38:47 +0200 Subject: [PATCH 11/39] v8: fix register_host_fun / adapt_fun --- crates/core/src/host/v8/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 62a9979de56..ae8768f8561 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -36,7 +36,7 @@ use std::sync::{Arc, LazyLock}; use std::time::Instant; use v8::{ Context, ContextOptions, ContextScope, Function, FunctionCallbackArguments, HandleScope, Isolate, IsolateHandle, - Local, Object, OwnedIsolate, Value, + Local, Object, OwnedIsolate, ReturnValue, Value, }; mod de; @@ -847,16 +847,16 @@ type FnRet<'s> = ExcResult>; fn register_host_fun( scope: &mut HandleScope<'_>, name: &str, - fun: impl for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, + fun: impl Copy + for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, ) { let name = v8_interned_string(scope, name).into(); - let fun = Function::new(scope, &adapt_fun(fun)).unwrap().into(); + let fun = Function::new(scope, adapt_fun(fun)).unwrap().into(); global(scope).set(scope, name, fun).unwrap(); } fn adapt_fun( - fun: impl for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, -) -> impl for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>, v8::ReturnValue) { + fun: impl Copy + for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, +) -> impl Copy + for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>, ReturnValue) { move |scope, args, mut rv| { if let Ok(value) = fun(scope, args) { rv.set(value); From 8ff09eabce2af20f6df44fa6053318e764efc201 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Tue, 23 Sep 2025 14:10:41 +0200 Subject: [PATCH 12/39] wip --- Cargo.lock | 363 +++++- Cargo.toml | 2 +- crates/core/src/host/instance_env.rs | 19 +- crates/core/src/host/v8/error.rs | 105 +- crates/core/src/host/v8/mod.rs | 505 ++------ crates/core/src/host/v8/ser.rs | 6 +- crates/core/src/host/v8/syscall.rs | 1061 +++++++++++++++++ crates/core/src/host/wasm_common.rs | 13 + .../src/host/wasm_common/instrumentation.rs | 5 + .../src/host/wasmtime/wasm_instance_env.rs | 43 +- 10 files changed, 1662 insertions(+), 460 deletions(-) create mode 100644 crates/core/src/host/v8/syscall.rs diff --git a/Cargo.lock b/Cargo.lock index a5ac90b2aa4..19f2bcb4acd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.71.1" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ "bitflags 2.9.0", "cexpr", @@ -730,6 +730,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "calendrical_calculations" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c5d386a9f2c8b97e1a036420bcf937db4e5c9df33eb0232025008ced6104c0" +dependencies = [ + "core_maths", + "displaydoc", +] + [[package]] name = "camino" version = "1.1.9" @@ -1099,6 +1109,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + [[package]] name = "cpp_demangle" version = "0.4.4" @@ -1610,6 +1629,38 @@ dependencies = [ "subtle", ] +[[package]] +name = "diplomat" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced081520ee8cf2b8c5b64a1a901eccd7030ece670dac274afe64607d6499b71" +dependencies = [ + "diplomat_core", + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "diplomat-runtime" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "098f9520ec5c190943b083bca3ea4cc4e67dc5f85a37062e528ecf1d25f04eb4" + +[[package]] +name = "diplomat_core" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad619d9fdee0e731bb6f8f7d797b6ecfdc2395e363f554d2f6377155955171eb" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "smallvec", + "strck", + "syn 2.0.101", +] + [[package]] name = "directories-next" version = "2.0.0" @@ -2668,6 +2719,29 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_calendar" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f6c40ba6481ed7ddd358437af0f000eb9f661345845977d9d1b38e606374e1f" +dependencies = [ + "calendrical_calculations", + "displaydoc", + "icu_calendar_data", + "icu_locale", + "icu_locale_core", + "icu_provider 2.0.0", + "tinystr 0.8.1", + "writeable 0.6.1", + "zerovec 0.11.4", +] + +[[package]] +name = "icu_calendar_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7219c8639ab936713a87b571eed2bc2615aa9137e8af6eb221446ee5644acc18" + [[package]] name = "icu_collections" version = "1.5.0" @@ -2675,11 +2749,59 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ "displaydoc", - "yoke", + "yoke 0.7.5", + "zerofrom", + "zerovec 0.10.4", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke 0.8.0", "zerofrom", - "zerovec", + "zerovec 0.11.4", +] + +[[package]] +name = "icu_locale" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ae5921528335e91da1b6c695dbf1ec37df5ac13faa3f91e5640be93aa2fbefd" +dependencies = [ + "displaydoc", + "icu_collections 2.0.0", + "icu_locale_core", + "icu_locale_data", + "icu_provider 2.0.0", + "potential_utf", + "tinystr 0.8.1", + "zerovec 0.11.4", ] +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap 0.8.0", + "tinystr 0.8.1", + "writeable 0.6.1", + "zerovec 0.11.4", +] + +[[package]] +name = "icu_locale_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fdef0c124749d06a743c69e938350816554eb63ac979166590e2b4ee4252765" + [[package]] name = "icu_locid" version = "1.5.0" @@ -2687,10 +2809,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", + "litemap 0.7.5", + "tinystr 0.7.6", + "writeable 0.5.5", + "zerovec 0.10.4", ] [[package]] @@ -2702,9 +2824,9 @@ dependencies = [ "displaydoc", "icu_locid", "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", + "icu_provider 1.5.0", + "tinystr 0.7.6", + "zerovec 0.10.4", ] [[package]] @@ -2720,15 +2842,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" dependencies = [ "displaydoc", - "icu_collections", + "icu_collections 1.5.0", "icu_normalizer_data", "icu_properties", - "icu_provider", + "icu_provider 1.5.0", "smallvec", "utf16_iter", "utf8_iter", "write16", - "zerovec", + "zerovec 0.10.4", ] [[package]] @@ -2744,12 +2866,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" dependencies = [ "displaydoc", - "icu_collections", + "icu_collections 1.5.0", "icu_locid_transform", "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", + "icu_provider 1.5.0", + "tinystr 0.7.6", + "zerovec 0.10.4", ] [[package]] @@ -2768,11 +2890,28 @@ dependencies = [ "icu_locid", "icu_provider_macros", "stable_deref_trait", - "tinystr", - "writeable", - "yoke", + "tinystr 0.7.6", + "writeable 0.5.5", + "yoke 0.7.5", "zerofrom", - "zerovec", + "zerovec 0.10.4", +] + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr 0.8.1", + "writeable 0.6.1", + "yoke 0.8.0", + "zerofrom", + "zerotrie", + "zerovec 0.11.4", ] [[package]] @@ -3004,6 +3143,15 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "ixdtf" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ef2d119044a672ceb96e59608dffe8677f29dc6ec48ed693a4b9ac84751e9b" +dependencies = [ + "displaydoc", +] + [[package]] name = "jemalloc_pprof" version = "0.8.1" @@ -3021,6 +3169,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524" + [[package]] name = "jni" version = "0.21.1" @@ -3220,6 +3374,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "lock_api" version = "0.4.12" @@ -4103,6 +4263,16 @@ dependencies = [ "postgres-protocol", ] +[[package]] +name = "potential_utf" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" +dependencies = [ + "serde", + "zerovec 0.11.4", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -5022,6 +5192,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "ry_temporal_capi" +version = "0.0.11-ry.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdfddefd45ee4814bd83d94b7196c95ef6af159f4a0035a6c67bd59edcff14ee" +dependencies = [ + "diplomat", + "diplomat-runtime", + "icu_calendar", + "icu_locale", + "num-traits", + "temporal_rs", + "writeable 0.6.1", +] + [[package]] name = "ryu" version = "1.0.20" @@ -6635,6 +6820,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" +[[package]] +name = "strck" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42316e70da376f3d113a68d138a60d8a9883c604fe97942721ec2068dab13a9f" +dependencies = [ + "unicode-ident", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -6905,6 +7099,25 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "temporal_rs" +version = "0.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7807e330b12e288b847a3e2a2b0dcd41ca764d0f90f9e8940f02c6ddd68cd2d7" +dependencies = [ + "combine", + "core_maths", + "icu_calendar", + "icu_locale", + "ixdtf", + "jiff-tzdb", + "num-traits", + "timezone_provider", + "tinystr 0.8.1", + "tzif", + "writeable 0.6.1", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -7079,6 +7292,17 @@ dependencies = [ "time-core", ] +[[package]] +name = "timezone_provider" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f357f8e2cddee6a7b56b69fbb4cab30a7e82914c80ee7f9a5eb799ee3de3f24d" +dependencies = [ + "tinystr 0.8.1", + "zerotrie", + "zerovec 0.11.4", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -7086,7 +7310,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ "displaydoc", - "zerovec", + "zerovec 0.10.4", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec 0.11.4", ] [[package]] @@ -7525,6 +7759,15 @@ dependencies = [ "spacetimedb 1.3.0", ] +[[package]] +name = "tzif" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e762ac355f0c204d09ae644b3d59423d5ddfc5603997d60c8c56f24e429a9d" +dependencies = [ + "combine", +] + [[package]] name = "unarray" version = "0.1.4" @@ -7665,17 +7908,18 @@ dependencies = [ [[package]] name = "v8" -version = "137.2.1" +version = "140.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca393e2032ddba2a57169e15cac5d0a81cdb3d872a8886f4468bc0f486098d2" +checksum = "4e0ad6613428e148bb814ee8890a23bb250547a8f837922ca1ea6eba90670514" dependencies = [ - "bindgen 0.71.1", + "bindgen 0.72.1", "bitflags 2.9.0", "fslock", "gzip-header", "home", "miniz_oxide", "paste", + "ry_temporal_capi", "which 6.0.3", ] @@ -8692,6 +8936,12 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + [[package]] name = "wyz" version = "0.5.1" @@ -8755,7 +9005,19 @@ checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive", + "yoke-derive 0.7.5", + "zerofrom", +] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive 0.8.0", "zerofrom", ] @@ -8771,6 +9033,18 @@ dependencies = [ "synstructure 0.13.2", ] +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", + "synstructure 0.13.2", +] + [[package]] name = "zerocopy" version = "0.8.25" @@ -8832,15 +9106,35 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", +] + [[package]] name = "zerovec" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ - "yoke", + "yoke 0.7.5", + "zerofrom", + "zerovec-derive 0.10.3", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke 0.8.0", "zerofrom", - "zerovec-derive", + "zerovec-derive 0.11.1", ] [[package]] @@ -8854,6 +9148,17 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "zip" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index f84eff4df7a..9c1be60a881 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -292,7 +292,7 @@ unicode-normalization = "0.1.23" url = "2.3.1" urlencoding = "2.1.2" uuid = { version = "1.2.1", features = ["v4"] } -v8 = "137.2" +v8 = "140.0" walkdir = "2.2.5" wasmbin = "0.6" webbrowser = "1.0.2" diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 5dc7973fd16..7ffa1da3f9e 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,5 +1,5 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; -use crate::database_logger::{BacktraceProvider, LogLevel, Record}; +use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; use crate::host::wasm_common::TimingSpan; @@ -197,10 +197,23 @@ impl InstanceEnv { } /// End a console timer by logging the span at INFO level. - pub fn console_timer_end(&self, span: &TimingSpan, function: Option<&str>, bt: &dyn BacktraceProvider) -> RtResult { + pub fn console_timer_end(&self, span: &TimingSpan, function: Option<&str>) { let elapsed = span.start.elapsed(); let message = format!("Timing span {:?}: {:?}", &span.name, elapsed); + /// A backtrace provider that provides nothing. + struct Noop; + impl BacktraceProvider for Noop { + fn capture(&self) -> Box { + Box::new(Noop) + } + } + impl ModuleBacktrace for Noop { + fn frames(&self) -> Vec> { + Vec::new() + } + } + let record = Record { ts: chrono::Utc::now(), target: None, @@ -209,7 +222,7 @@ impl InstanceEnv { function, message: &message, }; - self.console_log(LogLevel::Info, &record, bt); + self.console_log(LogLevel::Info, &record, &Noop); } /// Project `cols` in `row_ref` encoded in BSATN to `buffer` diff --git a/crates/core/src/host/v8/error.rs b/crates/core/src/host/v8/error.rs index 9491d326ce8..49dd6f72e1b 100644 --- a/crates/core/src/host/v8/error.rs +++ b/crates/core/src/host/v8/error.rs @@ -1,6 +1,10 @@ //! Utilities for error handling when dealing with V8. +use crate::database_logger::{BacktraceFrame, BacktraceProvider, ModuleBacktrace}; + +use super::serialize_to_js; use core::fmt; +use spacetimedb_sats::Serialize; use v8::{Exception, HandleScope, Local, StackFrame, StackTrace, TryCatch, Value}; /// The result of trying to convert a [`Value`] in scope `'scope` to some type `T`. @@ -58,6 +62,60 @@ impl<'scope, M: IntoJsString> IntoException<'scope> for RangeError { } } +/// A catchable termination error thrown in callbacks to indicate a host error. +#[derive(Serialize)] +pub(super) struct TerminationError { + __terminated__: String, +} + +impl TerminationError { + /// Convert `anyhow::Error` to a termination error. + pub(super) fn from_error<'scope>( + scope: &mut HandleScope<'scope>, + error: &anyhow::Error, + ) -> ExcResult> { + let __terminated__ = format!("{error}"); + let error = Self { __terminated__ }; + serialize_to_js(scope, &error).map(ExceptionValue) + } +} + +/// A catchable error code thrown in callbacks +/// to indicate bad arguments to a syscall. +#[derive(Serialize)] +pub(super) struct CodeError { + __code_error__: u16, +} + +impl CodeError { + /// Create a code error from a code. + pub(super) fn from_code<'scope>( + scope: &mut HandleScope<'scope>, + __code_error__: u16, + ) -> ExcResult> { + let error = Self { __code_error__ }; + serialize_to_js(scope, &error).map(ExceptionValue) + } +} + +/// A catchable error code thrown in callbacks +/// to indicate that a buffer was too small and the minimum size required. +#[derive(Serialize)] +pub(super) struct BufferTooSmall { + __buffer_too_small__: u32, +} + +impl BufferTooSmall { + /// Create a code error from a code. + pub(super) fn from_requirement<'scope>( + scope: &mut HandleScope<'scope>, + __buffer_too_small__: u32, + ) -> ExcResult> { + let error = Self { __buffer_too_small__ }; + serialize_to_js(scope, &error).map(ExceptionValue) + } +} + #[derive(Debug)] pub(crate) struct ExceptionThrown { _priv: (), @@ -134,14 +192,14 @@ impl fmt::Display for JsError { } /// A V8 stack trace that is independent of a `'scope`. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub(super) struct JsStackTrace { frames: Box<[JsStackTraceFrame]>, } impl JsStackTrace { /// Converts a V8 [`StackTrace`] into one independent of `'scope`. - fn from_trace<'scope>(scope: &mut HandleScope<'scope>, trace: Local<'scope, StackTrace>) -> Self { + pub(super) fn from_trace<'scope>(scope: &mut HandleScope<'scope>, trace: Local<'scope, StackTrace>) -> Self { let frames = (0..trace.get_frame_count()) .map(|index| { let frame = trace.get_frame(scope, index).unwrap(); @@ -150,6 +208,12 @@ impl JsStackTrace { .collect::>(); Self { frames } } + + /// Construct a backtrace from `scope`. + pub(super) fn from_current_stack_trace(scope: &mut HandleScope<'_>) -> ExcResult { + let trace = StackTrace::current_stack_trace(scope, 1024).ok_or_else(exception_already_thrown)?; + Ok(Self::from_trace(scope, trace)) + } } impl fmt::Display for JsStackTrace { @@ -162,8 +226,26 @@ impl fmt::Display for JsStackTrace { } } +impl BacktraceProvider for JsStackTrace { + fn capture(&self) -> Box { + Box::new(self.clone()) + } +} + +impl ModuleBacktrace for JsStackTrace { + fn frames(&self) -> Vec> { + self.frames + .iter() + .map(|frame| BacktraceFrame { + module_name: frame.script_name.as_deref(), + func_name: frame.fn_name.as_deref(), + }) + .collect() + } +} + /// A V8 stack trace frame that is independent of a `'scope`. -#[derive(Debug)] +#[derive(Debug, Clone)] pub(super) struct JsStackTraceFrame { line: usize, column: usize, @@ -197,12 +279,22 @@ impl JsStackTraceFrame { is_user_js: frame.is_user_javascript(), } } + + /// Returns the name of the function that was called. + fn fn_name(&self) -> &str { + self.fn_name.as_deref().unwrap_or("") + } + + /// Returns the name of the script where the function resides. + fn script_name(&self) -> &str { + self.script_name.as_deref().unwrap_or("") + } } impl fmt::Display for JsStackTraceFrame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let fn_name = self.fn_name.as_deref().unwrap_or(""); - let script_name = self.script_name.as_deref().unwrap_or(""); + let fn_name = self.fn_name(); + let script_name = self.script_name(); // This isn't exactly the same format as chrome uses, // but it's close enough for now. @@ -267,7 +359,8 @@ pub(super) fn catch_exception<'scope, T>( body: impl FnOnce(&mut HandleScope<'scope>) -> Result>, ) -> Result> { let scope = &mut TryCatch::new(scope); - body(scope).map_err(|e| match e { + let ret = body(scope); + ret.map_err(|e| match e { ErrorOrException::Err(e) => ErrorOrException::Err(e), ErrorOrException::Exception(_) => ErrorOrException::Exception(JsError::from_caught(scope)), }) diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index ae8768f8561..9d6337231be 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -1,42 +1,40 @@ #![allow(dead_code)] -use super::module_common::{build_common_module_from_raw, ModuleCommon}; +use super::module_common::{build_common_module_from_raw, run_describer, ModuleCommon}; use super::module_host::{CallReducerParams, DynModule, Module, ModuleInfo, ModuleInstance, ModuleRuntime}; use super::UpdateDatabaseResult; -use crate::database_logger::{BacktraceFrame, BacktraceProvider, ModuleBacktrace, Record}; use crate::host::instance_env::{ChunkPool, InstanceEnv}; -use crate::host::module_common::run_describer; -use crate::host::v8::de::{scratch_buf, v8_interned_string}; +use crate::host::v8::error::{BufferTooSmall, JsStackTrace}; +use crate::host::v8::syscall::{register_host_funs, FnRet}; use crate::host::wasm_common::instrumentation::CallTimes; use crate::host::wasm_common::module_host_actor::{ DescribeError, EnergyStats, ExecuteResult, ExecutionTimings, InstanceCommon, ReducerOp, }; -use crate::host::wasm_common::{RowIterIdx, RowIters, TimingSpan, TimingSpanIdx, TimingSpanSet}; +use crate::host::wasm_common::{RowIters, TimingSpanSet}; use crate::host::wasmtime::{epoch_ticker, ticks_in_duration, EPOCH_TICKS_PER_SECOND}; use crate::host::ArgsTuple; use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; -use anyhow::Context as _; use core::ffi::c_void; use core::sync::atomic::{AtomicBool, Ordering}; use core::time::Duration; -use core::{ptr, str}; +use core::{iter, ptr, str}; use de::deserialize_js; -use error::{catch_exception, exception_already_thrown, log_traceback, ExcResult, Throwable}; +use error::{catch_exception, exception_already_thrown, log_traceback, CodeError, TerminationError, Throwable}; use from_value::cast; use key_cache::get_or_create_key_cache; use ser::serialize_to_js; use spacetimedb_client_api_messages::energy::ReducerBudget; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::traits::Program; -use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; use spacetimedb_primitives::{ColId, IndexId, TableId}; use spacetimedb_sats::Serialize; +use spacetimedb_schema::auto_migrate::MigrationPolicy; use std::sync::{Arc, LazyLock}; use std::time::Instant; use v8::{ - Context, ContextOptions, ContextScope, Function, FunctionCallbackArguments, HandleScope, Isolate, IsolateHandle, - Local, Object, OwnedIsolate, ReturnValue, Value, + Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, Object, OwnedIsolate, + Value, }; mod de; @@ -44,6 +42,7 @@ mod error; mod from_value; mod key_cache; mod ser; +mod syscall; mod to_value; /// The V8 runtime, for modules written in e.g., JS or TypeScript. @@ -73,21 +72,24 @@ struct V8RuntimeInner { } impl V8RuntimeInner { + /// Initializes the V8 platform and engine. + /// + /// Should only be called once but it isn't unsound to call it more times. fn init() -> Self { // Our current configuration: // - will pick a number of worker threads for background jobs based on the num CPUs. // - does not allow idle tasks - let platform = v8::new_default_platform(0, false).make_shared(); + let platform = v8::new_single_threaded_default_platform(false).make_shared(); // Initialize V8. Internally, this uses a global lock so it's safe that we don't. v8::V8::initialize_platform(platform); v8::V8::initialize(); Self { _priv: () } } +} +impl ModuleRuntime for V8RuntimeInner { fn make_actor(&self, mcc: ModuleCreationContext<'_>) -> anyhow::Result { - #![allow(unreachable_code, unused_variables)] - log::trace!( "Making new V8 module host actor for database {} with module {}", mcc.replica_ctx.database_identity, @@ -130,10 +132,10 @@ impl DynModule for JsModule { impl Module for JsModule { type Instance = JsInstance; - type InitialInstances<'a> = std::iter::Empty; + type InitialInstances<'a> = iter::Empty; fn initial_instances(&mut self) -> Self::InitialInstances<'_> { - std::iter::empty() + iter::empty() } fn info(&self) -> Arc { @@ -145,12 +147,12 @@ impl Module for JsModule { let common = &self.common; let instance_env = InstanceEnv::new(common.replica_ctx().clone(), common.scheduler().clone()); - let instance = Some(JsInstanceEnv { + let instance = JsInstanceEnvSlot::new(JsInstanceEnv { instance_env, reducer_start: Instant::now(), call_times: CallTimes::new(), - iters: Default::default(), - reducer_name: String::from(""), + iters: <_>::default(), + reducer_name: "".into(), chunk_pool: <_>::default(), timing_spans: <_>::default(), }); @@ -169,16 +171,51 @@ impl Module for JsModule { } } -const EXPECT_ENV: &str = "there should be a `JsInstanceEnv`"; +/// The [`JsInstance`]'s way of holding a [`JsInstanceEnv`] +/// with possible temporary extraction. +struct JsInstanceEnvSlot { + /// NOTE(centril): The `Option<_>` is due to moving the environment + /// into [`Isolate`]s and back. + instance: Option, +} -fn env_on_isolate(isolate: &mut Isolate) -> &mut JsInstanceEnv { - isolate.get_slot_mut().expect(EXPECT_ENV) +impl JsInstanceEnvSlot { + /// Creates a new slot to hold `instance`. + fn new(instance: JsInstanceEnv) -> Self { + Self { + instance: Some(instance), + } + } + + const EXPECT_ENV: &str = "there should be a `JsInstanceEnv`"; + + /// Provides exclusive access to the instance's environment, + /// assuming it hasn't been moved to an [`Isolate`]. + fn get_mut(&mut self) -> &mut JsInstanceEnv { + self.instance.as_mut().expect(Self::EXPECT_ENV) + } + + /// Moves the instance's environment to `isolate`, + /// assuming it hasn't already been moved there. + fn move_to_isolate(&mut self, isolate: &mut Isolate) { + isolate.set_slot(self.instance.take().expect(Self::EXPECT_ENV)); + } + + /// Steals the instance's environment back from `isolate`, + /// assuming `isolate` still has it in a slot. + fn take_from_isolate(&mut self, isolate: &mut Isolate) { + self.instance = isolate.remove_slot(); + } } -fn env_on_instance(inst: &mut JsInstance) -> &mut JsInstanceEnv { - inst.instance.as_mut().expect(EXPECT_ENV) +/// Access the `JsInstanceEnv` temporarily bound to an [`Isolate`]. +/// +/// This assumes that the slot has been set in the isolate already. +fn env_on_isolate(isolate: &mut Isolate) -> &mut JsInstanceEnv { + isolate.get_slot_mut().expect(JsInstanceEnvSlot::EXPECT_ENV) } +/// The environment of a [`JsInstance`]. struct JsInstanceEnv { instance_env: InstanceEnv, @@ -221,6 +258,13 @@ impl JsInstanceEnv { &self.reducer_name } + /// Returns the name of the most recent reducer to be run in this environment, + /// or `None` if no reducer is actively being invoked. + fn log_record_function(&self) -> Option<&str> { + let function = self.reducer_name(); + (!function.is_empty()).then_some(function) + } + /// Returns the name of the most recent reducer to be run in this environment. pub fn reducer_start(&self) -> Instant { self.reducer_start @@ -240,11 +284,26 @@ impl JsInstanceEnv { wasm_instance_env_call_times, } } + + /// Returns the [`ReplicaContext`] for this environment. + fn replica_ctx(&self) -> &Arc { + &self.instance_env.replica_ctx + } } struct JsInstance { + /// Information common to instances of all runtimes. + /// + /// (The type is shared, the data is not.) common: InstanceCommon, - instance: Option, + + /// The environment of the instance. + instance: JsInstanceEnvSlot, + + /// The module's program (JS code). + /// Used to startup the [`Isolate`]s. + /// + // TODO(v8): replace with snapshots. program: Arc, } @@ -259,43 +318,53 @@ impl ModuleInstance for JsInstance { old_module_info: Arc, policy: MigrationPolicy, ) -> anyhow::Result { - let replica_ctx = &env_on_instance(self).instance_env.replica_ctx.clone(); + let replica_ctx = self.instance.get_mut().replica_ctx(); self.common .update_database(replica_ctx, program, old_module_info, policy) } fn call_reducer(&mut self, tx: Option, params: CallReducerParams) -> super::ReducerCallResult { - let replica_ctx = env_on_instance(self).instance_env.replica_ctx.clone(); + let replica_ctx = &self.instance.get_mut().replica_ctx().clone(); self.common - .call_reducer_with_tx(&replica_ctx, tx, params, log_traceback, |tx, op, budget| { - let callback_every = EPOCH_TICKS_PER_SECOND; - extern "C" fn callback(isolate: &mut Isolate, _: *mut c_void) { + .call_reducer_with_tx(replica_ctx, tx, params, log_traceback, |tx, op, budget| { + /// Called by a thread separate to V8 execution + /// every [`EPOCH_TICKS_PER_SECOND`] ticks (~every 1 second) + /// to log that the reducer is still running. + extern "C" fn cb_log_long_running(isolate: &mut Isolate, _: *mut c_void) { let env = env_on_isolate(isolate); let database = env.instance_env.replica_ctx.database_identity; let reducer = env.reducer_name(); let dur = env.reducer_start().elapsed(); - tracing::warn!(reducer, ?database, "Wasm has been running for {dur:?}"); + tracing::warn!(reducer, ?database, "JavaScript has been running for {dur:?}"); } - // Prepare the isolate with the env. + // Start timer and prepare the isolate with the env. let mut isolate = Isolate::new(<_>::default()); - isolate.set_slot(self.instance.take().expect(EXPECT_ENV)); + self.instance.get_mut().instance_env.start_reducer(op.timestamp); + self.instance.move_to_isolate(&mut isolate); - // TODO(v8): snapshots, module->host calls + // TODO(v8): snapshots // Call the reducer. - env_on_isolate(&mut isolate).instance_env.start_reducer(op.timestamp); - let (mut isolate, (tx, call_result)) = - with_script(isolate, &self.program, callback_every, callback, budget, |scope, _| { + let (mut isolate, (tx, call_result)) = with_script( + isolate, + &self.program, + EPOCH_TICKS_PER_SECOND, + cb_log_long_running, + budget, + |scope, _| { let (tx, call_result) = env_on_isolate(scope) .instance_env .tx .clone() .set(tx, || call_call_reducer_from_op(scope, op)); (tx, call_result) - }); - let timings = env_on_isolate(&mut isolate).finish_reducer(); - self.instance = isolate.remove_slot(); + }, + ); + + // Steal back the env and finish timings. + self.instance.take_from_isolate(&mut isolate); + let timings = self.instance.get_mut().finish_reducer(); // Derive energy stats. let used = duration_to_budget(timings.total_duration); @@ -321,7 +390,7 @@ fn with_script( isolate: OwnedIsolate, code: &str, callback_every: u64, - callback: IsolateCallback, + callback: InterruptCallback, budget: ReducerBudget, logic: impl for<'scope> FnOnce(&mut HandleScope<'scope>, Local<'scope, Value>) -> R, ) -> (OwnedIsolate, R) { @@ -329,7 +398,7 @@ fn with_script( let code = v8::String::new(scope, code).unwrap(); let script_val = v8::Script::compile(scope, code, None).unwrap().run(scope).unwrap(); - register_host_funs(scope); + register_host_funs(scope).unwrap(); logic(scope, script_val) }) @@ -339,7 +408,7 @@ fn with_script( pub(crate) fn with_scope( mut isolate: OwnedIsolate, callback_every: u64, - callback: IsolateCallback, + callback: InterruptCallback, budget: ReducerBudget, logic: impl FnOnce(&mut HandleScope<'_>) -> R, ) -> (OwnedIsolate, R) { @@ -361,7 +430,8 @@ pub(crate) fn with_scope( (isolate, ret) } -type IsolateCallback = extern "C" fn(&mut Isolate, *mut c_void); +/// A callback passed to [`IsolateHandle::request_interrupt`]. +type InterruptCallback = extern "C" fn(&mut Isolate, *mut c_void); /// Spawns a thread that will terminate reducer execution /// when `budget` has been used up. @@ -369,12 +439,14 @@ type IsolateCallback = extern "C" fn(&mut Isolate, *mut c_void); /// Every `callback_every` ticks, `callback` is called. fn run_reducer_timeout( callback_every: u64, - callback: IsolateCallback, + callback: InterruptCallback, budget: ReducerBudget, isolate_handle: IsolateHandle, ) -> Arc { + // When `execution_done_flag` is set, the ticker thread will stop. let execution_done_flag = Arc::new(AtomicBool::new(false)); let execution_done_flag2 = execution_done_flag.clone(); + let timeout = budget_to_duration(budget); let max_ticks = ticks_in_duration(timeout); @@ -418,25 +490,24 @@ fn duration_to_budget(_duration: Duration) -> ReducerBudget { ReducerBudget::ZERO } +/// Returns the global object. fn global<'scope>(scope: &mut HandleScope<'scope>) -> Local<'scope, Object> { scope.get_current_context().global(scope) } /// Returns the global property `key`. -fn get_global_property<'scope>( - scope: &mut HandleScope<'scope>, - key: Local<'scope, v8::String>, -) -> ExcResult> { +fn get_global_property<'scope>(scope: &mut HandleScope<'scope>, key: Local<'scope, v8::String>) -> FnRet<'scope> { global(scope) .get(scope, key.into()) .ok_or_else(exception_already_thrown) } +/// Calls free function `fun` with `args`. fn call_free_fun<'scope>( scope: &mut HandleScope<'scope>, fun: Local<'scope, Function>, args: &[Local<'scope, Value>], -) -> ExcResult> { +) -> FnRet<'scope> { let receiver = v8::undefined(scope).into(); fun.call(scope, receiver, args).ok_or_else(exception_already_thrown) } @@ -530,340 +601,6 @@ fn call_describe_module(scope: &mut HandleScope<'_>) -> anyhow::Result(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let name: &str = deserialize_js(scope, args.get(0))?; - let id = env_on_isolate(scope).instance_env.table_id_from_name(name).unwrap(); - let ret = serialize_to_js(scope, &id)?; - Ok(ret) -} - -fn index_id_from_name<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let name: &str = deserialize_js(scope, args.get(0))?; - let id = env_on_isolate(scope).instance_env.index_id_from_name(name).unwrap(); - let ret = serialize_to_js(scope, &id)?; - Ok(ret) -} - -fn datastore_table_row_count<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let table_id: TableId = deserialize_js(scope, args.get(0))?; - let count = env_on_isolate(scope) - .instance_env - .datastore_table_row_count(table_id) - .unwrap(); - serialize_to_js(scope, &count) -} - -fn datastore_table_scan_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let table_id: TableId = deserialize_js(scope, args.get(0))?; - - let env = env_on_isolate(scope); - // Collect the iterator chunks. - let chunks = env - .instance_env - .datastore_table_scan_bsatn_chunks(&mut env.chunk_pool, table_id) - .unwrap(); - - // Register the iterator and get back the index to write to `out`. - // Calls to the iterator are done through dynamic dispatch. - let idx = env.iters.insert(chunks.into_iter()); - - let ret = serialize_to_js(scope, &idx.0)?; - Ok(ret) -} - -fn convert_u32_to_col_id(col_id: u32) -> anyhow::Result { - let col_id: u16 = col_id.try_into().context("ABI violation, a `ColId` must be a `u16`")?; - Ok(col_id.into()) -} - -fn datastore_index_scan_range_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let index_id: IndexId = deserialize_js(scope, args.get(0))?; - - let prefix_elems: u32 = deserialize_js(scope, args.get(2))?; - let prefix_elems = convert_u32_to_col_id(prefix_elems).unwrap(); - - let prefix: &[u8] = if prefix_elems.idx() == 0 { - &[] - } else { - deserialize_js(scope, args.get(1))? - }; - - let rstart: &[u8] = deserialize_js(scope, args.get(3))?; - let rend: &[u8] = deserialize_js(scope, args.get(4))?; - - let env = env_on_isolate(scope); - - // Find the relevant rows. - let chunks = env - .instance_env - .datastore_index_scan_range_bsatn_chunks(&mut env.chunk_pool, index_id, prefix, prefix_elems, rstart, rend) - .unwrap(); - - // Insert the encoded + concatenated rows into a new buffer and return its id. - let idx = env.iters.insert(chunks.into_iter()); - - let ret = serialize_to_js(scope, &idx.0)?; - Ok(ret) -} - -fn row_iter_bsatn_advance<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; - let row_iter_idx = RowIterIdx(row_iter_idx); - let buffer_max_len: u32 = deserialize_js(scope, args.get(1))?; - - // Retrieve the iterator by `row_iter_idx`, or error. - let env = env_on_isolate(scope); - let iter = env.iters.get_mut(row_iter_idx).unwrap(); - - // Allocate a buffer with `buffer_max_len` capacity. - let mut buffer = vec![0; buffer_max_len as usize]; - // Fill the buffer as much as possible. - let written = InstanceEnv::fill_buffer_from_iter(iter, &mut buffer, &mut env.chunk_pool); - buffer.truncate(written); - - let ret = match (written, iter.as_slice().first()) { - // Nothing was written and the iterator is not exhausted. - (0, Some(_chunk)) => { - unimplemented!() - } - // The iterator is exhausted, destroy it, and tell the caller. - (_, None) => { - env.iters.take(row_iter_idx); - serialize_to_js(scope, &AdvanceRet { flag: -1, buffer })? - } - // Something was written, but the iterator is not exhausted. - (_, Some(_)) => serialize_to_js(scope, &AdvanceRet { flag: 0, buffer })?, - }; - Ok(ret) -} - -#[derive(Serialize)] -struct AdvanceRet { - buffer: Vec, - flag: i32, -} - -fn row_iter_bsatn_close<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; - let row_iter_idx = RowIterIdx(row_iter_idx); - - // Retrieve the iterator by `row_iter_idx`, or error. - let env = env_on_isolate(scope); - - // Retrieve the iterator by `row_iter_idx`, or error. - Ok(match env.iters.take(row_iter_idx) { - None => unimplemented!(), - // TODO(Centril): consider putting these into a pool for reuse. - Some(_) => serialize_to_js(scope, &0u32)?, - }) -} - -fn datastore_insert_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let table_id: TableId = deserialize_js(scope, args.get(0))?; - let mut row: Vec = deserialize_js(scope, args.get(1))?; - - // Insert the row into the DB and write back the generated column values. - let env: &mut JsInstanceEnv = env_on_isolate(scope); - let row_len = env.instance_env.insert(table_id, &mut row).unwrap(); - row.truncate(row_len); - - serialize_to_js(scope, &row) -} - -fn datastore_update_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let table_id: TableId = deserialize_js(scope, args.get(0))?; - let index_id: IndexId = deserialize_js(scope, args.get(1))?; - let mut row: Vec = deserialize_js(scope, args.get(2))?; - - // Insert the row into the DB and write back the generated column values. - let env: &mut JsInstanceEnv = env_on_isolate(scope); - let row_len = env.instance_env.update(table_id, index_id, &mut row).unwrap(); - row.truncate(row_len); - - serialize_to_js(scope, &row) -} - -fn datastore_delete_by_index_scan_range_bsatn<'s>( - scope: &mut HandleScope<'s>, - args: FunctionCallbackArguments<'s>, -) -> FnRet<'s> { - let index_id: IndexId = deserialize_js(scope, args.get(0))?; - - let prefix_elems: u32 = deserialize_js(scope, args.get(2))?; - let prefix_elems = convert_u32_to_col_id(prefix_elems).unwrap(); - - let prefix: &[u8] = if prefix_elems.idx() == 0 { - &[] - } else { - deserialize_js(scope, args.get(1))? - }; - - let rstart: &[u8] = deserialize_js(scope, args.get(3))?; - let rend: &[u8] = deserialize_js(scope, args.get(4))?; - - let env = env_on_isolate(scope); - - // Delete the relevant rows. - let num = env - .instance_env - .datastore_delete_by_index_scan_range_bsatn(index_id, prefix, prefix_elems, rstart, rend) - .unwrap(); - - serialize_to_js(scope, &num) -} - -fn datastore_delete_all_by_eq_bsatn<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let table_id: TableId = deserialize_js(scope, args.get(0))?; - let relation: &[u8] = deserialize_js(scope, args.get(1))?; - - let env = env_on_isolate(scope); - let num = env - .instance_env - .datastore_delete_all_by_eq_bsatn(table_id, relation) - .unwrap(); - - serialize_to_js(scope, &num) -} - -fn volatile_nonatomic_schedule_immediate<'s>( - scope: &mut HandleScope<'s>, - args: FunctionCallbackArguments<'s>, -) -> FnRet<'s> { - let name: String = deserialize_js(scope, args.get(0))?; - let args: Vec = deserialize_js(scope, args.get(1))?; - - let env = env_on_isolate(scope); - env.instance_env - .scheduler - .volatile_nonatomic_schedule_immediate(name, crate::host::ReducerArgs::Bsatn(args.into())); - - Ok(v8::undefined(scope).into()) -} - -fn console_log<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let level: u32 = deserialize_js(scope, args.get(0))?; - - let msg = args.get(1).cast::(); - let mut buf = scratch_buf::<128>(); - let msg = msg.to_rust_cow_lossy(scope, &mut buf); - let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) - .ok_or_else(exception_already_thrown)? - .get_frame(scope, 1) - .ok_or_else(exception_already_thrown)?; - let mut buf = scratch_buf::<32>(); - let filename = frame - .get_script_name(scope) - .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); - let record = Record { - // TODO: figure out whether to use walltime now or logical reducer now (env.reducer_start) - ts: chrono::Utc::now(), - target: None, - filename: filename.as_deref(), - line_number: Some(frame.get_line_number() as u32), - message: &msg, - }; - - let env = env_on_isolate(scope); - env.instance_env.console_log((level as u8).into(), &record, &Noop); - - Ok(v8::undefined(scope).into()) -} - -struct Noop; -impl BacktraceProvider for Noop { - fn capture(&self) -> Box { - Box::new(Noop) - } -} -impl ModuleBacktrace for Noop { - fn frames(&self) -> Vec> { - Vec::new() - } -} - -fn console_timer_start<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let name = args.get(0).cast::(); - let mut buf = scratch_buf::<128>(); - let name = name.to_rust_cow_lossy(scope, &mut buf).into_owned(); - - let env = env_on_isolate(scope); - let span_id = env.timing_spans.insert(TimingSpan::new(name)).0; - serialize_to_js(scope, &span_id) -} - -fn console_timer_end<'s>(scope: &mut HandleScope<'s>, args: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let span_id: u32 = deserialize_js(scope, args.get(0))?; - - let env = env_on_isolate(scope); - let span = env.timing_spans.take(TimingSpanIdx(span_id)).unwrap(); - env.instance_env.console_timer_end(&span, &Noop); - - serialize_to_js(scope, &0u32) -} - -fn identity<'s>(scope: &mut HandleScope<'s>, _: FunctionCallbackArguments<'s>) -> FnRet<'s> { - let env = env_on_isolate(scope); - let identity = *env.instance_env.database_identity(); - serialize_to_js(scope, &identity) -} - -fn register_host_funs(scope: &mut HandleScope<'_>) { - register_host_fun(scope, "table_id_from_name", table_id_from_name); - register_host_fun(scope, "index_id_from_name", index_id_from_name); - register_host_fun(scope, "datastore_table_row_count", datastore_table_row_count); - register_host_fun(scope, "datastore_table_scan_bsatn", datastore_table_scan_bsatn); - register_host_fun( - scope, - "datastore_index_scan_range_bsatn", - datastore_index_scan_range_bsatn, - ); - register_host_fun(scope, "row_iter_bsatn_advance", row_iter_bsatn_advance); - register_host_fun(scope, "row_iter_bsatn_close", row_iter_bsatn_close); - register_host_fun(scope, "datastore_insert_bsatn", datastore_insert_bsatn); - register_host_fun(scope, "datastore_update_bsatn", datastore_update_bsatn); - register_host_fun( - scope, - "datastore_delete_by_index_scan_range_bsatn", - datastore_delete_by_index_scan_range_bsatn, - ); - register_host_fun( - scope, - "datastore_delete_all_by_eq_bsatn", - datastore_delete_all_by_eq_bsatn, - ); - register_host_fun( - scope, - "volatile_nonatomic_schedule_immediate", - volatile_nonatomic_schedule_immediate, - ); - register_host_fun(scope, "console_log", console_log); - register_host_fun(scope, "console_timer_start", console_timer_start); - register_host_fun(scope, "console_timer_end", console_timer_end); - register_host_fun(scope, "identity", identity); -} - -type FnRet<'s> = ExcResult>; - -fn register_host_fun( - scope: &mut HandleScope<'_>, - name: &str, - fun: impl Copy + for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, -) { - let name = v8_interned_string(scope, name).into(); - let fun = Function::new(scope, adapt_fun(fun)).unwrap().into(); - global(scope).set(scope, name, fun).unwrap(); -} - -fn adapt_fun( - fun: impl Copy + for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>) -> FnRet<'s>, -) -> impl Copy + for<'s> Fn(&mut HandleScope<'s>, FunctionCallbackArguments<'s>, ReturnValue) { - move |scope, args, mut rv| { - if let Ok(value) = fun(scope, args) { - rv.set(value); - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/crates/core/src/host/v8/ser.rs b/crates/core/src/host/v8/ser.rs index 2965df81aa1..c9ed1887f68 100644 --- a/crates/core/src/host/v8/ser.rs +++ b/crates/core/src/host/v8/ser.rs @@ -3,6 +3,7 @@ use super::de::intern_field_name; use super::error::{exception_already_thrown, ExcResult, ExceptionThrown, RangeError, Throwable, TypeError}; use super::key_cache::{get_or_create_key_cache, KeyCache}; +use super::syscall::FnRet; use super::to_value::ToValue; use derive_more::From; use spacetimedb_sats::{ @@ -13,10 +14,7 @@ use spacetimedb_sats::{ use v8::{Array, ArrayBuffer, HandleScope, IntegrityLevel, Local, Object, Uint8Array, Value}; /// Serializes `value` into a V8 into `scope`. -pub(super) fn serialize_to_js<'scope>( - scope: &mut HandleScope<'scope>, - value: &impl Serialize, -) -> ExcResult> { +pub(super) fn serialize_to_js<'scope>(scope: &mut HandleScope<'scope>, value: &impl Serialize) -> FnRet<'scope> { let key_cache = get_or_create_key_cache(scope); let key_cache = &mut *key_cache.borrow_mut(); value diff --git a/crates/core/src/host/v8/syscall.rs b/crates/core/src/host/v8/syscall.rs new file mode 100644 index 00000000000..d03c765a837 --- /dev/null +++ b/crates/core/src/host/v8/syscall.rs @@ -0,0 +1,1061 @@ +use super::de::{deserialize_js, scratch_buf, v8_interned_string}; +use super::error::ExcResult; +use super::ser::serialize_to_js; +use super::{env_on_isolate, exception_already_thrown}; +use crate::database_logger::{LogLevel, Record}; +use crate::error::NodesError; +use crate::host::instance_env::InstanceEnv; +use crate::host::v8::error::ExceptionThrown; +use crate::host::v8::{global, BufferTooSmall, CodeError, JsInstanceEnv, JsStackTrace, TerminationError, Throwable}; +use crate::host::wasm_common::instrumentation::span; +use crate::host::wasm_common::{err_to_errno_and_log, RowIterIdx, TimingSpan, TimingSpanIdx}; +use crate::host::AbiCall; +use spacetimedb_lib::Identity; +use spacetimedb_primitives::{errno, ColId, IndexId, TableId}; +use spacetimedb_sats::Serialize; +use v8::{Function, FunctionCallbackArguments, HandleScope, Local, Value}; + +/// Registers all module -> host syscalls. +pub(super) fn register_host_funs(scope: &mut HandleScope<'_>) -> ExcResult<()> { + macro_rules! register { + ($wrapper:ident, $abi_call:expr, $fun:ident) => { + register_host_fun(scope, stringify!($fun), |s, a| $wrapper($abi_call, s, a, $fun))?; + }; + } + + register!(with_sys_result, AbiCall::TableIdFromName, table_id_from_name); + register!(with_sys_result, AbiCall::IndexIdFromName, index_id_from_name); + register!( + with_sys_result, + AbiCall::DatastoreTableRowCount, + datastore_table_row_count + ); + register!( + with_sys_result, + AbiCall::DatastoreTableScanBsatn, + datastore_table_scan_bsatn + ); + register!( + with_sys_result, + AbiCall::DatastoreIndexScanRangeBsatn, + datastore_index_scan_range_bsatn + ); + register!(with_sys_result, AbiCall::RowIterBsatnAdvance, row_iter_bsatn_advance); + register!(with_span, AbiCall::RowIterBsatnClose, row_iter_bsatn_close); + register!(with_sys_result, AbiCall::DatastoreInsertBsatn, datastore_insert_bsatn); + register!(with_sys_result, AbiCall::DatastoreUpdateBsatn, datastore_update_bsatn); + register!( + with_sys_result, + AbiCall::DatastoreDeleteByIndexScanRangeBsatn, + datastore_delete_by_index_scan_range_bsatn + ); + register!( + with_sys_result, + AbiCall::DatastoreDeleteAllByEqBsatn, + datastore_delete_all_by_eq_bsatn + ); + register!( + with_span, + AbiCall::VolatileNonatomicScheduleImmediate, + volatile_nonatomic_schedule_immediate + ); + register!(with_span, AbiCall::ConsoleLog, console_log); + register!(with_span, AbiCall::ConsoleTimerStart, console_timer_start); + register!(with_span, AbiCall::ConsoleTimerEnd, console_timer_end); + register!(with_sys_result, AbiCall::Identity, identity); + Ok(()) +} + +/// The return type of a module -> host syscall. +pub(super) type FnRet<'scope> = ExcResult>; + +/// Registers a module -> host syscall in `scope` +/// where the function has `name` and `body` +fn register_host_fun( + scope: &mut HandleScope<'_>, + name: &str, + body: impl Copy + for<'scope> Fn(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, +) -> ExcResult<()> { + let name = v8_interned_string(scope, name).into(); + let fun = Function::new(scope, adapt_fun(body)) + .ok_or_else(exception_already_thrown)? + .into(); + global(scope) + .set(scope, name, fun) + .ok_or_else(exception_already_thrown)?; + Ok(()) +} + +/// A flag set in [`handle_nodes_error`]. +/// The flag should be checked in every module -> host ABI. +/// If the flag is set, the call is prevented. +struct TerminationFlag; + +/// Adapts `fun`, which returns a [`Value`] to one that works on [`v8::ReturnValue`]. +fn adapt_fun( + fun: impl Copy + for<'scope> Fn(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, +) -> impl Copy + for<'scope> Fn(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>, v8::ReturnValue) { + move |scope, args, mut rv| { + // If the flag was set in `handle_nodes_error`, + // we need to block all module -> host ABI calls. + if scope.get_slot::().is_some() { + let err = anyhow::anyhow!("execution is being terminated"); + if let Ok(exception) = TerminationError::from_error(scope, &err) { + exception.throw(scope); + } + return; + } + + // Set the result `value` on success. + if let Ok(value) = fun(scope, args) { + rv.set(value); + } + } +} + +/// Either an exception, already thrown, or [`NodesError`] arising from [`InstanceEnv`]. +#[derive(derive_more::From)] +enum SysCallError { + Error(NodesError), + Exception(ExceptionThrown), +} + +type SysCallResult = Result; + +/// Wraps `run` in [`with_span`] and returns the return value of `run` to JS. +/// Handles [`SysCallError`] if it occurs by throwing exceptions into JS. +fn with_sys_result<'scope, O: Serialize>( + abi_call: AbiCall, + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, + run: impl FnOnce(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> SysCallResult, +) -> FnRet<'scope> { + match with_span(abi_call, scope, args, run) { + Ok(ret) => serialize_to_js(scope, &ret), + Err(SysCallError::Exception(exc)) => Err(exc), + Err(SysCallError::Error(error)) => Err(throw_nodes_error(abi_call, scope, error)), + } +} + +/// Turns a [`NodesError`] into a thrown exception. +fn throw_nodes_error(abi_call: AbiCall, scope: &mut HandleScope<'_>, error: NodesError) -> ExceptionThrown { + let res = match err_to_errno_and_log::(abi_call, error) { + Ok(code) => CodeError::from_code(scope, code), + Err(err) => { + // Terminate execution ASAP and throw a catchable exception (`TerminationError`). + // Unfortunately, JS execution won't be terminated once the callback returns, + // so we set a slot that all callbacks immediately check + // to ensure that the module won't be able to do anything to the host + // while it's being terminated (eventually). + scope.terminate_execution(); + scope.set_slot(TerminationFlag); + TerminationError::from_error(scope, &err) + } + }; + collapse_exc_thrown(scope, res) +} + +/// Collapses `res` where the `Ok(x)` where `x` is throwable. +fn collapse_exc_thrown<'scope>( + scope: &mut HandleScope<'scope>, + res: ExcResult>, +) -> ExceptionThrown { + let (Ok(thrown) | Err(thrown)) = res.map(|ev| ev.throw(scope)); + thrown +} + +/// Tracks the span of `body` under the label `abi_call`. +fn with_span<'scope, R>( + abi_call: AbiCall, + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, + body: impl FnOnce(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> R, +) -> R { + // Start the span. + let span_start = span::CallSpanStart::new(abi_call); + + // Call `fun` with `args` in `scope`. + let result = body(scope, args); + + // Track the span of this call. + let span = span_start.end(); + span::record_span(&mut env_on_isolate(scope).call_times, span); + + result +} + +/// Module ABI that finds the `TableId` for a table name. +/// +/// # Signature +/// +/// ``` +/// table_id_from_name(name: string) -> u32 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_TABLE +/// } +/// ``` +/// +/// # Types +/// +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns an `u32` containing the id of the table. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_TABLE`] +/// when `name` is not the name of a table. +/// +/// Throws a `TypeError` if: +/// - `name` is not `string`. +fn table_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { + let name: &str = deserialize_js(scope, args.get(0))?; + Ok(env_on_isolate(scope).instance_env.table_id_from_name(name)?) +} + +/// Module ABI that finds the `IndexId` for an index name. +/// +/// # Signature +/// +/// ``` +/// index_id_from_name(name: string) -> u32 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_INDEX +/// } +/// ``` +/// +/// # Types +/// +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns an `u32` containing the id of the index. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_INDEX`] +/// when `name` is not the name of an index. +/// +/// Throws a `TypeError`: +/// - if `name` is not `string`. +fn index_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { + let name: &str = deserialize_js(scope, args.get(0))?; + Ok(env_on_isolate(scope).instance_env.index_id_from_name(name)?) +} + +/// Module ABI that returns the number of rows currently in table identified by `table_id`. +/// +/// # Signature +/// +/// ``` +/// datastore_table_row_count(table_id: u32) -> u64 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_TABLE +/// } +/// ``` +/// +/// # Types +/// +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// - `u64` is `bigint` in JS restricted to unsigned 64-bit integers. +/// +/// # Returns +/// +/// Returns a `u64` containing the number of rows in the table. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_TABLE`] +/// when `table_id` is not a known ID of a table. +/// +/// Throws a `TypeError` if: +/// - `table_id` is not a `u32`. +fn datastore_table_row_count(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + Ok(env_on_isolate(scope).instance_env.datastore_table_row_count(table_id)?) +} + +/// Module ABI that starts iteration on each row, as BSATN-encoded, +/// of a table identified by `table_id`. +/// +/// # Signature +/// +/// ``` +/// datastore_table_scan_bsatn(table_id: u32) -> u32 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_TABLE +/// } +/// ``` +/// +/// # Types +/// +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// - `u64` is `bigint` in JS restricted to unsigned 64-bit integers. +/// +/// # Returns +/// +/// Returns a `u32` that is the iterator handle. +/// This handle can be advanced by [`row_iter_bsatn_advance`]. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_TABLE`] +/// when `table_id` is not a known ID of a table. +/// +/// Throws a `TypeError`: +/// - if `table_id` is not a `u32`. +fn datastore_table_scan_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + + let env = env_on_isolate(scope); + // Collect the iterator chunks. + let chunks = env + .instance_env + .datastore_table_scan_bsatn_chunks(&mut env.chunk_pool, table_id)?; + + // Register the iterator and get back the index to write to `out`. + // Calls to the iterator are done through dynamic dispatch. + Ok(env.iters.insert(chunks.into_iter()).0) +} + +/// Module ABI that finds all rows in the index identified by `index_id`, +/// according to `prefix`, `rstart`, and `rend`. +/// +/// The index itself has a schema/type. +/// The `prefix` is decoded to the initial `prefix_elems` `AlgebraicType`s +/// whereas `rstart` and `rend` are decoded to the `prefix_elems + 1` `AlgebraicType` +/// where the `AlgebraicValue`s are wrapped in `Bound`. +/// That is, `rstart, rend` are BSATN-encoded `Bound`s. +/// +/// Matching is then defined by equating `prefix` +/// to the initial `prefix_elems` columns of the index +/// and then imposing `rstart` as the starting bound +/// and `rend` as the ending bound on the `prefix_elems + 1` column of the index. +/// Remaining columns of the index are then unbounded. +/// Note that the `prefix` in this case can be empty (`prefix_elems = 0`), +/// in which case this becomes a ranged index scan on a single-col index +/// or even a full table scan if `rstart` and `rend` are both unbounded. +/// +/// The relevant table for the index is found implicitly via the `index_id`, +/// which is unique for the module. +/// +/// On success, the iterator handle is written to the `out` pointer. +/// This handle can be advanced by [`row_iter_bsatn_advance`]. +/// +/// # Non-obvious queries +/// +/// For an index on columns `[a, b, c]`: +/// +/// - `a = x, b = y` is encoded as a prefix `[x, y]` +/// and a range `Range::Unbounded`, +/// or as a prefix `[x]` and a range `rstart = rend = Range::Inclusive(y)`. +/// - `a = x, b = y, c = z` is encoded as a prefix `[x, y]` +/// and a range `rstart = rend = Range::Inclusive(z)`. +/// - A sorted full scan is encoded as an empty prefix +/// and a range `Range::Unbounded`. +/// +/// # Signature +/// +/// ``` +/// datastore_index_scan_range_bsatn( +/// index_id: u32, +/// prefix: u8[], +/// prefix_elems: u16, +/// rstart: u8[], +/// rend: u8[], +/// ) -> u32 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_INDEX | BSATN_DECODE_ERROR +/// } +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// - `u64` is `bigint` in JS restricted to unsigned 64-bit integers. +/// +/// # Returns +/// +/// Returns a `u32` that is the iterator handle. +/// This handle can be advanced by [`row_iter_bsatn_advance`]. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_INDEX`] +/// when `index_id` is not a known ID of an index. +/// +/// - [`spacetimedb_primitives::errno::BSATN_DECODE_ERROR`] +/// when `prefix` cannot be decoded to +/// a `prefix_elems` number of `AlgebraicValue` +/// typed at the initial `prefix_elems` `AlgebraicType`s of the index's key type. +/// Or when `rstart` or `rend` cannot be decoded to an `Bound` +/// where the inner `AlgebraicValue`s are +/// typed at the `prefix_elems + 1` `AlgebraicType` of the index's key type. +/// +/// Throws a `TypeError` if: +/// - `table_id` is not a `u32`. +/// - `prefix`, `rstart`, and `rend` are not arrays of `u8`s. +/// - `prefix_elems` is not a `u16`. +fn datastore_index_scan_range_bsatn( + scope: &mut HandleScope<'_>, + args: FunctionCallbackArguments<'_>, +) -> SysCallResult { + let index_id: IndexId = deserialize_js(scope, args.get(0))?; + let mut prefix: &[u8] = deserialize_js(scope, args.get(1))?; + let prefix_elems: ColId = deserialize_js(scope, args.get(2))?; + let rstart: &[u8] = deserialize_js(scope, args.get(3))?; + let rend: &[u8] = deserialize_js(scope, args.get(4))?; + + if prefix_elems.idx() == 0 { + prefix = &[]; + } + + let env = env_on_isolate(scope); + + // Find the relevant rows. + let chunks = env.instance_env.datastore_index_scan_range_bsatn_chunks( + &mut env.chunk_pool, + index_id, + prefix, + prefix_elems, + rstart, + rend, + )?; + + // Insert the encoded + concatenated rows into a new buffer and return its id. + Ok(env.iters.insert(chunks.into_iter()).0) +} + +/// Throws `{ __code_error__: NO_SUCH_ITER }`. +fn no_such_iter(scope: &mut HandleScope<'_>) -> ExceptionThrown { + let res = CodeError::from_code(scope, errno::NO_SUCH_ITER.get()); + collapse_exc_thrown(scope, res) +} + +/// Module ABI that reads rows from the given iterator registered under `iter`. +/// +/// Takes rows from the iterator with id `iter` +/// and returns them encoded in the BSATN format. +/// +/// The rows returned take up at most `buffer_max_len` bytes. +/// A row is never broken up between calls. +/// +/// Aside from the BSATN, +/// the function also returns `true` when the iterator been exhausted +/// and there are no more rows to read. +/// This leads to the iterator being immediately destroyed. +/// Conversely, `false` is returned if there are more rows to read. +/// Note that the host is free to reuse allocations in a pool, +/// destroying the handle logically does not entail that memory is necessarily reclaimed. +/// +/// # Signature +/// +/// ``` +/// row_iter_bsatn_advance(iter: u32, buffer_max_len: u32) -> (boolean, u8[]) throws +/// { __code_error__: NO_SUCH_ITER } | { __buffer_too_small__: number } +/// ``` +/// +/// # Types +/// +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns `(exhausted: boolean, rows_bsatn: u8[])` where: +/// - `exhausted` is `true` if there are no more rows to read, +/// - `rows_bsatn` are the BSATN-encoded row bytes, concatenated. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_ITER`] +/// when `iter` is not a valid iterator. +/// +/// Throws `{ __buffer_too_small__: number }` +/// when there are rows left but they cannot fit in `buffer`. +/// When this occurs, `__buffer_too_small__` contains the size of the next item in the iterator. +/// To make progress, the caller should call `row_iter_bsatn_advance` +/// with `buffer_max_len >= __buffer_too_small__` and try again. +/// +/// Throws a `TypeError` if: +/// - `iter` and `buffer_max_len` are not `u32`s. +fn row_iter_bsatn_advance<'scope>( + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, +) -> SysCallResult<(bool, Vec)> { + let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; + let row_iter_idx = RowIterIdx(row_iter_idx); + let buffer_max_len: u32 = deserialize_js(scope, args.get(1))?; + + // Retrieve the iterator by `row_iter_idx`, or error. + let env = env_on_isolate(scope); + let Some(iter) = env.iters.get_mut(row_iter_idx) else { + return Err(no_such_iter(scope).into()); + }; + + // Allocate a buffer with `buffer_max_len` capacity. + let mut buffer = vec![0; buffer_max_len as usize]; + // Fill the buffer as much as possible. + let written = InstanceEnv::fill_buffer_from_iter(iter, &mut buffer, &mut env.chunk_pool); + buffer.truncate(written); + + match (written, iter.as_slice().first().map(|c| c.len().try_into().unwrap())) { + // Nothing was written and the iterator is not exhausted. + (0, Some(min_len)) => { + let exc = BufferTooSmall::from_requirement(scope, min_len)?; + Err(exc.throw(scope).into()) + } + // The iterator is exhausted, destroy it, and tell the caller. + (_, None) => { + env.iters.take(row_iter_idx); + Ok((true, buffer)) + } + // Something was written, but the iterator is not exhausted. + (_, Some(_)) => Ok((false, buffer)), + } +} + +/// Module ABI that destroys the iterator registered under `iter`. +/// +/// Once `row_iter_bsatn_close` is called on `iter`, the `iter` is invalid. +/// That is, `row_iter_bsatn_close(iter)` the second time will yield `NO_SUCH_ITER`. +/// +/// # Signature +/// +/// ``` +/// row_iter_bsatn_close(iter: u32) -> undefined throws { +/// __code_error__: NO_SUCH_ITER +/// } +/// ``` +/// +/// # Types +/// +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns nothing. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_ITER`] +/// when `iter` is not a valid iterator. +/// +/// Throws a `TypeError` if: +/// - `iter` is not a `u32`. +fn row_iter_bsatn_close<'scope>( + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, +) -> FnRet<'scope> { + let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; + let row_iter_idx = RowIterIdx(row_iter_idx); + + // Retrieve the iterator by `row_iter_idx`, or error. + let env = env_on_isolate(scope); + + // Retrieve the iterator by `row_iter_idx`, or error. + if env.iters.take(row_iter_idx).is_none() { + return Err(no_such_iter(scope)); + } else { + // TODO(Centril): consider putting these into a pool for reuse. + } + + Ok(v8::undefined(scope).into()) +} + +/// Module ABI that inserts a row into the table identified by `table_id`, +/// where the `row` is an array of bytes. +/// +/// The byte array `row` must be a BSATN-encoded `ProductValue` +/// typed at the table's `ProductType` row-schema. +/// +/// To handle auto-incrementing columns, +/// when the call is successful, +/// the an array of bytes is returned, containing the generated sequence values. +/// These values are written as a BSATN-encoded `pv: ProductValue`. +/// Each `v: AlgebraicValue` in `pv` is typed at the sequence's column type. +/// The `v`s in `pv` are ordered by the order of the columns, in the schema of the table. +/// When the table has no sequences, +/// this implies that the `pv`, and thus `row`, will be empty. +/// +/// # Signature +/// +/// ``` +/// datastore_insert_bsatn(table_id: u32, row: u8[]) -> u8[] throws { +/// __code_error__: +/// NOT_IN_TRANSACTION +/// | NOT_SUCH_TABLE +/// | BSATN_DECODE_ERROR +/// | UNIQUE_ALREADY_EXISTS +/// | SCHEDULE_AT_DELAY_TOO_LONG +/// } +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns the generated sequence values encoded in BSATN (see above). +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// - [`spacetimedb_primitives::errno::NOT_SUCH_TABLE`] +/// when `table_id` is not a known ID of a table. +/// - [`spacetimedb_primitives::errno::`BSATN_DECODE_ERROR`] +/// when `row` cannot be decoded to a `ProductValue`. +/// typed at the `ProductType` the table's schema specifies. +/// - [`spacetimedb_primitives::errno::`UNIQUE_ALREADY_EXISTS`] +/// when inserting `row` would violate a unique constraint. +/// - [`spacetimedb_primitives::errno::`SCHEDULE_AT_DELAY_TOO_LONG`] +/// when the delay specified in the row was too long. +/// +/// Throws a `TypeError` if: +/// - `table_id` is not a `u32`. +/// - `row` is not an array of `u8`s. +fn datastore_insert_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let mut row: Vec = deserialize_js(scope, args.get(1))?; + + // Insert the row into the DB and write back the generated column values. + let env: &mut JsInstanceEnv = env_on_isolate(scope); + let row_len = env.instance_env.insert(table_id, &mut row)?; + row.truncate(row_len); + + Ok(row) +} + +/// Module ABI that updates a row into the table identified by `table_id`, +/// where the `row` is an array of bytes. +/// +/// The byte array `row` must be a BSATN-encoded `ProductValue` +/// typed at the table's `ProductType` row-schema. +/// +/// The row to update is found by projecting `row` +/// to the type of the *unique* index identified by `index_id`. +/// If no row is found, the error `NO_SUCH_ROW` is returned. +/// +/// To handle auto-incrementing columns, +/// when the call is successful, +/// the `row` is written back to with the generated sequence values. +/// These values are written as a BSATN-encoded `pv: ProductValue`. +/// Each `v: AlgebraicValue` in `pv` is typed at the sequence's column type. +/// The `v`s in `pv` are ordered by the order of the columns, in the schema of the table. +/// When the table has no sequences, +/// this implies that the `pv`, and thus `row`, will be empty. +/// +/// # Signature +/// +/// ``` +/// datastore_update_bsatn(table_id: u32, index_id: u32, row: u8[]) -> u8[] throws { +/// __code_error__: +/// NOT_IN_TRANSACTION +/// | NOT_SUCH_TABLE +/// | NO_SUCH_INDEX +/// | INDEX_NOT_UNIQUE +/// | BSATN_DECODE_ERROR +/// | NO_SUCH_ROW +/// | UNIQUE_ALREADY_EXISTS +/// | SCHEDULE_AT_DELAY_TOO_LONG +/// } +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns the generated sequence values encoded in BSATN (see above). +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// - [`spacetimedb_primitives::errno::NOT_SUCH_TABLE`] +/// when `table_id` is not a known ID of a table. +/// - [`spacetimedb_primitives::errno::NO_SUCH_INDEX`] +/// when `index_id` is not a known ID of an index. +/// - [`spacetimedb_primitives::errno::INDEX_NOT_UNIQUE`] +/// when the index was not unique. +/// - [`spacetimedb_primitives::errno::`BSATN_DECODE_ERROR`] +/// when `row` cannot be decoded to a `ProductValue`. +/// typed at the `ProductType` the table's schema specifies +/// or when it cannot be projected to the index identified by `index_id`. +/// - [`spacetimedb_primitives::errno::`NO_SUCH_ROW`] +/// when the row was not found in the unique index. +/// - [`spacetimedb_primitives::errno::`UNIQUE_ALREADY_EXISTS`] +/// when inserting `row` would violate a unique constraint. +/// - [`spacetimedb_primitives::errno::`SCHEDULE_AT_DELAY_TOO_LONG`] +/// when the delay specified in the row was too long. +/// +/// Throws a `TypeError` if: +/// - `table_id` is not a `u32`. +/// - `row` is not an array of `u8`s. +fn datastore_update_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult> { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let index_id: IndexId = deserialize_js(scope, args.get(1))?; + let mut row: Vec = deserialize_js(scope, args.get(2))?; + + // Insert the row into the DB and write back the generated column values. + let env: &mut JsInstanceEnv = env_on_isolate(scope); + let row_len = env.instance_env.update(table_id, index_id, &mut row)?; + row.truncate(row_len); + + Ok(row) +} + +/// Module ABI that deletes all rows found in the index identified by `index_id`, +/// according to `prefix`, `rstart`, and `rend`. +/// +/// This syscall will delete all the rows found by +/// [`datastore_index_scan_range_bsatn`] with the same arguments passed, +/// including `prefix_elems`. +/// See `datastore_index_scan_range_bsatn` for details. +/// +/// # Signature +/// +/// ``` +/// datastore_index_scan_range_bsatn( +/// index_id: u32, +/// prefix: u8[], +/// prefix_elems: u16, +/// rstart: u8[], +/// rend: u8[], +/// ) -> u32 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_INDEX | BSATN_DECODE_ERROR +/// } +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns a `u32` that is the number of rows deleted. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_INDEX`] +/// when `index_id` is not a known ID of an index. +/// +/// - [`spacetimedb_primitives::errno::BSATN_DECODE_ERROR`] +/// when `prefix` cannot be decoded to +/// a `prefix_elems` number of `AlgebraicValue` +/// typed at the initial `prefix_elems` `AlgebraicType`s of the index's key type. +/// Or when `rstart` or `rend` cannot be decoded to an `Bound` +/// where the inner `AlgebraicValue`s are +/// typed at the `prefix_elems + 1` `AlgebraicType` of the index's key type. +/// +/// Throws a `TypeError` if: +/// - `table_id` is not a `u32`. +/// - `prefix`, `rstart`, and `rend` are not arrays of `u8`s. +/// - `prefix_elems` is not a `u16`. +fn datastore_delete_by_index_scan_range_bsatn( + scope: &mut HandleScope<'_>, + args: FunctionCallbackArguments<'_>, +) -> SysCallResult { + let index_id: IndexId = deserialize_js(scope, args.get(0))?; + let mut prefix: &[u8] = deserialize_js(scope, args.get(1))?; + let prefix_elems: ColId = deserialize_js(scope, args.get(2))?; + let rstart: &[u8] = deserialize_js(scope, args.get(3))?; + let rend: &[u8] = deserialize_js(scope, args.get(4))?; + + if prefix_elems.idx() == 0 { + prefix = &[]; + } + + let env = env_on_isolate(scope); + + // Delete the relevant rows. + Ok(env + .instance_env + .datastore_delete_by_index_scan_range_bsatn(index_id, prefix, prefix_elems, rstart, rend)?) +} + +/// Module ABI that deletes those rows, in the table identified by `table_id`, +/// that match any row in `relation`. +/// +/// Matching is defined by first BSATN-decoding +/// the array of bytes `relation` to a `Vec` +/// according to the row schema of the table +/// and then using `Ord for AlgebraicValue`. +/// A match happens when `Ordering::Equal` is returned from `fn cmp`. +/// This occurs exactly when the row's BSATN-encoding is equal to the encoding of the `ProductValue`. +/// +/// # Signature +/// +/// ``` +/// datastore_delete_all_by_eq_bsatn(table_id: u32, relation: u8[]) -> u32 throws { +/// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_INDEX | BSATN_DECODE_ERROR +/// } +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u16` is `number` in JS restricted to unsigned 16-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns a `u32` that is the number of rows deleted. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NOT_IN_TRANSACTION`] +/// when called outside of a transaction. +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_TABLE`] +/// when `table_id` is not a known ID of a table. +/// +/// - [`spacetimedb_primitives::errno::BSATN_DECODE_ERROR`] +/// when `relation` cannot be decoded to `Vec` +/// where each `ProductValue` is typed at the `ProductType` the table's schema specifies. +/// +/// Throws a `TypeError` if: +/// - `table_id` is not a `u32`. +/// - `relation` is not an array of `u8`s. +fn datastore_delete_all_by_eq_bsatn( + scope: &mut HandleScope<'_>, + args: FunctionCallbackArguments<'_>, +) -> SysCallResult { + let table_id: TableId = deserialize_js(scope, args.get(0))?; + let relation: &[u8] = deserialize_js(scope, args.get(1))?; + + let env = env_on_isolate(scope); + Ok(env.instance_env.datastore_delete_all_by_eq_bsatn(table_id, relation)?) +} + +/// # Signature +/// +/// ``` +/// volatile_nonatomic_schedule_immediate(reducer_name: string, args: u8[]) -> undefined +/// ``` +fn volatile_nonatomic_schedule_immediate<'scope>( + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, +) -> FnRet<'scope> { + let name: String = deserialize_js(scope, args.get(0))?; + let args: Vec = deserialize_js(scope, args.get(1))?; + + let env = env_on_isolate(scope); + env.instance_env + .scheduler + .volatile_nonatomic_schedule_immediate(name, crate::host::ReducerArgs::Bsatn(args.into())); + + Ok(v8::undefined(scope).into()) +} + +/// Module ABI that logs at `level` a `message` message occurring +/// at the parent stack frame. +/// +/// The `message` is interpreted lossily as a UTF-8 string. +/// +/// # Signature +/// +/// ``` +/// console_log(level: u8, message: string) -> u32 +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns nothing. +fn console_log<'scope>(scope: &mut HandleScope<'scope>, args: FunctionCallbackArguments<'scope>) -> FnRet<'scope> { + let level: u32 = deserialize_js(scope, args.get(0))?; + + let msg = args.get(1).cast::(); + let mut buf = scratch_buf::<128>(); + let msg = msg.to_rust_cow_lossy(scope, &mut buf); + + let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) + .ok_or_else(exception_already_thrown)? + .get_frame(scope, 1) + .ok_or_else(exception_already_thrown)?; + let mut buf = scratch_buf::<32>(); + let filename = frame + .get_script_name(scope) + .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); + + let record = Record { + // TODO: figure out whether to use walltime now or logical reducer now (env.reducer_start) + ts: chrono::Utc::now(), + target: None, + filename: filename.as_deref(), + line_number: Some(frame.get_line_number() as u32), + message: &msg, + }; + + let level = (level as u8).into(); + let trace = if level == LogLevel::Panic { + JsStackTrace::from_current_stack_trace(scope)? + } else { + <_>::default() + }; + + let env = env_on_isolate(scope); + env.instance_env.console_log(level, &record, &trace); + + Ok(v8::undefined(scope).into()) +} + +/// Module ABI that begins a timing span with `name`. +/// +/// When the returned `ConsoleTimerId` is passed to [`console_timer_end`], +/// the duration between the calls will be printed to the module's logs. +/// +/// The `name` is interpreted lossily as a UTF-8 string. +/// +/// # Signature +/// +/// ``` +/// console_timer_start(name: string) -> u32 +/// ``` +/// +/// # Types +/// +/// - `u8` is `number` in JS restricted to unsigned 8-bit integers. +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns a `u32` that is the `ConsoleTimerId`. +/// +/// # Throws +/// +/// Throws a `TypeError` if: +/// - `name` is not a `string`. +fn console_timer_start<'scope>( + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, +) -> FnRet<'scope> { + let name = args.get(0).cast::(); + let mut buf = scratch_buf::<128>(); + let name = name.to_rust_cow_lossy(scope, &mut buf).into_owned(); + + let env = env_on_isolate(scope); + let span_id = env.timing_spans.insert(TimingSpan::new(name)).0; + serialize_to_js(scope, &span_id) +} + +/// Module ABI that ends a timing span with `span_id`. +/// +/// # Signature +/// +/// ``` +/// console_timer_end(span_id: u32) -> undefined throws { +/// __code_error__: NO_SUCH_CONSOLE_TIMER +/// } +/// ``` +/// +/// # Types +///s +/// - `u32` is `number` in JS restricted to unsigned 32-bit integers. +/// +/// # Returns +/// +/// Returns nothing. +/// +/// # Throws +/// +/// Throws `{ __code_error__: u16 }` where `__code_error__` is: +/// +/// - [`spacetimedb_primitives::errno::NO_SUCH_CONSOLE_TIMER`] +/// when `span_id` doesn't refer to an active timing span. +/// +/// Throws a `TypeError` if: +/// - `span_id` is not a `u32`. +fn console_timer_end<'scope>( + scope: &mut HandleScope<'scope>, + args: FunctionCallbackArguments<'scope>, +) -> FnRet<'scope> { + let span_id: u32 = deserialize_js(scope, args.get(0))?; + + let env = env_on_isolate(scope); + let Some(span) = env.timing_spans.take(TimingSpanIdx(span_id)) else { + let exc = CodeError::from_code(scope, errno::NO_SUCH_CONSOLE_TIMER.get())?; + return Err(exc.throw(scope)); + }; + env.instance_env.console_timer_end(&span); + + Ok(v8::undefined(scope).into()) +} + +/// Module ABI that returns the module identity. +/// +/// # Signature +/// +/// ``` +/// identity() -> { __identity__: u256 } +/// ``` +/// +/// # Types +/// +/// - `u256` is `bigint` in JS restricted to unsigned 256-bit integers. +/// +/// # Returns +/// +/// Returns the module identity. +fn identity<'scope>(scope: &mut HandleScope<'scope>, _: FunctionCallbackArguments<'scope>) -> SysCallResult { + Ok(*env_on_isolate(scope).instance_env.database_identity()) +} diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 3393e57a579..08a492f9a4b 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -337,6 +337,7 @@ impl TimingSpan { decl_index!(TimingSpanIdx => TimingSpan); pub(super) type TimingSpanSet = ResourceSlab; +/// Converts a [`NodesError`] to an error code, if possible. pub fn err_to_errno(err: &NodesError) -> Option { match err { NodesError::NotInTransaction => Some(errno::NOT_IN_TRANSACTION), @@ -362,6 +363,18 @@ pub fn err_to_errno(err: &NodesError) -> Option { } } +/// Converts a [`NodesError`] to an error code and logs, if possible. +pub fn err_to_errno_and_log>(func: AbiCall, err: NodesError) -> anyhow::Result { + let Some(errno) = err_to_errno(&err) else { + return Err(AbiRuntimeError { func, err }.into()); + }; + log::debug!( + "abi call to {func} returned an errno: {errno} ({})", + errno::strerror(errno).unwrap_or("") + ); + Ok(errno.get().into()) +} + #[derive(Debug, thiserror::Error)] #[error("runtime error calling {func}: {err}")] pub struct AbiRuntimeError { diff --git a/crates/core/src/host/wasm_common/instrumentation.rs b/crates/core/src/host/wasm_common/instrumentation.rs index e2d3550e626..cf5c00fdc7c 100644 --- a/crates/core/src/host/wasm_common/instrumentation.rs +++ b/crates/core/src/host/wasm_common/instrumentation.rs @@ -118,3 +118,8 @@ impl CallTimes { std::mem::replace(self, Self::new()) } } + +#[cfg(not(feature = "spacetimedb-wasm-instance-env-times"))] +pub use noop as span; +#[cfg(feature = "spacetimedb-wasm-instance-env-times")] +pub use op as span; diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 10d4065d999..f0648bebd5b 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -1,30 +1,20 @@ #![allow(clippy::too_many_arguments)] use std::num::NonZeroU32; -use std::time::Instant; - +use super::{Mem, MemView, NullableMemOp, WasmError, WasmPointee, WasmPtr}; use crate::database_logger::{BacktraceFrame, BacktraceProvider, ModuleBacktrace, Record}; use crate::host::instance_env::{ChunkPool, InstanceEnv}; -use crate::host::wasm_common::instrumentation; +use crate::host::wasm_common::instrumentation::{span, CallTimes}; use crate::host::wasm_common::module_host_actor::ExecutionTimings; -use crate::host::wasm_common::{ - err_to_errno, instrumentation::CallTimes, AbiRuntimeError, RowIterIdx, RowIters, TimingSpan, TimingSpanIdx, - TimingSpanSet, -}; +use crate::host::wasm_common::{err_to_errno_and_log, RowIterIdx, RowIters, TimingSpan, TimingSpanIdx, TimingSpanSet}; use crate::host::AbiCall; use anyhow::Context as _; use spacetimedb_data_structures::map::IntMap; use spacetimedb_lib::Timestamp; use spacetimedb_primitives::{errno, ColId}; +use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; -use super::{Mem, MemView, NullableMemOp, WasmError, WasmPointee, WasmPtr}; - -#[cfg(not(feature = "spacetimedb-wasm-instance-env-times"))] -use instrumentation::noop as span; -#[cfg(feature = "spacetimedb-wasm-instance-env-times")] -use instrumentation::op as span; - /// A stream of bytes which the WASM module can read from /// using [`WasmInstanceEnv::bytes_source_read`]. /// @@ -302,20 +292,11 @@ impl WasmInstanceEnv { } fn convert_wasm_result>(func: AbiCall, err: WasmError) -> RtResult { - Err(match err { - WasmError::Db(err) => match err_to_errno(&err) { - Some(errno) => { - log::debug!( - "abi call to {func} returned an errno: {errno} ({})", - errno::strerror(errno).unwrap_or("") - ); - return Ok(errno.get().into()); - } - None => anyhow::Error::from(AbiRuntimeError { func, err }), - }, - WasmError::BufferTooSmall => return Ok(errno::BUFFER_TOO_SMALL.get().into()), - WasmError::Wasm(err) => err, - }) + match err { + WasmError::Db(err) => err_to_errno_and_log(func, err), + WasmError::BufferTooSmall => Ok(errno::BUFFER_TOO_SMALL.get().into()), + WasmError::Wasm(err) => Err(err), + } } /// Call the function `run` with the name `func`. @@ -1318,12 +1299,8 @@ impl WasmInstanceEnv { let Some(span) = caller.data_mut().timing_spans.take(TimingSpanIdx(span_id)) else { return Ok(errno::NO_SUCH_CONSOLE_TIMER.get().into()); }; - let function = caller.data().log_record_function(); - caller - .data() - .instance_env - .console_timer_end(&span, function, &caller.as_context()); + caller.data().instance_env.console_timer_end(&span, function); Ok(0) }) } From 8de9a4600459ea3d36c54b948887baf9cc9c8750 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 24 Sep 2025 12:17:18 +0200 Subject: [PATCH 13/39] fix rebase fallout --- crates/core/src/host/instance_env.rs | 4 ++-- crates/core/src/host/v8/mod.rs | 2 -- crates/core/src/host/v8/syscall.rs | 22 +++++++++++++--------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 7ffa1da3f9e..bcc552003ba 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -187,7 +187,7 @@ impl InstanceEnv { } #[tracing::instrument(level = "trace", skip_all)] - pub fn console_log(&self, level: LogLevel, record: &Record, bt: &dyn BacktraceProvider) { + pub(crate) fn console_log(&self, level: LogLevel, record: &Record, bt: &dyn BacktraceProvider) { self.replica_ctx.logger.write(level, record, bt); log::trace!( "MOD({}): {}", @@ -197,7 +197,7 @@ impl InstanceEnv { } /// End a console timer by logging the span at INFO level. - pub fn console_timer_end(&self, span: &TimingSpan, function: Option<&str>) { + pub(crate) fn console_timer_end(&self, span: &TimingSpan, function: Option<&str>) { let elapsed = span.start.elapsed(); let message = format!("Timing span {:?}: {:?}", &span.name, elapsed); diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 9d6337231be..7a34e59e8be 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -27,8 +27,6 @@ use spacetimedb_client_api_messages::energy::ReducerBudget; use spacetimedb_datastore::locking_tx_datastore::MutTxId; use spacetimedb_datastore::traits::Program; use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; -use spacetimedb_primitives::{ColId, IndexId, TableId}; -use spacetimedb_sats::Serialize; use spacetimedb_schema::auto_migrate::MigrationPolicy; use std::sync::{Arc, LazyLock}; use std::time::Instant; diff --git a/crates/core/src/host/v8/syscall.rs b/crates/core/src/host/v8/syscall.rs index d03c765a837..f42b7a2354b 100644 --- a/crates/core/src/host/v8/syscall.rs +++ b/crates/core/src/host/v8/syscall.rs @@ -937,23 +937,26 @@ fn console_log<'scope>(scope: &mut HandleScope<'scope>, args: FunctionCallbackAr .get_script_name(scope) .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); + let level = (level as u8).into(); + let trace = if level == LogLevel::Panic { + JsStackTrace::from_current_stack_trace(scope)? + } else { + <_>::default() + }; + + let env = env_on_isolate(scope); + + let function = env.log_record_function(); let record = Record { // TODO: figure out whether to use walltime now or logical reducer now (env.reducer_start) ts: chrono::Utc::now(), target: None, filename: filename.as_deref(), line_number: Some(frame.get_line_number() as u32), + function, message: &msg, }; - let level = (level as u8).into(); - let trace = if level == LogLevel::Panic { - JsStackTrace::from_current_stack_trace(scope)? - } else { - <_>::default() - }; - - let env = env_on_isolate(scope); env.instance_env.console_log(level, &record, &trace); Ok(v8::undefined(scope).into()) @@ -1036,7 +1039,8 @@ fn console_timer_end<'scope>( let exc = CodeError::from_code(scope, errno::NO_SUCH_CONSOLE_TIMER.get())?; return Err(exc.throw(scope)); }; - env.instance_env.console_timer_end(&span); + let function = env.log_record_function(); + env.instance_env.console_timer_end(&span, function); Ok(v8::undefined(scope).into()) } From 526d5d1a77ddabaf01eb061a589aa7c79495afd0 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 26 Sep 2025 11:14:15 +0200 Subject: [PATCH 14/39] cli/publish: use --js-path instead of --javascript --- crates/cli/src/subcommands/publish.rs | 41 ++++++++++++++++----------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 3a399bacde0..b1e5f5689e0 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -46,16 +46,18 @@ pub fn cli() -> clap::Command { .short('b') .conflicts_with("project_path") .conflicts_with("build_options") + .conflicts_with("js_file") .help("The system path (absolute or relative) to the compiled wasm binary we should publish, instead of building the project."), ) - // TODO(v8): needs better UX but good enough for a demo... .arg( - Arg::new("javascript") - .long("javascript") - .action(SetTrue) - .requires("wasm_file") - .hide(true) - .help("UNSTABLE: interpret `--bin-path` as a JS module"), + Arg::new("js_file") + .value_parser(clap::value_parser!(PathBuf)) + .long("js-path") + .short('j') + .conflicts_with("project_path") + .conflicts_with("build_options") + .conflicts_with("wasm_file") + .help("UNSTABLE: The system path (absolute or relative) to the javascript file we should publish, instead of building the project."), ) .arg( Arg::new("num_replicas") @@ -99,8 +101,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E let force = args.get_flag("force"); let anon_identity = args.get_flag("anon_identity"); let wasm_file = args.get_one::("wasm_file"); - // TODO(v8): needs better UX but good enough for a demo... - let wasm_file_is_really_js = args.get_flag("javascript"); + let js_file = args.get_one::("js_file"); let database_host = config.get_host_url(server)?; let build_options = args.get_one::("build_options").unwrap(); let num_replicas = args.get_one::("num_replicas"); @@ -119,17 +120,23 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E )); } - let path_to_wasm = if let Some(path) = wasm_file { - println!("Skipping build. Instead we are publishing {}", path.display()); - path.clone() + // Decide program file path and read program. + // Optionally build the program. + let (path_to_program, host_type) = if let Some(path) = wasm_file { + println!("(WASM) Skipping build. Instead we are publishing {}", path.display()); + (path.clone(), None) + } else if let Some(path) = js_file { + println!("(JS) Skipping build. Instead we are publishing {}", path.display()); + (path.clone(), Some("Js")) } else { - build::exec_with_argstring(config.clone(), path_to_project, build_options).await? + let path = build::exec_with_argstring(config.clone(), path_to_project, build_options).await?; + (path, None) }; - let program_bytes = fs::read(path_to_wasm)?; + let program_bytes = fs::read(path_to_program)?; - // TODO(v8): needs better UX but good enough for a demo... - if wasm_file_is_really_js { - builder = builder.query(&[("host_type", "Js")]); + // The host type is not the default (WASM). + if let Some(host_type) = host_type { + builder = builder.query(&[("host_type", host_type)]); } let server_address = { From 9521ecdae0a5b50cfea8b56f074c526a75722218 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 26 Sep 2025 11:41:06 +0200 Subject: [PATCH 15/39] v8 syscalls: ignore signatures; not rust code --- crates/core/src/host/v8/syscall.rs | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/core/src/host/v8/syscall.rs b/crates/core/src/host/v8/syscall.rs index f42b7a2354b..cf053045067 100644 --- a/crates/core/src/host/v8/syscall.rs +++ b/crates/core/src/host/v8/syscall.rs @@ -188,7 +188,7 @@ fn with_span<'scope, R>( /// /// # Signature /// -/// ``` +/// ```ignore /// table_id_from_name(name: string) -> u32 throws { /// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_TABLE /// } @@ -224,7 +224,7 @@ fn table_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArgumen /// /// # Signature /// -/// ``` +/// ```ignore /// index_id_from_name(name: string) -> u32 throws { /// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_INDEX /// } @@ -260,7 +260,7 @@ fn index_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArgumen /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_table_row_count(table_id: u32) -> u64 throws { /// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_TABLE /// } @@ -298,7 +298,7 @@ fn datastore_table_row_count(scope: &mut HandleScope<'_>, args: FunctionCallback /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_table_scan_bsatn(table_id: u32) -> u32 throws { /// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_TABLE /// } @@ -379,7 +379,7 @@ fn datastore_table_scan_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbac /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_index_scan_range_bsatn( /// index_id: u32, /// prefix: u8[], @@ -479,7 +479,7 @@ fn no_such_iter(scope: &mut HandleScope<'_>) -> ExceptionThrown { /// /// # Signature /// -/// ``` +/// ```ignore /// row_iter_bsatn_advance(iter: u32, buffer_max_len: u32) -> (boolean, u8[]) throws /// { __code_error__: NO_SUCH_ITER } | { __buffer_too_small__: number } /// ``` @@ -553,7 +553,7 @@ fn row_iter_bsatn_advance<'scope>( /// /// # Signature /// -/// ``` +/// ```ignore /// row_iter_bsatn_close(iter: u32) -> undefined throws { /// __code_error__: NO_SUCH_ITER /// } @@ -614,7 +614,7 @@ fn row_iter_bsatn_close<'scope>( /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_insert_bsatn(table_id: u32, row: u8[]) -> u8[] throws { /// __code_error__: /// NOT_IN_TRANSACTION @@ -687,7 +687,7 @@ fn datastore_insert_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArg /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_update_bsatn(table_id: u32, index_id: u32, row: u8[]) -> u8[] throws { /// __code_error__: /// NOT_IN_TRANSACTION @@ -760,7 +760,7 @@ fn datastore_update_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArg /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_index_scan_range_bsatn( /// index_id: u32, /// prefix: u8[], @@ -838,7 +838,7 @@ fn datastore_delete_by_index_scan_range_bsatn( /// /// # Signature /// -/// ``` +/// ```ignore /// datastore_delete_all_by_eq_bsatn(table_id: u32, relation: u8[]) -> u32 throws { /// __code_error__: NOT_IN_TRANSACTION | NO_SUCH_INDEX | BSATN_DECODE_ERROR /// } @@ -884,7 +884,7 @@ fn datastore_delete_all_by_eq_bsatn( /// # Signature /// -/// ``` +/// ```ignore /// volatile_nonatomic_schedule_immediate(reducer_name: string, args: u8[]) -> undefined /// ``` fn volatile_nonatomic_schedule_immediate<'scope>( @@ -909,7 +909,7 @@ fn volatile_nonatomic_schedule_immediate<'scope>( /// /// # Signature /// -/// ``` +/// ```ignore /// console_log(level: u8, message: string) -> u32 /// ``` /// @@ -971,7 +971,7 @@ fn console_log<'scope>(scope: &mut HandleScope<'scope>, args: FunctionCallbackAr /// /// # Signature /// -/// ``` +/// ```ignore /// console_timer_start(name: string) -> u32 /// ``` /// @@ -1005,7 +1005,7 @@ fn console_timer_start<'scope>( /// /// # Signature /// -/// ``` +/// ```ignore /// console_timer_end(span_id: u32) -> undefined throws { /// __code_error__: NO_SUCH_CONSOLE_TIMER /// } @@ -1049,7 +1049,7 @@ fn console_timer_end<'scope>( /// /// # Signature /// -/// ``` +/// ```ignore /// identity() -> { __identity__: u256 } /// ``` /// From 065161deedb78b1033b5f7c8c860a305dbd8a7b2 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 26 Sep 2025 13:43:59 +0200 Subject: [PATCH 16/39] v8 -> 140.2, s/HandleScope/PinScope/g --- Cargo.lock | 4 +- Cargo.toml | 2 +- crates/core/src/host/v8/de.rs | 62 +++++++++++++-------------- crates/core/src/host/v8/error.rs | 38 ++++++++-------- crates/core/src/host/v8/from_value.rs | 14 +++--- crates/core/src/host/v8/key_cache.rs | 14 +++--- crates/core/src/host/v8/mod.rs | 35 +++++++-------- crates/core/src/host/v8/ser.rs | 40 ++++++++--------- crates/core/src/host/v8/syscall.rs | 58 ++++++++++++------------- crates/core/src/host/v8/to_value.rs | 16 +++---- 10 files changed, 142 insertions(+), 141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19f2bcb4acd..62f9b944849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7908,9 +7908,9 @@ dependencies = [ [[package]] name = "v8" -version = "140.0.0" +version = "140.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0ad6613428e148bb814ee8890a23bb250547a8f837922ca1ea6eba90670514" +checksum = "8827809a2884fb68530d678a8ef15b1ed1344bbf844879194d68c140c6f844f9" dependencies = [ "bindgen 0.72.1", "bitflags 2.9.0", diff --git a/Cargo.toml b/Cargo.toml index 9c1be60a881..446a3e35c9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -292,7 +292,7 @@ unicode-normalization = "0.1.23" url = "2.3.1" urlencoding = "2.1.2" uuid = { version = "1.2.1", features = ["v4"] } -v8 = "140.0" +v8 = "140.2" walkdir = "2.2.5" wasmbin = "0.6" webbrowser = "1.0.2" diff --git a/crates/core/src/host/v8/de.rs b/crates/core/src/host/v8/de.rs index 7828d5bdf65..0eed5810c4f 100644 --- a/crates/core/src/host/v8/de.rs +++ b/crates/core/src/host/v8/de.rs @@ -11,11 +11,11 @@ use derive_more::From; use spacetimedb_sats::de::{self, ArrayVisitor, DeserializeSeed, ProductVisitor, SliceVisitor, SumVisitor}; use spacetimedb_sats::{i256, u256}; use std::borrow::{Borrow, Cow}; -use v8::{Array, HandleScope, Local, Name, Object, Uint8Array, Value}; +use v8::{Array, Local, Name, Object, PinScope, Uint8Array, Value}; /// Deserializes a `T` from `val` in `scope`, using `seed` for any context needed. pub(super) fn deserialize_js_seed<'de, T: DeserializeSeed<'de>>( - scope: &mut HandleScope<'de>, + scope: &mut PinScope<'de, '_>, val: Local<'_, Value>, seed: T, ) -> ExcResult { @@ -27,21 +27,21 @@ pub(super) fn deserialize_js_seed<'de, T: DeserializeSeed<'de>>( /// Deserializes a `T` from `val` in `scope`. pub(super) fn deserialize_js<'de, T: de::Deserialize<'de>>( - scope: &mut HandleScope<'de>, + scope: &mut PinScope<'de, '_>, val: Local<'_, Value>, ) -> ExcResult { deserialize_js_seed(scope, val, PhantomData) } /// Deserializes from V8 values. -struct Deserializer<'this, 'scope> { - common: DeserializerCommon<'this, 'scope>, +struct Deserializer<'this, 'scope, 'isolate> { + common: DeserializerCommon<'this, 'scope, 'isolate>, input: Local<'scope, Value>, } -impl<'this, 'scope> Deserializer<'this, 'scope> { +impl<'this, 'scope, 'isolate> Deserializer<'this, 'scope, 'isolate> { /// Creates a new deserializer from `input` in `scope`. - fn new(scope: &'this mut HandleScope<'scope>, input: Local<'_, Value>, key_cache: &'this mut KeyCache) -> Self { + fn new(scope: &'this mut PinScope<'scope, 'isolate>, input: Local<'_, Value>, key_cache: &'this mut KeyCache) -> Self { let input = Local::new(scope, input); let common = DeserializerCommon { scope, key_cache }; Deserializer { input, common } @@ -51,15 +51,15 @@ impl<'this, 'scope> Deserializer<'this, 'scope> { /// Things shared between various [`Deserializer`]s. /// /// The lifetime `'scope` is that of the scope of values deserialized. -struct DeserializerCommon<'this, 'scope> { +struct DeserializerCommon<'this, 'scope, 'isolate> { /// The scope of values to deserialize. - scope: &'this mut HandleScope<'scope>, + scope: &'this mut PinScope<'scope, 'isolate>, /// A cache for frequently used strings. key_cache: &'this mut KeyCache, } -impl<'scope> DeserializerCommon<'_, 'scope> { - fn reborrow(&mut self) -> DeserializerCommon<'_, 'scope> { +impl<'scope, 'isolate> DeserializerCommon<'_, 'scope, 'isolate> { + fn reborrow(&mut self) -> DeserializerCommon<'_, 'scope, 'isolate> { DeserializerCommon { scope: self.scope, key_cache: self.key_cache, @@ -76,7 +76,7 @@ enum Error<'scope> { } impl<'scope> Throwable<'scope> for Error<'scope> { - fn throw(self, scope: &mut HandleScope<'scope>) -> ExceptionThrown { + fn throw(self, scope: &PinScope<'scope, '_>) -> ExceptionThrown { match self { Self::Unthrown(exception) => exception.throw(scope), Self::Thrown(thrown) => thrown, @@ -118,7 +118,7 @@ macro_rules! deserialize_primitive { }; } -impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'scope> { +impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'scope, '_> { type Error = Error<'scope>; // Deserialization of primitive types defers to `FromValue`. @@ -156,7 +156,7 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco } fn deserialize_sum>(self, visitor: V) -> Result { - let scope = &mut *self.common.scope; + let scope = &*self.common.scope; let sum_name = visitor.sum_name().unwrap_or(""); // We expect a canonical representation of a sum value in JS to be @@ -187,7 +187,7 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco fn deserialize_str>(self, visitor: V) -> Result { let val = cast!(self.common.scope, self.input, v8::String, "`string`")?; let mut buf = scratch_buf::<64>(); - match val.to_rust_cow_lossy(self.common.scope, &mut buf) { + match val.to_rust_cow_lossy(&mut *self.common.scope, &mut buf) { Cow::Borrowed(s) => visitor.visit(s), Cow::Owned(string) => visitor.visit_owned(string), } @@ -212,8 +212,8 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco /// Provides access to the field names and values in a JS object /// under the assumption that it's a product. -struct ProductAccess<'this, 'scope> { - common: DeserializerCommon<'this, 'scope>, +struct ProductAccess<'this, 'scope, 'isolate> { + common: DeserializerCommon<'this, 'scope, 'isolate>, /// The input object being deserialized. object: Local<'scope, Object>, /// A field's value, to deserialize next in [`NamedProductAccess::get_field_value_seed`]. @@ -223,7 +223,7 @@ struct ProductAccess<'this, 'scope> { } // Creates an interned [`v8::String`]. -pub(super) fn v8_interned_string<'scope>(scope: &mut HandleScope<'scope>, field: &str) -> Local<'scope, v8::String> { +pub(super) fn v8_interned_string<'scope>(scope: &PinScope<'scope, '_>, field: &str) -> Local<'scope, v8::String> { // Internalized v8 strings are significantly faster than "normal" v8 strings // since v8 deduplicates re-used strings minimizing new allocations // see: https://github.com/v8/v8/blob/14ac92e02cc3db38131a57e75e2392529f405f2f/include/v8.h#L3165-L3171 @@ -232,7 +232,7 @@ pub(super) fn v8_interned_string<'scope>(scope: &mut HandleScope<'scope>, field: /// Normalizes `field` into an interned `v8::String`. pub(super) fn intern_field_name<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, field: Option<&str>, index: usize, ) -> Local<'scope, Name> { @@ -243,11 +243,11 @@ pub(super) fn intern_field_name<'scope>( v8_interned_string(scope, &field).into() } -impl<'de, 'scope: 'de> de::NamedProductAccess<'de> for ProductAccess<'_, 'scope> { +impl<'de, 'scope: 'de> de::NamedProductAccess<'de> for ProductAccess<'_, 'scope, '_> { type Error = Error<'scope>; fn get_field_ident>(&mut self, visitor: V) -> Result, Self::Error> { - let scope = &mut *self.common.scope; + let scope = &*self.common.scope; let mut field_names = visitor.field_names(); while let Some(field) = field_names.nth(self.index) { // Get and advance the current index. @@ -296,17 +296,17 @@ impl<'de, 'scope: 'de> de::NamedProductAccess<'de> for ProductAccess<'_, 'scope> /// Used in `Deserializer::deserialize_sum` to translate a `tag` property of a JS object /// to a variant and to provide a deserializer for its value/payload. -struct SumAccess<'this, 'scope> { - common: DeserializerCommon<'this, 'scope>, +struct SumAccess<'this, 'scope, 'isolate> { + common: DeserializerCommon<'this, 'scope, 'isolate>, /// The tag of the sum value. tag: Local<'scope, v8::String>, /// The value of the sum value. value: Local<'scope, Value>, } -impl<'de, 'this, 'scope: 'de> de::SumAccess<'de> for SumAccess<'this, 'scope> { +impl<'de, 'this, 'scope: 'de, 'isolate> de::SumAccess<'de> for SumAccess<'this, 'scope, 'isolate> { type Error = Error<'scope>; - type Variant = Deserializer<'this, 'scope>; + type Variant = Deserializer<'this, 'scope, 'isolate>; fn variant>(self, visitor: V) -> Result<(V::Output, Self::Variant), Self::Error> { // Read the `tag` property in JS. @@ -328,7 +328,7 @@ impl<'de, 'this, 'scope: 'de> de::SumAccess<'de> for SumAccess<'this, 'scope> { } } -impl<'de, 'this, 'scope: 'de> de::VariantAccess<'de> for Deserializer<'this, 'scope> { +impl<'de, 'this, 'scope: 'de> de::VariantAccess<'de> for Deserializer<'this, 'scope, '_> { type Error = Error<'scope>; fn deserialize_seed>(self, seed: T) -> Result { @@ -338,18 +338,18 @@ impl<'de, 'this, 'scope: 'de> de::VariantAccess<'de> for Deserializer<'this, 'sc /// Used by an `ArrayVisitor` to deserialize every element of a JS array /// to a SATS array. -struct ArrayAccess<'this, 'scope, T> { - common: DeserializerCommon<'this, 'scope>, +struct ArrayAccess<'this, 'scope, 'isolate, T> { + common: DeserializerCommon<'this, 'scope, 'isolate>, arr: Local<'scope, Array>, seeds: RepeatN, index: u32, } -impl<'de, 'this, 'scope, T> ArrayAccess<'this, 'scope, T> +impl<'de, 'this, 'scope, 'isolate, T> ArrayAccess<'this, 'scope, 'isolate, T> where T: DeserializeSeed<'de> + Clone, { - fn new(arr: Local<'scope, Array>, common: DeserializerCommon<'this, 'scope>, seed: T) -> Self { + fn new(arr: Local<'scope, Array>, common: DeserializerCommon<'this, 'scope, 'isolate>, seed: T) -> Self { Self { arr, common, @@ -359,7 +359,7 @@ where } } -impl<'de, 'scope: 'de, T: DeserializeSeed<'de> + Clone> de::ArrayAccess<'de> for ArrayAccess<'_, 'scope, T> { +impl<'de, 'scope: 'de, T: DeserializeSeed<'de> + Clone> de::ArrayAccess<'de> for ArrayAccess<'_, 'scope, '_, T> { type Element = T::Output; type Error = Error<'scope>; diff --git a/crates/core/src/host/v8/error.rs b/crates/core/src/host/v8/error.rs index 49dd6f72e1b..1169845a9a4 100644 --- a/crates/core/src/host/v8/error.rs +++ b/crates/core/src/host/v8/error.rs @@ -5,7 +5,7 @@ use crate::database_logger::{BacktraceFrame, BacktraceProvider, ModuleBacktrace} use super::serialize_to_js; use core::fmt; use spacetimedb_sats::Serialize; -use v8::{Exception, HandleScope, Local, StackFrame, StackTrace, TryCatch, Value}; +use v8::{tc_scope, Exception, HandleScope, Local, PinScope, PinnedRef, StackFrame, StackTrace, TryCatch, Value}; /// The result of trying to convert a [`Value`] in scope `'scope` to some type `T`. pub(super) type ValueResult<'scope, T> = Result>; @@ -13,11 +13,11 @@ pub(super) type ValueResult<'scope, T> = Result>; /// Types that can convert into a JS string type. pub(super) trait IntoJsString { /// Converts `self` into a JS string. - fn into_string<'scope>(self, scope: &mut HandleScope<'scope>) -> Local<'scope, v8::String>; + fn into_string<'scope>(self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String>; } impl IntoJsString for String { - fn into_string<'scope>(self, scope: &mut HandleScope<'scope>) -> Local<'scope, v8::String> { + fn into_string<'scope>(self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { v8::String::new(scope, &self).unwrap() } } @@ -31,11 +31,11 @@ pub(super) struct ExceptionValue<'scope>(Local<'scope, Value>); /// Error types that can convert into JS exception values. pub(super) trait IntoException<'scope> { /// Converts `self` into a JS exception value. - fn into_exception(self, scope: &mut HandleScope<'scope>) -> ExceptionValue<'scope>; + fn into_exception(self, scope: &PinScope<'scope, '_>) -> ExceptionValue<'scope>; } impl<'scope> IntoException<'scope> for ExceptionValue<'scope> { - fn into_exception(self, _: &mut HandleScope<'scope>) -> ExceptionValue<'scope> { + fn into_exception(self, _: &PinScope<'scope, '_>) -> ExceptionValue<'scope> { self } } @@ -45,7 +45,7 @@ impl<'scope> IntoException<'scope> for ExceptionValue<'scope> { pub struct TypeError(pub M); impl<'scope, M: IntoJsString> IntoException<'scope> for TypeError { - fn into_exception(self, scope: &mut HandleScope<'scope>) -> ExceptionValue<'scope> { + fn into_exception(self, scope: &PinScope<'scope, '_>) -> ExceptionValue<'scope> { let msg = self.0.into_string(scope); ExceptionValue(Exception::type_error(scope, msg)) } @@ -56,7 +56,7 @@ impl<'scope, M: IntoJsString> IntoException<'scope> for TypeError { pub struct RangeError(pub M); impl<'scope, M: IntoJsString> IntoException<'scope> for RangeError { - fn into_exception(self, scope: &mut HandleScope<'scope>) -> ExceptionValue<'scope> { + fn into_exception(self, scope: &PinScope<'scope, '_>) -> ExceptionValue<'scope> { let msg = self.0.into_string(scope); ExceptionValue(Exception::range_error(scope, msg)) } @@ -71,7 +71,7 @@ pub(super) struct TerminationError { impl TerminationError { /// Convert `anyhow::Error` to a termination error. pub(super) fn from_error<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, error: &anyhow::Error, ) -> ExcResult> { let __terminated__ = format!("{error}"); @@ -90,7 +90,7 @@ pub(super) struct CodeError { impl CodeError { /// Create a code error from a code. pub(super) fn from_code<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, __code_error__: u16, ) -> ExcResult> { let error = Self { __code_error__ }; @@ -108,7 +108,7 @@ pub(super) struct BufferTooSmall { impl BufferTooSmall { /// Create a code error from a code. pub(super) fn from_requirement<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, __buffer_too_small__: u32, ) -> ExcResult> { let error = Self { __buffer_too_small__ }; @@ -135,11 +135,11 @@ pub(super) trait Throwable<'scope> { /// /// If an exception has already been thrown, /// [`ExceptionThrown`] can be returned directly. - fn throw(self, scope: &mut HandleScope<'scope>) -> ExceptionThrown; + fn throw(self, scope: &PinScope<'scope, '_>) -> ExceptionThrown; } impl<'scope, T: IntoException<'scope>> Throwable<'scope> for T { - fn throw(self, scope: &mut HandleScope<'scope>) -> ExceptionThrown { + fn throw(self, scope: &PinScope<'scope, '_>) -> ExceptionThrown { let ExceptionValue(exception) = self.into_exception(scope); scope.throw_exception(exception); exception_already_thrown() @@ -199,7 +199,7 @@ pub(super) struct JsStackTrace { impl JsStackTrace { /// Converts a V8 [`StackTrace`] into one independent of `'scope`. - pub(super) fn from_trace<'scope>(scope: &mut HandleScope<'scope>, trace: Local<'scope, StackTrace>) -> Self { + pub(super) fn from_trace<'scope>(scope: &PinScope<'scope, '_>, trace: Local<'scope, StackTrace>) -> Self { let frames = (0..trace.get_frame_count()) .map(|index| { let frame = trace.get_frame(scope, index).unwrap(); @@ -210,7 +210,7 @@ impl JsStackTrace { } /// Construct a backtrace from `scope`. - pub(super) fn from_current_stack_trace(scope: &mut HandleScope<'_>) -> ExcResult { + pub(super) fn from_current_stack_trace(scope: &PinScope<'_, '_>) -> ExcResult { let trace = StackTrace::current_stack_trace(scope, 1024).ok_or_else(exception_already_thrown)?; Ok(Self::from_trace(scope, trace)) } @@ -260,7 +260,7 @@ pub(super) struct JsStackTraceFrame { impl JsStackTraceFrame { /// Converts a V8 [`StackFrame`] into one independent of `'scope`. - fn from_frame<'scope>(scope: &mut HandleScope<'scope>, frame: Local<'scope, StackFrame>) -> Self { + fn from_frame<'scope>(scope: &PinScope<'scope, '_>, frame: Local<'scope, StackFrame>) -> Self { let script_name = frame .get_script_name_or_source_url(scope) .map(|s| s.to_rust_string_lossy(scope)); @@ -326,7 +326,7 @@ impl fmt::Display for JsStackTraceFrame { impl JsError { /// Turns a caught JS exception in `scope` into a [`JSError`]. - fn from_caught(scope: &mut TryCatch<'_, HandleScope<'_>>) -> Self { + fn from_caught(scope: &PinnedRef<'_, TryCatch<'_, '_, HandleScope<'_>>>) -> Self { match scope.message() { Some(message) => Self { trace: message @@ -355,10 +355,10 @@ pub(super) fn log_traceback(func_type: &str, func: &str, e: &anyhow::Error) { /// Run `body` within a try-catch context and capture any JS exception thrown as a [`JsError`]. pub(super) fn catch_exception<'scope, T>( - scope: &mut HandleScope<'scope>, - body: impl FnOnce(&mut HandleScope<'scope>) -> Result>, + scope: &mut PinScope<'scope, '_>, + body: impl FnOnce(&mut PinScope<'scope, '_>) -> Result>, ) -> Result> { - let scope = &mut TryCatch::new(scope); + tc_scope!(scope, scope); let ret = body(scope); ret.map_err(|e| match e { ErrorOrException::Err(e) => ErrorOrException::Err(e), diff --git a/crates/core/src/host/v8/from_value.rs b/crates/core/src/host/v8/from_value.rs index e3322eff0e9..61f70676c78 100644 --- a/crates/core/src/host/v8/from_value.rs +++ b/crates/core/src/host/v8/from_value.rs @@ -5,12 +5,12 @@ use crate::host::v8::error::ExceptionValue; use super::error::{IntoException as _, TypeError, ValueResult}; use bytemuck::{AnyBitPattern, NoUninit}; use spacetimedb_sats::{i256, u256}; -use v8::{BigInt, Boolean, HandleScope, Int32, Local, Number, Uint32, Value}; +use v8::{BigInt, Boolean, Int32, Local, Number, PinScope, Uint32, Value}; /// Types that a v8 [`Value`] can be converted into. pub(super) trait FromValue: Sized { /// Converts `val` in `scope` to `Self` if possible. - fn from_value<'scope>(val: Local<'_, Value>, scope: &mut HandleScope<'scope>) -> ValueResult<'scope, Self>; + fn from_value<'scope>(val: Local<'_, Value>, scope: &PinScope<'scope, '_>) -> ValueResult<'scope, Self>; } /// Provides a [`FromValue`] implementation. @@ -19,7 +19,7 @@ macro_rules! impl_from_value { impl FromValue for $ty { fn from_value<'scope>( $val: Local<'_, Value>, - $scope: &mut HandleScope<'scope>, + $scope: &PinScope<'scope, '_>, ) -> ValueResult<'scope, Self> { $logic } @@ -29,7 +29,7 @@ macro_rules! impl_from_value { /// Tries to cast `Value` into `T` or raises a JS exception as a returned `Err` value. pub(super) fn try_cast<'scope_a, 'scope_b, T>( - scope: &mut HandleScope<'scope_a>, + scope: &PinScope<'scope_a, '_>, val: Local<'scope_b, Value>, on_err: impl FnOnce(&str) -> String, ) -> ValueResult<'scope_a, Local<'scope_b, T>> @@ -50,13 +50,13 @@ pub(super) use cast; /// Returns a JS exception value indicating that a value overflowed /// when converting to the type `rust_ty`. -fn value_overflowed<'scope>(rust_ty: &str, scope: &mut HandleScope<'scope>) -> ExceptionValue<'scope> { +fn value_overflowed<'scope>(rust_ty: &str, scope: &PinScope<'scope, '_>) -> ExceptionValue<'scope> { TypeError(format!("Value overflowed `{rust_ty}`")).into_exception(scope) } /// Returns a JS exception value indicating that a value underflowed /// when converting to the type `rust_ty`. -fn value_underflowed<'scope>(rust_ty: &str, scope: &mut HandleScope<'scope>) -> ExceptionValue<'scope> { +fn value_underflowed<'scope>(rust_ty: &str, scope: &PinScope<'scope, '_>) -> ExceptionValue<'scope> { TypeError(format!("Value underflowed `{rust_ty}`")).into_exception(scope) } @@ -120,7 +120,7 @@ int64_from_value!(i64, i64_value); /// - `bigint` is the integer to convert. fn bigint_to_bytes<'scope, const N: usize, const W: usize, const UNSIGNED: bool>( rust_ty: &str, - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, bigint: &BigInt, ) -> ValueResult<'scope, (bool, [u8; N])> where diff --git a/crates/core/src/host/v8/key_cache.rs b/crates/core/src/host/v8/key_cache.rs index bd0356d8eca..27c76d7f60e 100644 --- a/crates/core/src/host/v8/key_cache.rs +++ b/crates/core/src/host/v8/key_cache.rs @@ -1,12 +1,12 @@ use super::de::v8_interned_string; use core::cell::RefCell; use std::rc::Rc; -use v8::{Global, HandleScope, Local}; +use v8::{Global, Local, PinScope}; /// Returns a `KeyCache` for the current `scope`. /// /// Creates the cache in the scope if it doesn't exist yet. -pub(super) fn get_or_create_key_cache(scope: &mut HandleScope<'_>) -> Rc> { +pub(super) fn get_or_create_key_cache(scope: &PinScope<'_, '_>) -> Rc> { let context = scope.get_current_context(); context.get_slot::>().unwrap_or_else(|| { let cache = Rc::default(); @@ -30,29 +30,29 @@ pub(super) struct KeyCache { impl KeyCache { /// Returns the `tag` property name. - pub(super) fn tag<'scope>(&mut self, scope: &mut HandleScope<'scope>) -> Local<'scope, v8::String> { + pub(super) fn tag<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { Self::get_or_create_key(scope, &mut self.tag, "tag") } /// Returns the `value` property name. - pub(super) fn value<'scope>(&mut self, scope: &mut HandleScope<'scope>) -> Local<'scope, v8::String> { + pub(super) fn value<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { Self::get_or_create_key(scope, &mut self.value, "value") } /// Returns the `__describe_module__` property name. - pub(super) fn describe_module<'scope>(&mut self, scope: &mut HandleScope<'scope>) -> Local<'scope, v8::String> { + pub(super) fn describe_module<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { Self::get_or_create_key(scope, &mut self.describe_module, "__describe_module__") } /// Returns the `__call_reducer__` property name. - pub(super) fn call_reducer<'scope>(&mut self, scope: &mut HandleScope<'scope>) -> Local<'scope, v8::String> { + pub(super) fn call_reducer<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { Self::get_or_create_key(scope, &mut self.call_reducer, "__call_reducer__") } /// Returns an interned string corresponding to `string` /// and memoizes the creation on the v8 side. fn get_or_create_key<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, slot: &mut Option>, string: &str, ) -> Local<'scope, v8::String> { diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 7a34e59e8be..9345555d288 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -31,8 +31,7 @@ use spacetimedb_schema::auto_migrate::MigrationPolicy; use std::sync::{Arc, LazyLock}; use std::time::Instant; use v8::{ - Context, ContextOptions, ContextScope, Function, HandleScope, Isolate, IsolateHandle, Local, Object, OwnedIsolate, - Value, + scope, Context, ContextScope, Function, Isolate, IsolateHandle, Local, Object, OwnedIsolate, PinScope, Value, }; mod de; @@ -390,7 +389,7 @@ fn with_script( callback_every: u64, callback: InterruptCallback, budget: ReducerBudget, - logic: impl for<'scope> FnOnce(&mut HandleScope<'scope>, Local<'scope, Value>) -> R, + logic: impl for<'scope> FnOnce(&mut PinScope<'scope, '_>, Local<'scope, Value>) -> R, ) -> (OwnedIsolate, R) { with_scope(isolate, callback_every, callback, budget, |scope| { let code = v8::String::new(scope, code).unwrap(); @@ -408,19 +407,21 @@ pub(crate) fn with_scope( callback_every: u64, callback: InterruptCallback, budget: ReducerBudget, - logic: impl FnOnce(&mut HandleScope<'_>) -> R, + logic: impl FnOnce(&mut PinScope<'_, '_>) -> R, ) -> (OwnedIsolate, R) { isolate.set_capture_stack_trace_for_uncaught_exceptions(true, 1024); let isolate_handle = isolate.thread_safe_handle(); - let mut scope_1 = HandleScope::new(&mut isolate); - let context = Context::new(&mut scope_1, ContextOptions::default()); - let mut scope_2 = ContextScope::new(&mut scope_1, context); + + let with_isolate = |isolate: &mut OwnedIsolate| -> R { + scope!(let scope, isolate); + let context = Context::new(scope, Default::default()); + let scope = &mut ContextScope::new(scope, context); + logic(scope) + }; let timeout_thread_cancel_flag = run_reducer_timeout(callback_every, callback, budget, isolate_handle); - let ret = logic(&mut scope_2); - drop(scope_2); - drop(scope_1); + let ret = with_isolate(&mut isolate); // Cancel the execution timeout in `run_reducer_timeout`. timeout_thread_cancel_flag.store(true, Ordering::Relaxed); @@ -489,12 +490,12 @@ fn duration_to_budget(_duration: Duration) -> ReducerBudget { } /// Returns the global object. -fn global<'scope>(scope: &mut HandleScope<'scope>) -> Local<'scope, Object> { +fn global<'scope>(scope: &PinScope<'scope, '_>) -> Local<'scope, Object> { scope.get_current_context().global(scope) } /// Returns the global property `key`. -fn get_global_property<'scope>(scope: &mut HandleScope<'scope>, key: Local<'scope, v8::String>) -> FnRet<'scope> { +fn get_global_property<'scope>(scope: &PinScope<'scope, '_>, key: Local<'scope, v8::String>) -> FnRet<'scope> { global(scope) .get(scope, key.into()) .ok_or_else(exception_already_thrown) @@ -502,7 +503,7 @@ fn get_global_property<'scope>(scope: &mut HandleScope<'scope>, key: Local<'scop /// Calls free function `fun` with `args`. fn call_free_fun<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, fun: Local<'scope, Function>, args: &[Local<'scope, Value>], ) -> FnRet<'scope> { @@ -511,7 +512,7 @@ fn call_free_fun<'scope>( } // Calls the `__call_reducer__` function on the global proxy object using `op`. -fn call_call_reducer_from_op(scope: &mut HandleScope<'_>, op: ReducerOp<'_>) -> anyhow::Result>> { +fn call_call_reducer_from_op(scope: &mut PinScope<'_, '_>, op: ReducerOp<'_>) -> anyhow::Result>> { call_call_reducer( scope, op.id.into(), @@ -524,7 +525,7 @@ fn call_call_reducer_from_op(scope: &mut HandleScope<'_>, op: ReducerOp<'_>) -> // Calls the `__call_reducer__` function on the global proxy object. fn call_call_reducer( - scope: &mut HandleScope<'_>, + scope: &mut PinScope<'_, '_>, reducer_id: u32, sender: &Identity, conn_id: &ConnectionId, @@ -578,7 +579,7 @@ fn extract_description(program: &str) -> Result { } // Calls the `__describe_module__` function on the global proxy object to extract a [`RawModuleDef`]. -fn call_describe_module(scope: &mut HandleScope<'_>) -> anyhow::Result { +fn call_describe_module(scope: &mut PinScope<'_, '_>) -> anyhow::Result { // Get a cached version of the `__describe_module__` property. let key_cache = get_or_create_key_cache(scope); let describe_module_key = key_cache.borrow_mut().describe_module(scope); @@ -607,7 +608,7 @@ mod test { fn with_script( code: &str, - logic: impl for<'scope> FnOnce(&mut HandleScope<'scope>, Local<'scope, Value>) -> R, + logic: impl for<'scope> FnOnce(&mut PinScope<'scope, '_>, Local<'scope, Value>) -> R, ) -> R { with_scope(|scope| { let code = v8::String::new(scope, code).unwrap(); diff --git a/crates/core/src/host/v8/ser.rs b/crates/core/src/host/v8/ser.rs index c9ed1887f68..51ef8ca073c 100644 --- a/crates/core/src/host/v8/ser.rs +++ b/crates/core/src/host/v8/ser.rs @@ -11,10 +11,10 @@ use spacetimedb_sats::{ ser::{self, Serialize}, u256, }; -use v8::{Array, ArrayBuffer, HandleScope, IntegrityLevel, Local, Object, Uint8Array, Value}; +use v8::{Array, ArrayBuffer, IntegrityLevel, Local, Object, PinScope, Uint8Array, Value}; /// Serializes `value` into a V8 into `scope`. -pub(super) fn serialize_to_js<'scope>(scope: &mut HandleScope<'scope>, value: &impl Serialize) -> FnRet<'scope> { +pub(super) fn serialize_to_js<'scope>(scope: &PinScope<'scope, '_>, value: &impl Serialize) -> FnRet<'scope> { let key_cache = get_or_create_key_cache(scope); let key_cache = &mut *key_cache.borrow_mut(); value @@ -23,20 +23,20 @@ pub(super) fn serialize_to_js<'scope>(scope: &mut HandleScope<'scope>, value: &i } /// Deserializes to V8 values. -struct Serializer<'this, 'scope> { +struct Serializer<'this, 'scope, 'isolate> { /// The scope to serialize values into. - scope: &'this mut HandleScope<'scope>, + scope: &'this PinScope<'scope, 'isolate>, /// A cache for frequently used strings. key_cache: &'this mut KeyCache, } -impl<'this, 'scope> Serializer<'this, 'scope> { +impl<'this, 'scope, 'isolate> Serializer<'this, 'scope, 'isolate> { /// Creates a new serializer into `scope`. - pub fn new(scope: &'this mut HandleScope<'scope>, key_cache: &'this mut KeyCache) -> Self { + pub fn new(scope: &'this PinScope<'scope, 'isolate>, key_cache: &'this mut KeyCache) -> Self { Self { scope, key_cache } } - fn reborrow(&mut self) -> Serializer<'_, 'scope> { + fn reborrow(&mut self) -> Serializer<'_, 'scope, 'isolate> { Serializer { scope: self.scope, key_cache: self.key_cache, @@ -56,7 +56,7 @@ enum Error { } impl<'scope> Throwable<'scope> for Error { - fn throw(self, scope: &mut HandleScope<'scope>) -> ExceptionThrown { + fn throw(self, scope: & PinScope<'scope, '_>) -> ExceptionThrown { match self { Self::StringTooLarge(len) => { RangeError(format!("`{len}` bytes is too large to be a JS string")).throw(scope) @@ -90,20 +90,20 @@ macro_rules! serialize_primitive { /// However, the values of existing properties may be modified, /// which can be useful if the module wants to modify a property /// and then send the object back. -fn seal_object(scope: &mut HandleScope<'_>, object: &Object) -> ExcResult<()> { +fn seal_object(scope: &PinScope<'_, '_>, object: &Object) -> ExcResult<()> { let _ = object .set_integrity_level(scope, IntegrityLevel::Sealed) .ok_or_else(exception_already_thrown)?; Ok(()) } -impl<'this, 'scope> ser::Serializer for Serializer<'this, 'scope> { +impl<'this, 'scope, 'isolate> ser::Serializer for Serializer<'this, 'scope, 'isolate> { type Ok = Local<'scope, Value>; type Error = Error; - type SerializeArray = SerializeArray<'this, 'scope>; + type SerializeArray = SerializeArray<'this, 'scope, 'isolate>; type SerializeSeqProduct = Self::SerializeNamedProduct; - type SerializeNamedProduct = SerializeNamedProduct<'this, 'scope>; + type SerializeNamedProduct = SerializeNamedProduct<'this, 'scope, 'isolate>; // Serialization of primitive types defers to `ToValue`. serialize_primitive!(serialize_bool, bool); @@ -184,13 +184,13 @@ impl<'this, 'scope> ser::Serializer for Serializer<'this, 'scope> { } /// Serializes array elements and finalizes the JS array. -struct SerializeArray<'this, 'scope> { - inner: Serializer<'this, 'scope>, +struct SerializeArray<'this, 'scope, 'isolate> { + inner: Serializer<'this, 'scope, 'isolate>, array: Local<'scope, Array>, next_index: u32, } -impl<'scope> ser::SerializeArray for SerializeArray<'_, 'scope> { +impl<'scope> ser::SerializeArray for SerializeArray<'_, 'scope, '_> { type Ok = Local<'scope, Value>; type Error = Error; @@ -214,13 +214,13 @@ impl<'scope> ser::SerializeArray for SerializeArray<'_, 'scope> { } /// Serializes into JS objects where field names are turned into property names. -struct SerializeNamedProduct<'this, 'scope> { - inner: Serializer<'this, 'scope>, +struct SerializeNamedProduct<'this, 'scope, 'isolate> { + inner: Serializer<'this, 'scope, 'isolate>, object: Local<'scope, Object>, next_index: usize, } -impl<'scope> ser::SerializeSeqProduct for SerializeNamedProduct<'_, 'scope> { +impl<'scope> ser::SerializeSeqProduct for SerializeNamedProduct<'_, 'scope, '_> { type Ok = Local<'scope, Value>; type Error = Error; @@ -233,7 +233,7 @@ impl<'scope> ser::SerializeSeqProduct for SerializeNamedProduct<'_, 'scope> { } } -impl<'scope> ser::SerializeNamedProduct for SerializeNamedProduct<'_, 'scope> { +impl<'scope> ser::SerializeNamedProduct for SerializeNamedProduct<'_, 'scope, '_> { type Ok = Local<'scope, Value>; type Error = Error; @@ -246,7 +246,7 @@ impl<'scope> ser::SerializeNamedProduct for SerializeNamedProduct<'_, 'scope> { let value = elem.serialize(self.inner.reborrow())?; // Figure out the object property to use. - let scope = &mut *self.inner.scope; + let scope = self.inner.scope; let index = self.next_index; self.next_index += 1; let key = intern_field_name(scope, field_name, index).into(); diff --git a/crates/core/src/host/v8/syscall.rs b/crates/core/src/host/v8/syscall.rs index cf053045067..80aff12aba7 100644 --- a/crates/core/src/host/v8/syscall.rs +++ b/crates/core/src/host/v8/syscall.rs @@ -13,10 +13,10 @@ use crate::host::AbiCall; use spacetimedb_lib::Identity; use spacetimedb_primitives::{errno, ColId, IndexId, TableId}; use spacetimedb_sats::Serialize; -use v8::{Function, FunctionCallbackArguments, HandleScope, Local, Value}; +use v8::{Function, FunctionCallbackArguments, Local, PinScope, Value}; /// Registers all module -> host syscalls. -pub(super) fn register_host_funs(scope: &mut HandleScope<'_>) -> ExcResult<()> { +pub(super) fn register_host_funs(scope: &mut PinScope<'_, '_>) -> ExcResult<()> { macro_rules! register { ($wrapper:ident, $abi_call:expr, $fun:ident) => { register_host_fun(scope, stringify!($fun), |s, a| $wrapper($abi_call, s, a, $fun))?; @@ -72,9 +72,9 @@ pub(super) type FnRet<'scope> = ExcResult>; /// Registers a module -> host syscall in `scope` /// where the function has `name` and `body` fn register_host_fun( - scope: &mut HandleScope<'_>, + scope: &mut PinScope<'_, '_>, name: &str, - body: impl Copy + for<'scope> Fn(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, + body: impl Copy + for<'scope> Fn(&mut PinScope<'scope, '_>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, ) -> ExcResult<()> { let name = v8_interned_string(scope, name).into(); let fun = Function::new(scope, adapt_fun(body)) @@ -93,8 +93,8 @@ struct TerminationFlag; /// Adapts `fun`, which returns a [`Value`] to one that works on [`v8::ReturnValue`]. fn adapt_fun( - fun: impl Copy + for<'scope> Fn(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, -) -> impl Copy + for<'scope> Fn(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>, v8::ReturnValue) { + fun: impl Copy + for<'scope> Fn(&mut PinScope<'scope, '_>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, +) -> impl Copy + for<'scope> Fn(&mut PinScope<'scope, '_>, FunctionCallbackArguments<'scope>, v8::ReturnValue) { move |scope, args, mut rv| { // If the flag was set in `handle_nodes_error`, // we need to block all module -> host ABI calls. @@ -126,9 +126,9 @@ type SysCallResult = Result; /// Handles [`SysCallError`] if it occurs by throwing exceptions into JS. fn with_sys_result<'scope, O: Serialize>( abi_call: AbiCall, - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, - run: impl FnOnce(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> SysCallResult, + run: impl FnOnce(&mut PinScope<'scope, '_>, FunctionCallbackArguments<'scope>) -> SysCallResult, ) -> FnRet<'scope> { match with_span(abi_call, scope, args, run) { Ok(ret) => serialize_to_js(scope, &ret), @@ -138,7 +138,7 @@ fn with_sys_result<'scope, O: Serialize>( } /// Turns a [`NodesError`] into a thrown exception. -fn throw_nodes_error(abi_call: AbiCall, scope: &mut HandleScope<'_>, error: NodesError) -> ExceptionThrown { +fn throw_nodes_error(abi_call: AbiCall, scope: &mut PinScope<'_, '_>, error: NodesError) -> ExceptionThrown { let res = match err_to_errno_and_log::(abi_call, error) { Ok(code) => CodeError::from_code(scope, code), Err(err) => { @@ -157,7 +157,7 @@ fn throw_nodes_error(abi_call: AbiCall, scope: &mut HandleScope<'_>, error: Node /// Collapses `res` where the `Ok(x)` where `x` is throwable. fn collapse_exc_thrown<'scope>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, res: ExcResult>, ) -> ExceptionThrown { let (Ok(thrown) | Err(thrown)) = res.map(|ev| ev.throw(scope)); @@ -167,9 +167,9 @@ fn collapse_exc_thrown<'scope>( /// Tracks the span of `body` under the label `abi_call`. fn with_span<'scope, R>( abi_call: AbiCall, - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, - body: impl FnOnce(&mut HandleScope<'scope>, FunctionCallbackArguments<'scope>) -> R, + body: impl FnOnce(&mut PinScope<'scope, '_>, FunctionCallbackArguments<'scope>) -> R, ) -> R { // Start the span. let span_start = span::CallSpanStart::new(abi_call); @@ -215,7 +215,7 @@ fn with_span<'scope, R>( /// /// Throws a `TypeError` if: /// - `name` is not `string`. -fn table_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { +fn table_id_from_name(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { let name: &str = deserialize_js(scope, args.get(0))?; Ok(env_on_isolate(scope).instance_env.table_id_from_name(name)?) } @@ -251,7 +251,7 @@ fn table_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArgumen /// /// Throws a `TypeError`: /// - if `name` is not `string`. -fn index_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { +fn index_id_from_name(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { let name: &str = deserialize_js(scope, args.get(0))?; Ok(env_on_isolate(scope).instance_env.index_id_from_name(name)?) } @@ -288,7 +288,7 @@ fn index_id_from_name(scope: &mut HandleScope<'_>, args: FunctionCallbackArgumen /// /// Throws a `TypeError` if: /// - `table_id` is not a `u32`. -fn datastore_table_row_count(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { +fn datastore_table_row_count(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { let table_id: TableId = deserialize_js(scope, args.get(0))?; Ok(env_on_isolate(scope).instance_env.datastore_table_row_count(table_id)?) } @@ -327,7 +327,7 @@ fn datastore_table_row_count(scope: &mut HandleScope<'_>, args: FunctionCallback /// /// Throws a `TypeError`: /// - if `table_id` is not a `u32`. -fn datastore_table_scan_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { +fn datastore_table_scan_bsatn(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult { let table_id: TableId = deserialize_js(scope, args.get(0))?; let env = env_on_isolate(scope); @@ -426,7 +426,7 @@ fn datastore_table_scan_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbac /// - `prefix`, `rstart`, and `rend` are not arrays of `u8`s. /// - `prefix_elems` is not a `u16`. fn datastore_index_scan_range_bsatn( - scope: &mut HandleScope<'_>, + scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>, ) -> SysCallResult { let index_id: IndexId = deserialize_js(scope, args.get(0))?; @@ -456,7 +456,7 @@ fn datastore_index_scan_range_bsatn( } /// Throws `{ __code_error__: NO_SUCH_ITER }`. -fn no_such_iter(scope: &mut HandleScope<'_>) -> ExceptionThrown { +fn no_such_iter(scope: &PinScope<'_, '_>) -> ExceptionThrown { let res = CodeError::from_code(scope, errno::NO_SUCH_ITER.get()); collapse_exc_thrown(scope, res) } @@ -511,7 +511,7 @@ fn no_such_iter(scope: &mut HandleScope<'_>) -> ExceptionThrown { /// Throws a `TypeError` if: /// - `iter` and `buffer_max_len` are not `u32`s. fn row_iter_bsatn_advance<'scope>( - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, ) -> SysCallResult<(bool, Vec)> { let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; @@ -578,7 +578,7 @@ fn row_iter_bsatn_advance<'scope>( /// Throws a `TypeError` if: /// - `iter` is not a `u32`. fn row_iter_bsatn_close<'scope>( - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, ) -> FnRet<'scope> { let row_iter_idx: u32 = deserialize_js(scope, args.get(0))?; @@ -654,7 +654,7 @@ fn row_iter_bsatn_close<'scope>( /// Throws a `TypeError` if: /// - `table_id` is not a `u32`. /// - `row` is not an array of `u8`s. -fn datastore_insert_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult> { +fn datastore_insert_bsatn(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult> { let table_id: TableId = deserialize_js(scope, args.get(0))?; let mut row: Vec = deserialize_js(scope, args.get(1))?; @@ -737,7 +737,7 @@ fn datastore_insert_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArg /// Throws a `TypeError` if: /// - `table_id` is not a `u32`. /// - `row` is not an array of `u8`s. -fn datastore_update_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArguments<'_>) -> SysCallResult> { +fn datastore_update_bsatn(scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>) -> SysCallResult> { let table_id: TableId = deserialize_js(scope, args.get(0))?; let index_id: IndexId = deserialize_js(scope, args.get(1))?; let mut row: Vec = deserialize_js(scope, args.get(2))?; @@ -805,7 +805,7 @@ fn datastore_update_bsatn(scope: &mut HandleScope<'_>, args: FunctionCallbackArg /// - `prefix`, `rstart`, and `rend` are not arrays of `u8`s. /// - `prefix_elems` is not a `u16`. fn datastore_delete_by_index_scan_range_bsatn( - scope: &mut HandleScope<'_>, + scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>, ) -> SysCallResult { let index_id: IndexId = deserialize_js(scope, args.get(0))?; @@ -872,7 +872,7 @@ fn datastore_delete_by_index_scan_range_bsatn( /// - `table_id` is not a `u32`. /// - `relation` is not an array of `u8`s. fn datastore_delete_all_by_eq_bsatn( - scope: &mut HandleScope<'_>, + scope: &mut PinScope<'_, '_>, args: FunctionCallbackArguments<'_>, ) -> SysCallResult { let table_id: TableId = deserialize_js(scope, args.get(0))?; @@ -888,7 +888,7 @@ fn datastore_delete_all_by_eq_bsatn( /// volatile_nonatomic_schedule_immediate(reducer_name: string, args: u8[]) -> undefined /// ``` fn volatile_nonatomic_schedule_immediate<'scope>( - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, ) -> FnRet<'scope> { let name: String = deserialize_js(scope, args.get(0))?; @@ -921,7 +921,7 @@ fn volatile_nonatomic_schedule_immediate<'scope>( /// # Returns /// /// Returns nothing. -fn console_log<'scope>(scope: &mut HandleScope<'scope>, args: FunctionCallbackArguments<'scope>) -> FnRet<'scope> { +fn console_log<'scope>(scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>) -> FnRet<'scope> { let level: u32 = deserialize_js(scope, args.get(0))?; let msg = args.get(1).cast::(); @@ -989,7 +989,7 @@ fn console_log<'scope>(scope: &mut HandleScope<'scope>, args: FunctionCallbackAr /// Throws a `TypeError` if: /// - `name` is not a `string`. fn console_timer_start<'scope>( - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, ) -> FnRet<'scope> { let name = args.get(0).cast::(); @@ -1029,7 +1029,7 @@ fn console_timer_start<'scope>( /// Throws a `TypeError` if: /// - `span_id` is not a `u32`. fn console_timer_end<'scope>( - scope: &mut HandleScope<'scope>, + scope: &mut PinScope<'scope, '_>, args: FunctionCallbackArguments<'scope>, ) -> FnRet<'scope> { let span_id: u32 = deserialize_js(scope, args.get(0))?; @@ -1060,6 +1060,6 @@ fn console_timer_end<'scope>( /// # Returns /// /// Returns the module identity. -fn identity<'scope>(scope: &mut HandleScope<'scope>, _: FunctionCallbackArguments<'scope>) -> SysCallResult { +fn identity<'scope>(scope: &mut PinScope<'scope, '_>, _: FunctionCallbackArguments<'scope>) -> SysCallResult { Ok(*env_on_isolate(scope).instance_env.database_identity()) } diff --git a/crates/core/src/host/v8/to_value.rs b/crates/core/src/host/v8/to_value.rs index e97dd491eba..9fecfa5a090 100644 --- a/crates/core/src/host/v8/to_value.rs +++ b/crates/core/src/host/v8/to_value.rs @@ -2,20 +2,20 @@ use bytemuck::{NoUninit, Pod}; use spacetimedb_sats::{i256, u256}; -use v8::{BigInt, Boolean, HandleScope, Integer, Local, Number, Value}; +use v8::{BigInt, Boolean, Integer, Local, Number, PinScope, Value}; /// Types that can be converted to a v8-stack-allocated [`Value`]. /// The conversion can be done without the possibility for error. pub(super) trait ToValue { /// Converts `self` within `scope` (a sort of stack management in V8) to a [`Value`]. - fn to_value<'scope>(&self, scope: &mut HandleScope<'scope>) -> Local<'scope, Value>; + fn to_value<'scope>(&self, scope: &PinScope<'scope, '_>) -> Local<'scope, Value>; } /// Provides a [`ToValue`] implementation. macro_rules! impl_to_value { ($ty:ty, ($val:ident, $scope:ident) => $logic:expr) => { impl ToValue for $ty { - fn to_value<'scope>(&self, $scope: &mut HandleScope<'scope>) -> Local<'scope, Value> { + fn to_value<'scope>(&self, $scope: &PinScope<'scope, '_>) -> Local<'scope, Value> { let $val = *self; $logic.into() } @@ -48,7 +48,7 @@ impl_to_value!(u64, (val, scope) => BigInt::new_from_u64(scope, val)); /// /// The `sign` is passed along to the `BigInt`. fn le_bytes_to_bigint<'scope, const N: usize, const W: usize>( - scope: &mut HandleScope<'scope>, + scope: &PinScope<'scope, '_>, sign: bool, le_bytes: [u8; N], ) -> Local<'scope, BigInt> @@ -73,7 +73,7 @@ pub(super) const WORD_MIN: u64 = i64::MIN as u64; /// `i64::MIN` becomes `-1 * WORD_MIN * (2^64)^0 = -1 * WORD_MIN` /// `i128::MIN` becomes `-1 * (0 * (2^64)^0 + WORD_MIN * (2^64)^1) = -1 * WORD_MIN * 2^64` /// `i256::MIN` becomes `-1 * (0 * (2^64)^0 + 0 * (2^64)^1 + WORD_MIN * (2^64)^2) = -1 * WORD_MIN * (2^128)` -fn signed_min_bigint<'scope, const WORDS: usize>(scope: &mut HandleScope<'scope>) -> Local<'scope, BigInt> { +fn signed_min_bigint<'scope, const WORDS: usize>(scope: &PinScope<'scope, '_>) -> Local<'scope, BigInt> { let words = &mut [0u64; WORDS]; if let [.., last] = words.as_mut_slice() { *last = WORD_MIN; @@ -110,14 +110,14 @@ pub(in super::super) mod test { use core::fmt::Debug; use proptest::prelude::*; use spacetimedb_sats::proptest::{any_i256, any_u256}; - use v8::{Context, ContextScope, HandleScope, Isolate}; + use v8::{scope, Context, ContextScope, Isolate}; /// Sets up V8 and runs `logic` with a [`HandleScope`]. - pub(in super::super) fn with_scope(logic: impl FnOnce(&mut HandleScope<'_>) -> R) -> R { + pub(in super::super) fn with_scope(logic: impl FnOnce(&mut PinScope<'_, '_>) -> R) -> R { V8Runtime::init_for_test(); let isolate = &mut Isolate::new(<_>::default()); isolate.set_capture_stack_trace_for_uncaught_exceptions(true, 1024); - let scope = &mut HandleScope::new(isolate); + scope!(let scope, isolate); let context = Context::new(scope, Default::default()); let scope = &mut ContextScope::new(scope, context); From 2c6aa82d4101b7f7276ddfb61e51db8a09daca85 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 26 Sep 2025 15:06:20 +0200 Subject: [PATCH 17/39] v8: use fast static strings for known strings --- crates/core/src/host/v8/de.rs | 21 +++----- crates/core/src/host/v8/from_value.rs | 5 +- crates/core/src/host/v8/key_cache.rs | 67 ------------------------- crates/core/src/host/v8/mod.rs | 24 ++++----- crates/core/src/host/v8/ser.rs | 37 ++++---------- crates/core/src/host/v8/string_const.rs | 33 ++++++++++++ crates/core/src/host/v8/syscall.rs | 20 +++++--- 7 files changed, 75 insertions(+), 132 deletions(-) delete mode 100644 crates/core/src/host/v8/key_cache.rs create mode 100644 crates/core/src/host/v8/string_const.rs diff --git a/crates/core/src/host/v8/de.rs b/crates/core/src/host/v8/de.rs index 0eed5810c4f..3faf6099bfb 100644 --- a/crates/core/src/host/v8/de.rs +++ b/crates/core/src/host/v8/de.rs @@ -2,7 +2,7 @@ use super::error::{exception_already_thrown, ExcResult, ExceptionThrown, ExceptionValue, Throwable, TypeError}; use super::from_value::{cast, FromValue}; -use super::key_cache::{get_or_create_key_cache, KeyCache}; +use super::string_const::{TAG, VALUE}; use core::fmt; use core::iter::{repeat_n, RepeatN}; use core::marker::PhantomData; @@ -19,9 +19,7 @@ pub(super) fn deserialize_js_seed<'de, T: DeserializeSeed<'de>>( val: Local<'_, Value>, seed: T, ) -> ExcResult { - let key_cache = get_or_create_key_cache(scope); - let key_cache = &mut *key_cache.borrow_mut(); - let de = Deserializer::new(scope, val, key_cache); + let de = Deserializer::new(scope, val); seed.deserialize(de).map_err(|e| e.throw(scope)) } @@ -41,9 +39,9 @@ struct Deserializer<'this, 'scope, 'isolate> { impl<'this, 'scope, 'isolate> Deserializer<'this, 'scope, 'isolate> { /// Creates a new deserializer from `input` in `scope`. - fn new(scope: &'this mut PinScope<'scope, 'isolate>, input: Local<'_, Value>, key_cache: &'this mut KeyCache) -> Self { + fn new(scope: &'this mut PinScope<'scope, 'isolate>, input: Local<'_, Value>) -> Self { let input = Local::new(scope, input); - let common = DeserializerCommon { scope, key_cache }; + let common = DeserializerCommon { scope }; Deserializer { input, common } } } @@ -54,16 +52,11 @@ impl<'this, 'scope, 'isolate> Deserializer<'this, 'scope, 'isolate> { struct DeserializerCommon<'this, 'scope, 'isolate> { /// The scope of values to deserialize. scope: &'this mut PinScope<'scope, 'isolate>, - /// A cache for frequently used strings. - key_cache: &'this mut KeyCache, } impl<'scope, 'isolate> DeserializerCommon<'_, 'scope, 'isolate> { fn reborrow(&mut self) -> DeserializerCommon<'_, 'scope, 'isolate> { - DeserializerCommon { - scope: self.scope, - key_cache: self.key_cache, - } + DeserializerCommon { scope: self.scope } } } @@ -161,7 +154,7 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco // We expect a canonical representation of a sum value in JS to be // `{ tag: "foo", value: a_value_for_foo }`. - let tag_field = self.common.key_cache.tag(scope); + let tag_field = TAG.string(scope); let object = cast!(scope, self.input, Object, "object for sum type `{}`", sum_name)?; // Extract the `tag` field. It needs to contain a string. @@ -171,7 +164,7 @@ impl<'de, 'this, 'scope: 'de> de::Deserializer<'de> for Deserializer<'this, 'sco let tag = cast!(scope, tag, v8::String, "string for sum tag of `{}`", sum_name)?; // Extract the `value` field. - let value_field = self.common.key_cache.value(scope); + let value_field = VALUE.string(scope); let value = object .get(scope, value_field.into()) .ok_or_else(exception_already_thrown)?; diff --git a/crates/core/src/host/v8/from_value.rs b/crates/core/src/host/v8/from_value.rs index 61f70676c78..114345434ee 100644 --- a/crates/core/src/host/v8/from_value.rs +++ b/crates/core/src/host/v8/from_value.rs @@ -17,10 +17,7 @@ pub(super) trait FromValue: Sized { macro_rules! impl_from_value { ($ty:ty, ($val:ident, $scope:ident) => $logic:expr) => { impl FromValue for $ty { - fn from_value<'scope>( - $val: Local<'_, Value>, - $scope: &PinScope<'scope, '_>, - ) -> ValueResult<'scope, Self> { + fn from_value<'scope>($val: Local<'_, Value>, $scope: &PinScope<'scope, '_>) -> ValueResult<'scope, Self> { $logic } } diff --git a/crates/core/src/host/v8/key_cache.rs b/crates/core/src/host/v8/key_cache.rs deleted file mode 100644 index 27c76d7f60e..00000000000 --- a/crates/core/src/host/v8/key_cache.rs +++ /dev/null @@ -1,67 +0,0 @@ -use super::de::v8_interned_string; -use core::cell::RefCell; -use std::rc::Rc; -use v8::{Global, Local, PinScope}; - -/// Returns a `KeyCache` for the current `scope`. -/// -/// Creates the cache in the scope if it doesn't exist yet. -pub(super) fn get_or_create_key_cache(scope: &PinScope<'_, '_>) -> Rc> { - let context = scope.get_current_context(); - context.get_slot::>().unwrap_or_else(|| { - let cache = Rc::default(); - context.set_slot(Rc::clone(&cache)); - cache - }) -} - -/// A cache for frequently used strings to avoid re-interning them. -#[derive(Default)] -pub(super) struct KeyCache { - /// The `tag` property for sum values in JS. - tag: Option>, - /// The `value` property for sum values in JS. - value: Option>, - /// The `__describe_module__` property on the global proxy object. - describe_module: Option>, - /// The `__call_reducer__` property on the global proxy object. - call_reducer: Option>, -} - -impl KeyCache { - /// Returns the `tag` property name. - pub(super) fn tag<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { - Self::get_or_create_key(scope, &mut self.tag, "tag") - } - - /// Returns the `value` property name. - pub(super) fn value<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { - Self::get_or_create_key(scope, &mut self.value, "value") - } - - /// Returns the `__describe_module__` property name. - pub(super) fn describe_module<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { - Self::get_or_create_key(scope, &mut self.describe_module, "__describe_module__") - } - - /// Returns the `__call_reducer__` property name. - pub(super) fn call_reducer<'scope>(&mut self, scope: &PinScope<'scope, '_>) -> Local<'scope, v8::String> { - Self::get_or_create_key(scope, &mut self.call_reducer, "__call_reducer__") - } - - /// Returns an interned string corresponding to `string` - /// and memoizes the creation on the v8 side. - fn get_or_create_key<'scope>( - scope: &PinScope<'scope, '_>, - slot: &mut Option>, - string: &str, - ) -> Local<'scope, v8::String> { - if let Some(s) = &*slot { - v8::Local::new(scope, s) - } else { - let s = v8_interned_string(scope, string); - *slot = Some(v8::Global::new(scope, s)); - s - } - } -} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 9345555d288..36c32432d2e 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -4,24 +4,24 @@ use super::module_common::{build_common_module_from_raw, run_describer, ModuleCo use super::module_host::{CallReducerParams, DynModule, Module, ModuleInfo, ModuleInstance, ModuleRuntime}; use super::UpdateDatabaseResult; use crate::host::instance_env::{ChunkPool, InstanceEnv}; -use crate::host::v8::error::{BufferTooSmall, JsStackTrace}; -use crate::host::v8::syscall::{register_host_funs, FnRet}; use crate::host::wasm_common::instrumentation::CallTimes; use crate::host::wasm_common::module_host_actor::{ DescribeError, EnergyStats, ExecuteResult, ExecutionTimings, InstanceCommon, ReducerOp, }; use crate::host::wasm_common::{RowIters, TimingSpanSet}; use crate::host::wasmtime::{epoch_ticker, ticks_in_duration, EPOCH_TICKS_PER_SECOND}; -use crate::host::ArgsTuple; -use crate::{host::Scheduler, module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; +use crate::host::{ArgsTuple, Scheduler}; +use crate::{module_host_context::ModuleCreationContext, replica_context::ReplicaContext}; use core::ffi::c_void; use core::sync::atomic::{AtomicBool, Ordering}; use core::time::Duration; use core::{iter, ptr, str}; use de::deserialize_js; -use error::{catch_exception, exception_already_thrown, log_traceback, CodeError, TerminationError, Throwable}; +use error::{ + catch_exception, exception_already_thrown, log_traceback, BufferTooSmall, CodeError, JsStackTrace, + TerminationError, Throwable, +}; use from_value::cast; -use key_cache::get_or_create_key_cache; use ser::serialize_to_js; use spacetimedb_client_api_messages::energy::ReducerBudget; use spacetimedb_datastore::locking_tx_datastore::MutTxId; @@ -30,6 +30,8 @@ use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; use spacetimedb_schema::auto_migrate::MigrationPolicy; use std::sync::{Arc, LazyLock}; use std::time::Instant; +use string_const::str_from_ident; +use syscall::{register_host_funs, FnRet}; use v8::{ scope, Context, ContextScope, Function, Isolate, IsolateHandle, Local, Object, OwnedIsolate, PinScope, Value, }; @@ -37,8 +39,8 @@ use v8::{ mod de; mod error; mod from_value; -mod key_cache; mod ser; +mod string_const; mod syscall; mod to_value; @@ -532,9 +534,7 @@ fn call_call_reducer( timestamp: i64, reducer_args: &ArgsTuple, ) -> anyhow::Result>> { - // Get a cached version of the `__call_reducer__` property. - let key_cache = get_or_create_key_cache(scope); - let call_reducer_key = key_cache.borrow_mut().call_reducer(scope); + let call_reducer_key = str_from_ident!(__call_reducer__).string(scope); catch_exception(scope, |scope| { // Serialize the arguments. @@ -580,9 +580,7 @@ fn extract_description(program: &str) -> Result { // Calls the `__describe_module__` function on the global proxy object to extract a [`RawModuleDef`]. fn call_describe_module(scope: &mut PinScope<'_, '_>) -> anyhow::Result { - // Get a cached version of the `__describe_module__` property. - let key_cache = get_or_create_key_cache(scope); - let describe_module_key = key_cache.borrow_mut().describe_module(scope); + let describe_module_key = str_from_ident!(__describe_module__).string(scope); catch_exception(scope, |scope| { // Get the function on the global proxy object and convert to a function. diff --git a/crates/core/src/host/v8/ser.rs b/crates/core/src/host/v8/ser.rs index 51ef8ca073c..dbbbb442e7d 100644 --- a/crates/core/src/host/v8/ser.rs +++ b/crates/core/src/host/v8/ser.rs @@ -2,7 +2,7 @@ use super::de::intern_field_name; use super::error::{exception_already_thrown, ExcResult, ExceptionThrown, RangeError, Throwable, TypeError}; -use super::key_cache::{get_or_create_key_cache, KeyCache}; +use super::string_const::{TAG, VALUE}; use super::syscall::FnRet; use super::to_value::ToValue; use derive_more::From; @@ -15,32 +15,20 @@ use v8::{Array, ArrayBuffer, IntegrityLevel, Local, Object, PinScope, Uint8Array /// Serializes `value` into a V8 into `scope`. pub(super) fn serialize_to_js<'scope>(scope: &PinScope<'scope, '_>, value: &impl Serialize) -> FnRet<'scope> { - let key_cache = get_or_create_key_cache(scope); - let key_cache = &mut *key_cache.borrow_mut(); - value - .serialize(Serializer::new(scope, key_cache)) - .map_err(|e| e.throw(scope)) + value.serialize(Serializer::new(scope)).map_err(|e| e.throw(scope)) } /// Deserializes to V8 values. +#[derive(Copy, Clone)] struct Serializer<'this, 'scope, 'isolate> { /// The scope to serialize values into. scope: &'this PinScope<'scope, 'isolate>, - /// A cache for frequently used strings. - key_cache: &'this mut KeyCache, } impl<'this, 'scope, 'isolate> Serializer<'this, 'scope, 'isolate> { /// Creates a new serializer into `scope`. - pub fn new(scope: &'this PinScope<'scope, 'isolate>, key_cache: &'this mut KeyCache) -> Self { - Self { scope, key_cache } - } - - fn reborrow(&mut self) -> Serializer<'_, 'scope, 'isolate> { - Serializer { - scope: self.scope, - key_cache: self.key_cache, - } + pub fn new(scope: &'this PinScope<'scope, 'isolate>) -> Self { + Self { scope } } } @@ -56,7 +44,7 @@ enum Error { } impl<'scope> Throwable<'scope> for Error { - fn throw(self, scope: & PinScope<'scope, '_>) -> ExceptionThrown { + fn throw(self, scope: &PinScope<'scope, '_>) -> ExceptionThrown { match self { Self::StringTooLarge(len) => { RangeError(format!("`{len}` bytes is too large to be a JS string")).throw(scope) @@ -158,22 +146,19 @@ impl<'this, 'scope, 'isolate> ser::Serializer for Serializer<'this, 'scope, 'iso } fn serialize_variant( - mut self, + self, tag: u8, var_name: Option<&str>, value: &T, ) -> Result { // Serialize the payload. - let value_value: Local<'scope, Value> = value.serialize(self.reborrow())?; + let value_value: Local<'scope, Value> = value.serialize(self)?; // Figure out the tag. let tag_value: Local<'scope, Value> = intern_field_name(self.scope, var_name, tag as usize).into(); let values = [tag_value, value_value]; // The property keys are always `"tag"` an `"value"`. - let names = [ - self.key_cache.tag(self.scope).into(), - self.key_cache.value(self.scope).into(), - ]; + let names = [TAG.string(self.scope).into(), VALUE.string(self.scope).into()]; // Stitch together the object. let prototype = v8::null(self.scope).into(); @@ -196,7 +181,7 @@ impl<'scope> ser::SerializeArray for SerializeArray<'_, 'scope, '_> { fn serialize_element(&mut self, elem: &T) -> Result<(), Self::Error> { // Serialize the current `elem`ent. - let value = elem.serialize(self.inner.reborrow())?; + let value = elem.serialize(self.inner)?; // Set the value to the array slot at `index`. let index = self.next_index; @@ -243,7 +228,7 @@ impl<'scope> ser::SerializeNamedProduct for SerializeNamedProduct<'_, 'scope, '_ elem: &T, ) -> Result<(), Self::Error> { // Serialize the field value. - let value = elem.serialize(self.inner.reborrow())?; + let value = elem.serialize(self.inner)?; // Figure out the object property to use. let scope = self.inner.scope; diff --git a/crates/core/src/host/v8/string_const.rs b/crates/core/src/host/v8/string_const.rs new file mode 100644 index 00000000000..8dc5ac6ee3f --- /dev/null +++ b/crates/core/src/host/v8/string_const.rs @@ -0,0 +1,33 @@ +use v8::{Local, OneByteConst, PinScope, String}; + +/// A string known at compile time to be ASCII. +pub(super) struct StringConst(OneByteConst); + +impl StringConst { + /// Returns a new string that is known to be ASCII and static. + pub(super) const fn new(string: &'static str) -> Self { + Self(String::create_external_onebyte_const(string.as_bytes())) + } + + /// Converts the string to a V8 string. + pub(super) fn string<'scope>(&'static self, scope: &PinScope<'scope, '_>) -> Local<'scope, String> { + String::new_from_onebyte_const(scope, &self.0) + .expect("`create_external_onebyte_const` should've asserted `.len() < kMaxLength`") + } +} + +/// Converts an identifier to a compile-time ASCII string. +#[macro_export] +macro_rules! str_from_ident { + ($ident:ident) => {{ + const STR: &$crate::host::v8::string_const::StringConst = + &$crate::host::v8::string_const::StringConst::new(stringify!($ident)); + STR + }}; +} +pub(super) use str_from_ident; + +/// The `tag` property of a sum object in JS. +pub(super) const TAG: &StringConst = str_from_ident!(tag); +/// The `value` property of a sum object in JS. +pub(super) const VALUE: &StringConst = str_from_ident!(value); diff --git a/crates/core/src/host/v8/syscall.rs b/crates/core/src/host/v8/syscall.rs index 80aff12aba7..84f90c414d9 100644 --- a/crates/core/src/host/v8/syscall.rs +++ b/crates/core/src/host/v8/syscall.rs @@ -1,12 +1,14 @@ -use super::de::{deserialize_js, scratch_buf, v8_interned_string}; -use super::error::ExcResult; +use super::de::{deserialize_js, scratch_buf}; +use super::error::{ExcResult, ExceptionThrown}; use super::ser::serialize_to_js; -use super::{env_on_isolate, exception_already_thrown}; +use super::string_const::{str_from_ident, StringConst}; +use super::{ + env_on_isolate, exception_already_thrown, global, BufferTooSmall, CodeError, JsInstanceEnv, JsStackTrace, + TerminationError, Throwable, +}; use crate::database_logger::{LogLevel, Record}; use crate::error::NodesError; use crate::host::instance_env::InstanceEnv; -use crate::host::v8::error::ExceptionThrown; -use crate::host::v8::{global, BufferTooSmall, CodeError, JsInstanceEnv, JsStackTrace, TerminationError, Throwable}; use crate::host::wasm_common::instrumentation::span; use crate::host::wasm_common::{err_to_errno_and_log, RowIterIdx, TimingSpan, TimingSpanIdx}; use crate::host::AbiCall; @@ -19,7 +21,9 @@ use v8::{Function, FunctionCallbackArguments, Local, PinScope, Value}; pub(super) fn register_host_funs(scope: &mut PinScope<'_, '_>) -> ExcResult<()> { macro_rules! register { ($wrapper:ident, $abi_call:expr, $fun:ident) => { - register_host_fun(scope, stringify!($fun), |s, a| $wrapper($abi_call, s, a, $fun))?; + register_host_fun(scope, str_from_ident!($fun), |s, a| { + $wrapper($abi_call, s, a, $fun) + })?; }; } @@ -73,10 +77,10 @@ pub(super) type FnRet<'scope> = ExcResult>; /// where the function has `name` and `body` fn register_host_fun( scope: &mut PinScope<'_, '_>, - name: &str, + name: &'static StringConst, body: impl Copy + for<'scope> Fn(&mut PinScope<'scope, '_>, FunctionCallbackArguments<'scope>) -> FnRet<'scope>, ) -> ExcResult<()> { - let name = v8_interned_string(scope, name).into(); + let name = name.string(scope).into(); let fun = Function::new(scope, adapt_fun(body)) .ok_or_else(exception_already_thrown)? .into(); From 97109474755a995c28e6cc7c7a4847ab0ebf778e Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 1 Oct 2025 11:07:58 +0200 Subject: [PATCH 18/39] cargo fmt --- crates/core/src/host/wasmtime/wasm_instance_env.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index f0648bebd5b..c41c4737034 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -1,6 +1,5 @@ #![allow(clippy::too_many_arguments)] -use std::num::NonZeroU32; use super::{Mem, MemView, NullableMemOp, WasmError, WasmPointee, WasmPtr}; use crate::database_logger::{BacktraceFrame, BacktraceProvider, ModuleBacktrace, Record}; use crate::host::instance_env::{ChunkPool, InstanceEnv}; @@ -12,6 +11,7 @@ use anyhow::Context as _; use spacetimedb_data_structures::map::IntMap; use spacetimedb_lib::Timestamp; use spacetimedb_primitives::{errno, ColId}; +use std::num::NonZeroU32; use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; From 5898e02f643bb04b00d44eb97524552f929e4032 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Wed, 1 Oct 2025 12:49:02 +0200 Subject: [PATCH 19/39] try to pin timezone_provider for v8 --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 446a3e35c9e..e14fd5a2ddf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -268,6 +268,7 @@ tempfile = "3.20" termcolor = "1.2.0" thin-vec = "0.2.13" thiserror = "1.0.37" +timezone_provider = "=0.0.14" # v8 becomes upset with anything below or above. tokio = { version = "1.37", features = ["full"] } tokio_metrics = { version = "0.4.0" } tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] } From 15c0871e39385d491cc4d9f107bc636aebb23b51 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Thu, 2 Oct 2025 17:57:41 +0200 Subject: [PATCH 20/39] publish: fix rebase fallout for host_type --- crates/cli/src/subcommands/publish.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index b1e5f5689e0..f05cb15eeae 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -134,11 +134,6 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E }; let program_bytes = fs::read(path_to_program)?; - // The host type is not the default (WASM). - if let Some(host_type) = host_type { - builder = builder.query(&[("host_type", host_type)]); - } - let server_address = { let url = Url::parse(&database_host)?; url.host_str().unwrap_or("").to_string() @@ -210,6 +205,11 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E builder = builder.query(&[("num_replicas", *n)]); } + // The host type is not the default (WASM). + if let Some(host_type) = host_type { + builder = builder.query(&[("host_type", host_type)]); + } + println!("Publishing module..."); builder = add_auth_header_opt(builder, &auth_header); From 3abd28e7f319b082b390323bdae5162ba97af04f Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 1 Oct 2025 15:40:39 -0500 Subject: [PATCH 21/39] Fix bindgen tests --- .github/workflows/ci.yml | 5 +++-- Cargo.toml | 1 - 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6898ab6d6c..afe768a5152 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,6 +168,9 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - run: echo ::add-matcher::.github/workflows/rust_matcher.json + - name: Run bindgen tests + run: cargo test -p spacetimedb-codegen + # Make sure the `Cargo.lock` file reflects the latest available versions. # This is what users would end up with on a fresh module, so we want to # catch any compile errors arising from a different transitive closure @@ -180,8 +183,6 @@ jobs: - name: Build module-test run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test - - name: Run bindgen tests - run: cargo test -p spacetimedb-codegen publish_checks: name: Check that packages are publishable diff --git a/Cargo.toml b/Cargo.toml index e14fd5a2ddf..446a3e35c9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -268,7 +268,6 @@ tempfile = "3.20" termcolor = "1.2.0" thin-vec = "0.2.13" thiserror = "1.0.37" -timezone_provider = "=0.0.14" # v8 becomes upset with anything below or above. tokio = { version = "1.37", features = ["full"] } tokio_metrics = { version = "0.4.0" } tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] } From 24b08bf5fcf7692d63405332276e6c48f72f8df7 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Thu, 2 Oct 2025 18:34:22 +0200 Subject: [PATCH 22/39] regen cli-reference.md --- docs/docs/cli-reference.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/cli-reference.md b/docs/docs/cli-reference.md index 66be4ef3aef..11045a959f1 100644 --- a/docs/docs/cli-reference.md +++ b/docs/docs/cli-reference.md @@ -89,6 +89,7 @@ Run `spacetime help publish` for more detailed information. Default value: `.` * `-b`, `--bin-path ` — The system path (absolute or relative) to the compiled wasm binary we should publish, instead of building the project. +* `-j`, `--js-path ` — UNSTABLE: The system path (absolute or relative) to the javascript file we should publish, instead of building the project. * `--break-clients` — Allow breaking changes when publishing to an existing database identity. This will break existing clients. * `--anonymous` — Perform this action with an anonymous identity * `-s`, `--server ` — The nickname, domain name or URL of the server to host the database. From a2d40da84937aa412fd9caf1c22f9a78d1af1ed2 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Thu, 2 Oct 2025 18:41:34 +0200 Subject: [PATCH 23/39] publish: always specify 'host_type' --- crates/cli/src/subcommands/publish.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index f05cb15eeae..bfaa96b17c6 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -124,13 +124,13 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E // Optionally build the program. let (path_to_program, host_type) = if let Some(path) = wasm_file { println!("(WASM) Skipping build. Instead we are publishing {}", path.display()); - (path.clone(), None) + (path.clone(), "Wasm") } else if let Some(path) = js_file { println!("(JS) Skipping build. Instead we are publishing {}", path.display()); - (path.clone(), Some("Js")) + (path.clone(), "Js") } else { let path = build::exec_with_argstring(config.clone(), path_to_project, build_options).await?; - (path, None) + (path, "Wasm") }; let program_bytes = fs::read(path_to_program)?; @@ -205,15 +205,13 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E builder = builder.query(&[("num_replicas", *n)]); } - // The host type is not the default (WASM). - if let Some(host_type) = host_type { - builder = builder.query(&[("host_type", host_type)]); - } - println!("Publishing module..."); builder = add_auth_header_opt(builder, &auth_header); + // Set the host type. + builder = builder.query(&[("host_type", host_type)]); + let res = builder.body(program_bytes).send().await?; if res.status() == StatusCode::UNAUTHORIZED && !anon_identity { // If we're not in the `anon_identity` case, then we have already forced the user to log in above (using `get_auth_header`), so this should be safe to unwrap. From 8876fb9cbd18d83306e7b0ce14e626d5d96ebfaa Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:22:08 -0700 Subject: [PATCH 24/39] [bfops/test-pr]: CI test --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe768a5152..a6138be5086 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: if: runner.os == 'Windows' run: | cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update + echo "Return code is: $?" Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432' cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From 9e55b850f87f68d866c0b4eef7cd7af79f2052b2 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:22:57 -0700 Subject: [PATCH 25/39] [bfops/test-pr]: reduce CI --- .github/workflows/ci.yml | 322 --------------------------------------- 1 file changed, 322 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6138be5086..0f202d44bc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,6 @@ jobs: strategy: matrix: include: - - { runner: spacetimedb-runner, smoketest_args: --docker } - { runner: windows-latest, smoketest_args: --no-build-cli } runner: [ spacetimedb-runner, windows-latest ] runs-on: ${{ matrix.runner }} @@ -72,324 +71,3 @@ jobs: - name: Stop containers (Linux) if: always() && runner.os == 'Linux' run: docker compose down - - test: - name: Test Suite - runs-on: spacetimedb-runner - steps: - - name: Find Git ref - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - PR_NUMBER="${{ github.event.inputs.pr_number || null }}" - if test -n "${PR_NUMBER}"; then - GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" - else - GIT_REF="${{ github.ref }}" - fi - echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - - - uses: dsherret/rust-toolchain-file@v1 - - - uses: actions/setup-dotnet@v3 - with: - global-json-file: global.json - - - name: Create /stdb dir - run: | - sudo mkdir /stdb - sudo chmod 777 /stdb - - - name: Run cargo test - #Note: Unreal tests will be run separately - run: cargo test --all -- --skip unreal - - - name: Check that the test outputs are up-to-date - run: bash tools/check-diff.sh - - - name: Ensure C# autogen bindings are up-to-date - run: | - cargo run -p spacetimedb-codegen --example regen-csharp-moduledef - bash tools/check-diff.sh crates/bindings-csharp - - - name: C# bindings tests - working-directory: crates/bindings-csharp - run: dotnet test -warnaserror - - lints: - name: Lints - runs-on: spacetimedb-runner - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - uses: dsherret/rust-toolchain-file@v1 - - run: echo ::add-matcher::.github/workflows/rust_matcher.json - - - uses: actions/setup-dotnet@v3 - with: - global-json-file: global.json - - - name: Run cargo fmt - run: cargo fmt --all -- --check - - - name: Run cargo clippy - run: cargo clippy --all --tests --benches -- -D warnings - - - name: Run C# formatting check - working-directory: crates/bindings-csharp - run: | - dotnet tool restore - dotnet csharpier --check . - - - name: Run `cargo doc` for bindings crate - # `bindings` is the only crate we care strongly about documenting, - # since we link to its docs.rs from our website. - # We won't pass `--no-deps`, though, - # since we want everything reachable through it to also work. - # This includes `sats` and `lib`. - working-directory: crates/bindings - env: - # Make `cargo doc` exit with error on warnings, most notably broken links - RUSTDOCFLAGS: '--deny warnings' - run: | - cargo doc - - wasm_bindings: - name: Build and test wasm bindings - runs-on: spacetimedb-runner - steps: - - uses: actions/checkout@v3 - - - uses: dsherret/rust-toolchain-file@v1 - - run: echo ::add-matcher::.github/workflows/rust_matcher.json - - - name: Run bindgen tests - run: cargo test -p spacetimedb-codegen - - # Make sure the `Cargo.lock` file reflects the latest available versions. - # This is what users would end up with on a fresh module, so we want to - # catch any compile errors arising from a different transitive closure - # of dependencies than what is in the workspace lock file. - # - # For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - - name: Update dependencies - run: cargo update - - - name: Build module-test - run: cargo run -p spacetimedb-cli -- build --project-path modules/module-test - - - publish_checks: - name: Check that packages are publishable - runs-on: ubuntu-latest - permissions: read-all - steps: - - uses: actions/checkout@v3 - - name: Set up Python env - run: | - test -d venv || python3 -m venv venv - venv/bin/pip3 install argparse toml - - name: Run checks - run: | - FAILED=0 - # This definition of ROOTS and invocation of find-publish-list.py is copied from publish-crates.sh - ROOTS=(bindings sdk cli standalone) - for crate in $(venv/bin/python3 tools/find-publish-list.py --recursive --quiet "${ROOTS[@]}"); do - if ! venv/bin/python3 tools/crate-publish-checks.py "crates/$crate"; then - FAILED=$(( $FAILED + 1 )) - fi - done - if [ $FAILED -gt 0 ]; then - exit 1 - fi - - update: - name: Test spacetimedb-update flow - permissions: read-all - strategy: - matrix: - include: - - { target: x86_64-unknown-linux-gnu, runner: spacetimedb-runner } - - { target: aarch64-unknown-linux-gnu, runner: arm-runner } - - { target: aarch64-apple-darwin, runner: macos-latest } - - { target: x86_64-pc-windows-msvc, runner: windows-latest } - runs-on: ${{ matrix.runner }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Rust - uses: dsherret/rust-toolchain-file@v1 - - - name: Install rust target - run: rustup target add ${{ matrix.target }} - - - name: Install packages - if: ${{ matrix.runner == 'arm-runner' }} - shell: bash - run: sudo apt install -y libssl-dev - - - name: Build spacetimedb-update - run: cargo build --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update - - - name: Run self-install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - ROOT_DIR="$(mktemp -d)" - # NOTE(bfops): We need the `github-token-auth` feature because we otherwise tend to get ratelimited when we try to fetch `/releases/latest`. - # My best guess is that, on the GitHub runners, the "anonymous" ratelimit is shared by *all* users of that runner (I think this because it - # happens very frequently on the `macos-runner`, but we haven't seen it on any others). - cargo run --features github-token-auth --target ${{ matrix.target }} -p spacetimedb-update -- self-install --root-dir="${ROOT_DIR}" --yes - "${ROOT_DIR}"/spacetime --root-dir="${ROOT_DIR}" help - - unreal_engine_tests: - name: Unreal Engine Tests - # This can't go on e.g. ubuntu-latest because that runner runs out of disk space. ChatGPT suggested that the general solution tends to be to use - # a custom runner. - runs-on: spacetimedb-runner - container: - image: ghcr.io/epicgames/unreal-engine:dev-5.6 - credentials: - # Note(bfops): I don't think that `github.actor` needs to match the user that the token is for, because I'm using a token for my account and - # it seems to be totally happy. - # However, the token needs to be for a user that has access to the EpicGames org (see - # https://dev.epicgames.com/documentation/en-us/unreal-engine/downloading-source-code-in-unreal-engine?application_version=5.6) - username: ${{ github.actor }} - password: ${{ secrets.GHCR_TOKEN }} - # Run as root because otherwise we get permission denied for various directories inside the container. I tried doing dances to allow it to run - # without this (reassigning env vars and stuff), but was unable to get it to work and it felt like an uphill battle. - options: --user 0:0 - steps: -# Uncomment this before merging so that it will run properly if run manually through the GH actions flow. It was playing weird with rolled back -# commits though. -# - name: Find Git ref -# env: -# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# shell: bash -# run: | -# PR_NUMBER="${{ github.event.inputs.pr_number || null }}" -# if test -n "${PR_NUMBER}"; then -# GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" -# else -# GIT_REF="${{ github.ref }}" -# fi -# echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - - uses: dsherret/rust-toolchain-file@v1 - - name: Run Unreal Engine tests - working-directory: sdks/unreal - env: - UE_ROOT_PATH: /home/ue4/UnrealEngine - run: | - - apt-get update - apt-get install -y acl curl ca-certificates - - REPO="$GITHUB_WORKSPACE" - # Let ue4 read/write the workspace & tool caches without changing ownership - for p in "$REPO" "${RUNNER_TEMP:-/__t}" "${RUNNER_TOOL_CACHE:-/__t}"; do - [ -d "$p" ] && setfacl -R -m u:ue4:rwX -m d:u:ue4:rwX "$p" || true - done - - # Rust tool caches live under the runner tool cache so they persist - export CARGO_HOME="${RUNNER_TOOL_CACHE:-/__t}/cargo" - export RUSTUP_HOME="${RUNNER_TOOL_CACHE:-/__t}/rustup" - mkdir -p "$CARGO_HOME" "$RUSTUP_HOME" - chown -R ue4:ue4 "$CARGO_HOME" "$RUSTUP_HOME" - - # Make sure the UE build script is executable (and parents traversable) - UE_DIR="${UE_ROOT_PATH:-/home/ue4/UnrealEngine}" - chmod a+rx "$UE_DIR" "$UE_DIR/Engine" "$UE_DIR/Engine/Build" "$UE_DIR/Engine/Build/BatchFiles/Linux" || true - chmod a+rx "$UE_DIR/Engine/Build/BatchFiles/Linux/Build.sh" || true - - # Run the build & tests as ue4 (who owns the UE tree) - sudo -E -H -u ue4 env \ - HOME=/home/ue4 \ - XDG_CONFIG_HOME=/home/ue4/.config \ - CARGO_HOME="$CARGO_HOME" \ - RUSTUP_HOME="$RUSTUP_HOME" \ - PATH="$CARGO_HOME/bin:$PATH" \ - bash -lc ' - set -euxo pipefail - # Install rustup for ue4 if needed (uses the shared caches) - if ! command -v cargo >/dev/null 2>&1; then - curl -sSf https://sh.rustup.rs | sh -s -- -y - fi - rustup show >/dev/null - git config --global --add safe.directory "$GITHUB_WORKSPACE" || true - - cd "$GITHUB_WORKSPACE/sdks/unreal" - cargo --version - cargo test - ' - - cli_docs: - name: Check CLI docs - permissions: read-all - runs-on: ubuntu-latest - steps: - - name: Find Git ref - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - shell: bash - run: | - PR_NUMBER="${{ github.event.inputs.pr_number || null }}" - if test -n "${PR_NUMBER}"; then - GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )" - else - GIT_REF="${{ github.ref }}" - fi - echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV" - - - name: Checkout sources - uses: actions/checkout@v4 - with: - ref: ${{ env.GIT_REF }} - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: pnpm/action-setup@v4 - with: - run_install: true - - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - uses: dsherret/rust-toolchain-file@v1 - - - name: Check for docs change - run: | - cargo run --features markdown-docs -p spacetimedb-cli > docs/docs/cli-reference.md - pnpm format - git status - if git diff --exit-code HEAD; then - echo "No docs changes detected" - else - echo "It looks like the CLI docs have changed:" - exit 1 - fi - From f292accbde821f238592f6dd6477187ea702092a Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:23:07 -0700 Subject: [PATCH 26/39] [bfops/test-pr]: reduce CI --- .github/workflows/csharp-test.yml | 164 ------------------------------ 1 file changed, 164 deletions(-) delete mode 100644 .github/workflows/csharp-test.yml diff --git a/.github/workflows/csharp-test.yml b/.github/workflows/csharp-test.yml deleted file mode 100644 index b1f8d834997..00000000000 --- a/.github/workflows/csharp-test.yml +++ /dev/null @@ -1,164 +0,0 @@ -name: C#/Unity - Test Suite - -on: - push: - branches: - - master - pull_request: - -jobs: - unity-testsuite: - runs-on: spacetimedb-runner - # Cancel any previous testsuites running on the same PR and/or ref. - concurrency: - group: unity-test-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - timeout-minutes: 30 - steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 - - # Run cheap .NET tests first. If those fail, no need to run expensive Unity tests. - - - name: Setup dotnet - uses: actions/setup-dotnet@v3 - with: - global-json-file: global.json - - - name: Override NuGet packages - run: | - dotnet pack crates/bindings-csharp/BSATN.Runtime - dotnet pack crates/bindings-csharp/Runtime - - # Write out the nuget config file to `nuget.config`. This causes the spacetimedb-csharp-sdk repository - # to be aware of the local versions of the `bindings-csharp` packages in SpacetimeDB, and use them if - # available. Otherwise, `spacetimedb-csharp-sdk` will use the NuGet versions of the packages. - # This means that (if version numbers match) we will test the local versions of the C# packages, even - # if they're not pushed to NuGet. - # See https://learn.microsoft.com/en-us/nuget/reference/nuget-config-file for more info on the config file. - cd sdks/csharp - ./tools~/write-nuget-config.sh ../.. - - - name: Run .NET tests - working-directory: sdks/csharp - run: dotnet test -warnaserror - - - name: Verify C# formatting - working-directory: sdks/csharp - run: dotnet format --no-restore --verify-no-changes SpacetimeDB.ClientSDK.sln - - # Now, setup the Unity tests. - - - name: Patch spacetimedb dependency in Cargo.toml - working-directory: demo/Blackholio/server-rust - run: | - sed -i "s|spacetimedb *=.*|spacetimedb = \{ path = \"../../../crates/bindings\" \}|" Cargo.toml - cat Cargo.toml - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - id: cache-rust-deps - with: - workspaces: demo/Blackholio/server-rust - key: ${{ steps.checkout-stdb.outputs.commit }} - # Cache Rust deps even if unit tests have failed. - cache-on-failure: true - # Cache the CLI as well. - cache-all-crates: true - - - name: Install SpacetimeDB CLI from the local checkout - # Rebuild only if we didn't get a precise cache hit. - if: steps.cache-rust-deps.outputs.cache-hit == 'false' - run: | - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $HOME/.cargo/bin/spacetimedb-cli $HOME/.cargo/bin/spacetime - env: - # Share the target directory with our local project to avoid rebuilding same SpacetimeDB crates twice. - CARGO_TARGET_DIR: demo/Blackholio/server-rust/target - - - name: Check quickstart-chat bindings are up to date - working-directory: sdks/csharp/examples~/quickstart-chat - run: | - bash ../../tools~/gen-quickstart.sh "${GITHUB_WORKSPACE}" - # This was copied from tools/check-diff.sh. - # It's required because `spacetime generate` creates lines with the SpacetimeDB commit - # version, which would make this `git diff` check very brittle if included. - PATTERN='^// This was generated using spacetimedb cli version.*' - git diff --exit-code --ignore-matching-lines="$PATTERN" -- . || { - echo "Error: quickstart-chat bindings have changed. Please regenerate the bindings and commit them to this branch." - exit 1 - } - - - name: Generate client bindings - working-directory: demo/Blackholio/server-rust - run: bash ./generate.sh -y - - - name: Check for changes - run: | - # This was copied from tools/check-diff.sh. - # It's required because `spacetime generate` creates lines with the SpacetimeDB commit - # version, which would make this `git diff` check very brittle if included. - PATTERN='^// This was generated using spacetimedb cli version.*' - git diff --exit-code --ignore-matching-lines="$PATTERN" -- demo/Blackholio/client-unity/Assets/Scripts/autogen || { - echo "Error: Bindings are dirty. Please generate bindings again and commit them to this branch." - exit 1 - } - - - name: Check Unity meta files - uses: DeNA/unity-meta-check@v3 - with: - enable_pr_comment: ${{ github.event_name == 'pull_request' }} - target_path: sdks/csharp - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - - name: Start SpacetimeDB - run: | - spacetime start & - disown - - - name: Run regression tests - run: bash sdks/csharp/tools~/run-regression-tests.sh . - - - name: Publish unity-tests module to SpacetimeDB - working-directory: demo/Blackholio/server-rust - run: | - spacetime logout && spacetime login --server-issued-login local - bash ./publish.sh - - - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json - working-directory: demo/Blackholio/client-unity/Packages - run: | - # Replace the com.clockworklabs.spacetimedbsdk dependency with the current branch. - # Note: Pointing to a local directory does not work, because our earlier steps nuke our meta files, which then causes Unity to not properly respect the DLLs (e.g. - # codegen does not work properly). - yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "https://github.com/clockworklabs/SpacetimeDB.git?path=sdks/csharp#${{ github.head_ref }}"' manifest.json - cat manifest.json - - - uses: actions/cache@v3 - with: - path: demo/Blackholio/client-unity/Library - key: Unity-${{ github.head_ref }} - restore-keys: Unity- - - - name: Run Unity tests - uses: game-ci/unity-test-runner@v4 - with: - unityVersion: 2022.3.32f1 # Adjust Unity version to a valid tag - projectPath: demo/Blackholio/client-unity # Path to the Unity project subdirectory - githubToken: ${{ secrets.GITHUB_TOKEN }} - testMode: playmode - useHostNetwork: true - env: - UNITY_EMAIL: ${{ secrets.UNITY_EMAIL }} - UNITY_PASSWORD: ${{ secrets.UNITY_PASSWORD }} - UNITY_SERIAL: ${{ secrets.UNITY_SERIAL }} - # Skip if this is an external contribution. - # The license secrets will be empty, so the step would fail anyway. - if: ${{ !github.event.pull_request.head.repo.fork }} From b70693ba594d2713b133ef2a2cc6727b85327f84 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:24:04 -0700 Subject: [PATCH 27/39] [bfops/test-pr]: reduce CI --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f202d44bc2..c0846df3379 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,6 @@ jobs: matrix: include: - { runner: windows-latest, smoketest_args: --no-build-cli } - runner: [ spacetimedb-runner, windows-latest ] runs-on: ${{ matrix.runner }} steps: - name: Find Git ref From 3b9cc0b3a8631cb6bad52ab1edba4a7ff8aa11a0 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:24:17 -0700 Subject: [PATCH 28/39] [bfops/test-pr]: reduce CI --- .github/workflows/typescript-lint.yml | 41 -------- .github/workflows/typescript-test.yml | 142 -------------------------- 2 files changed, 183 deletions(-) delete mode 100644 .github/workflows/typescript-lint.yml delete mode 100644 .github/workflows/typescript-test.yml diff --git a/.github/workflows/typescript-lint.yml b/.github/workflows/typescript-lint.yml deleted file mode 100644 index fe6649fd501..00000000000 --- a/.github/workflows/typescript-lint.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: TypeScript - Lint - -on: - pull_request: - push: - branches: - - master - merge_group: - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: pnpm/action-setup@v4 - with: - run_install: true - - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Lint - run: pnpm lint diff --git a/.github/workflows/typescript-test.yml b/.github/workflows/typescript-test.yml deleted file mode 100644 index 49c183f80db..00000000000 --- a/.github/workflows/typescript-test.yml +++ /dev/null @@ -1,142 +0,0 @@ -name: TypeScript - Tests - -on: - push: - branches: - - master - pull_request: - merge_group: - -jobs: - build-and-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: pnpm/action-setup@v4 - with: - run_install: true - - - name: Get pnpm store directory - shell: bash - working-directory: crates/bindings-typescript - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - uses: actions/cache@v4 - name: Setup pnpm cache - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Build module library and SDK - working-directory: crates/bindings-typescript - run: pnpm build - - - name: Run module library and SDK tests - working-directory: crates/bindings-typescript - run: pnpm test - - # - name: Extract SpacetimeDB branch name from file - # id: extract-branch - # run: | - # # Define the path to the branch file - # BRANCH_FILE=".github/spacetimedb-branch.txt" - - # # Default to master if file doesn't exist - # if [ ! -f "$BRANCH_FILE" ]; then - # echo "::notice::No SpacetimeDB branch file found, using 'master'" - # echo "branch=master" >> $GITHUB_OUTPUT - # exit 0 - # fi - - # # Read and trim whitespace from the file - # branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - - # # Fallback to master if empty - # if [ -z "$branch" ]; then - # echo "::warning::SpacetimeDB branch file is empty, using 'master'" - # branch="master" - # fi - - # echo "branch=$branch" >> $GITHUB_OUTPUT - # echo "Using SpacetimeDB branch from file: $branch" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache Rust dependencies - uses: Swatinem/rust-cache@v2 - with: - workspaces: modules/quickstart-chat - shared-key: quickstart-chat-test - - - name: Install SpacetimeDB CLI from the local checkout - run: | - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - rm -f $HOME/.cargo/bin/spacetime - ln -s $HOME/.cargo/bin/spacetimedb-cli $HOME/.cargo/bin/spacetime - # Clear any existing information - spacetime server clear -y - env: - # Share the target directory with our local project to avoid rebuilding same SpacetimeDB crates twice. - CARGO_TARGET_DIR: modules/quickstart-chat/target - - - name: Generate client bindings - working-directory: modules/quickstart-chat - run: | - spacetime generate --lang typescript --out-dir ../../crates/bindings-typescript/examples/quickstart-chat/src/module_bindings - cd ../../crates/bindings-typescript - pnpm format - - - name: Check for changes - working-directory: crates/bindings-typescript - run: | - # This was copied from SpacetimeDB/tools/check-diff.sh. - # It's required because `spacetime generate` creates lines with the SpacetimeDB commit - # version, which would make this `git diff` check very brittle if included. - PATTERN='^// This was generated using spacetimedb cli version.*' - if ! git diff --exit-code --ignore-matching-lines="$PATTERN" -- examples/quickstart-chat/src/module_bindings; then - echo "Error: Bindings are dirty. Please generate bindings again and commit them to this branch." - exit 1 - fi - - # - name: Start SpacetimeDB - # run: | - # spacetime start & - # disown - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/modules/quickstart-chat - # run: | - # spacetime logout && spacetime login --server-issued-login local - # spacetime publish -s local quickstart-chat -c -y - - # - name: Publish module to SpacetimeDB - # working-directory: SpacetimeDB/modules/quickstart-chat - # run: | - # spacetime logs quickstart-chat - - - name: Check that quickstart-chat builds - working-directory: crates/bindings-typescript/examples/quickstart-chat - run: pnpm build - - # - name: Run quickstart-chat tests - # working-directory: examples/quickstart-chat - # run: pnpm test - # - # # Run this step always, even if the previous steps fail - # - name: Print rows in the user table - # if: always() - # run: spacetime sql quickstart-chat "SELECT * FROM user" From 24b27f230e08bf35d7779c68a9912c2888f533e0 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:32:16 -0700 Subject: [PATCH 29/39] [bfops/test-pr]: reduce CI --- .github/workflows/ci.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0846df3379..753cdbc4535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,21 +52,9 @@ jobs: - name: Build and start database (Windows) if: runner.os == 'Windows' run: | - cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update + cargo build -p spacetimedb-standalone echo "Return code is: $?" - Start-Process target/debug/spacetimedb-cli.exe -ArgumentList 'start --pg-port 5432' cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them dotnet workload config --update-mode manifests dotnet workload update - - uses: actions/setup-python@v5 - with: { python-version: '3.12' } - if: runner.os == 'Windows' - - name: Install psycopg2 - run: python -m pip install psycopg2-binary - - name: Run smoketests - # Note: clear_database and replication only work in private - run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication - - name: Stop containers (Linux) - if: always() && runner.os == 'Linux' - run: docker compose down From 3bb39e1b35bc6cb9f7f781e76311777ef6758a18 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 12:47:06 -0700 Subject: [PATCH 30/39] [bfops/test-pr]: try CI fix --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 753cdbc4535..e018fe91cae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,10 @@ jobs: - name: Build and start database (Windows) if: runner.os == 'Windows' run: | + # Fail properly if any individual command fails + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + cargo build -p spacetimedb-standalone echo "Return code is: $?" cd modules From a29556b4645621e4c2c9c58fea0349120a6ea7ef Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Thu, 2 Oct 2025 14:52:43 -0700 Subject: [PATCH 31/39] [bfops/test-pr]: add link flags --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e018fe91cae..343cb92e042 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,8 +56,8 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true + $env:RUSTFLAGS='-C link-arg=/VERBOSE:LIB' cargo build -p spacetimedb-standalone - echo "Return code is: $?" cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them dotnet workload config --update-mode manifests From 599da6245efe22902999d5e809a0df1b2f045c9f Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 09:21:53 -0700 Subject: [PATCH 32/39] [bfops/test-pr]: try fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 343cb92e042..38293aae5bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $env:RUSTFLAGS='-C link-arg=/VERBOSE:LIB' + $env:RUSTFLAGS='-C link-arg=/VERBOSE:LIB -C link-arg=/INCREMENTAL:NO -C link-arg=/DEBUG:FASTLINK' cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From 85af16c94e8c5eae33fdd727d592aea74a27ec8f Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:12:35 -0700 Subject: [PATCH 33/39] [bfops/test-pr]: try this --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38293aae5bb..8efb1ac34dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,8 @@ jobs: $PSNativeCommandUseErrorActionPreference = $true $env:RUSTFLAGS='-C link-arg=/VERBOSE:LIB -C link-arg=/INCREMENTAL:NO -C link-arg=/DEBUG:FASTLINK' + # $env:RUSTFLAGS='-Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' + " -C link-arg=/PDB:`"$pdb`" -C link-arg=/PDBSTRIPPED:`"$pdb.stripped`"" + $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -C debuginfo=1 -C link-arg=/DEBUG:FULL' cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From 177003ac90f2ca118bd0286035312c7076da206d Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:15:21 -0700 Subject: [PATCH 34/39] [bfops/test-pr]: separate pdb file --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8efb1ac34dd..df892ae27dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,9 +56,9 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $env:RUSTFLAGS='-C link-arg=/VERBOSE:LIB -C link-arg=/INCREMENTAL:NO -C link-arg=/DEBUG:FASTLINK' - # $env:RUSTFLAGS='-Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' + " -C link-arg=/PDB:`"$pdb`" -C link-arg=/PDBSTRIPPED:`"$pdb.stripped`"" - $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -C debuginfo=1 -C link-arg=/DEBUG:FULL' + + $pdb = "$env:RUNNER_TEMP\app-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT.pdb" + $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' + " -C link-arg=/PDB:`"$pdb`" -C link-arg=/PDBSTRIPPED:`"$pdb.stripped`"" cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From ff3cbf191f88b74619687fa5376f47bc60e25cd2 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:20:00 -0700 Subject: [PATCH 35/39] [bfops/test-pr]: also run package workflow to see if it fails --- .github/workflows/package.yml | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index f1cf9a8eb4f..3f1e0eec47e 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -7,6 +7,7 @@ on: branches: - master - release/* + - bfops/test-pr jobs: build-cli: @@ -14,14 +15,6 @@ jobs: fail-fast: false matrix: include: - - { name: x86_64 Linux, target: x86_64-unknown-linux-gnu, runner: bare-metal, container: 'debian:bookworm' } - - { name: aarch64 Linux, target: aarch64-unknown-linux-gnu, runner: arm-runner } - # Disabled because musl builds weren't working and we didn't want to investigate. See https://github.com/clockworklabs/SpacetimeDB/pull/2964. - # - { name: x86_64 Linux musl, target: x86_64-unknown-linux-musl, runner: bare-metal, container: alpine } - # FIXME: arm musl build. "JavaScript Actions in Alpine containers are only supported on x64 Linux runners" - # - { name: aarch64 Linux musl, target: aarch64-unknown-linux-musl, runner: arm-runner } - - { name: aarch64 macOS, target: aarch64-apple-darwin, runner: macos-latest } - - { name: x86_64 macOS, target: x86_64-apple-darwin, runner: macos-latest } - { name: x86_64 Windows, target: x86_64-pc-windows-msvc, runner: windows-latest } name: Build CLI for ${{ matrix.name }} @@ -72,18 +65,3 @@ jobs: cd target/${{matrix.target}}/release cp spacetimedb-update.exe ../../../build/spacetimedb-update-${{matrix.target}}.exe 7z a ../../../build/spacetime-${{matrix.target}}.zip spacetimedb-cli.exe spacetimedb-standalone.exe - - - name: Extract branch name - shell: bash - run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - id: extract_branch - - - name: Upload to DO Spaces - uses: shallwefootball/s3-upload-action@master - with: - aws_key_id: ${{ secrets.AWS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}} - aws_bucket: ${{ vars.AWS_BUCKET }} - source_dir: build - endpoint: https://nyc3.digitaloceanspaces.com - destination_dir: ${{ steps.extract_branch.outputs.branch }} From 4809e4f81bc6f183428cb1324cab8517f3869e65 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:33:37 -0700 Subject: [PATCH 36/39] [bfops/test-pr]: remove custom pdb --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df892ae27dc..22c815fbc74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,9 +56,7 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - - $pdb = "$env:RUNNER_TEMP\app-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT.pdb" - $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' + " -C link-arg=/PDB:`"$pdb`" -C link-arg=/PDBSTRIPPED:`"$pdb.stripped`"" + $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From 86a2dec9d52a3c1b8aa6752c45a2a2d426d81c77 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:41:44 -0700 Subject: [PATCH 37/39] [bfops/test-pr]: try without lld-link --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22c815fbc74..65623de6c02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,8 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' + $pdb = "$env:RUNNER_TEMP/app-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT.pdb" + $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -C debuginfo=1 -C link-arg=/DEBUG:FULL' + " -C link-arg=/PDB:`"$pdb`" -C link-arg=/PDBSTRIPPED:`"$pdb.stripped`"" cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From 3f016bf9607322f3361e47bb9feff22b943f8436 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:54:37 -0700 Subject: [PATCH 38/39] [bfops/test-pr]: revert --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65623de6c02..22c815fbc74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,8 +56,7 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $pdb = "$env:RUNNER_TEMP/app-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT.pdb" - $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -C debuginfo=1 -C link-arg=/DEBUG:FULL' + " -C link-arg=/PDB:`"$pdb`" -C link-arg=/PDBSTRIPPED:`"$pdb.stripped`"" + $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them From e14a0998d8a70efca648ada9efd47dc4cef4bba4 Mon Sep 17 00:00:00 2001 From: Zeke Foppa Date: Fri, 3 Oct 2025 10:54:53 -0700 Subject: [PATCH 39/39] [bfops/test-pr]: simplified args --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22c815fbc74..7d06de201ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - $env:RUSTFLAGS='-C link-arg=/INCREMENTAL:NO -Clinker=lld-link -C debuginfo=1 -C link-arg=/DEBUG:FULL' + $env:RUSTFLAGS='-Clinker=lld-link' cargo build -p spacetimedb-standalone cd modules # the sdk-manifests on windows-latest are messed up, so we need to update them