From e149b680beb8e8dc9743169b98790c182074d551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Alejandro=20Montoya=20Corte=CC=81s?= Date: Mon, 13 Nov 2023 13:18:22 -0500 Subject: [PATCH 1/5] Refactoring IDs for system objects --- Cargo.lock | 105 ++++++-- crates/bench/Cargo.toml | 25 +- crates/bench/src/spacetime_module.rs | 2 +- crates/bench/src/spacetime_raw.rs | 7 +- crates/bench/src/sqlite.rs | 4 +- crates/bindings-macro/src/lib.rs | 29 +- crates/bindings/src/lib.rs | 6 +- crates/bindings/src/rt.rs | 29 +- crates/cli/Cargo.toml | 1 + crates/cli/src/subcommands/generate/csharp.rs | 6 +- crates/cli/src/subcommands/generate/python.rs | 10 +- crates/cli/src/subcommands/generate/rust.rs | 4 +- .../src/subcommands/generate/typescript.rs | 17 +- crates/core/src/db/commit_log.rs | 12 +- .../db/datastore/locking_tx_datastore/mod.rs | 73 +++--- crates/core/src/db/datastore/system_tables.rs | 60 ++--- crates/core/src/db/datastore/traits.rs | 28 +- crates/core/src/db/relational_db.rs | 92 +++---- crates/core/src/db/update.rs | 2 +- crates/core/src/host/instance_env.rs | 9 +- crates/core/src/host/mod.rs | 20 +- crates/core/src/host/module_host.rs | 18 +- .../src/host/wasm_common/module_host_actor.rs | 7 +- .../core/src/host/wasmer/wasm_instance_env.rs | 2 +- crates/core/src/sql/ast.rs | 66 ++--- crates/core/src/sql/compiler.rs | 6 +- crates/core/src/sql/execute.rs | 4 +- crates/core/src/subscription/query.rs | 27 +- crates/core/src/subscription/subscription.rs | 12 +- crates/core/src/vm.rs | 36 ++- crates/lib/Cargo.toml | 1 - crates/lib/src/identity.rs | 4 +- crates/lib/src/lib.rs | 24 +- crates/sats/src/db/attr.rs | 91 ++++++- crates/sats/src/db/def.rs | 247 ++++++++---------- crates/sats/src/db/mod.rs | 17 +- crates/sats/src/lib.rs | 2 +- crates/sats/src/product_value.rs | 22 +- crates/vm/src/expr.rs | 119 +++++---- 39 files changed, 690 insertions(+), 556 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53c4520dbc0..341d0353af1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1585,6 +1585,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "2.0.0" @@ -2026,7 +2032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" dependencies = [ "bitmaps", - "rand_core", + "rand_core 0.6.4", "rand_xoshiro", "sized-chunks", "typenum", @@ -3015,7 +3021,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand", + "rand 0.8.5", "sha2", "stringprep", ] @@ -3123,7 +3129,7 @@ dependencies = [ "bitflags 2.4.1", "lazy_static", "num-traits", - "rand", + "rand 0.8.5", "rand_chacha", "rand_xorshift", "regex-syntax 0.7.5", @@ -3295,6 +3301,19 @@ dependencies = [ "nibble_vec", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -3303,7 +3322,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3313,9 +3332,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -3331,7 +3365,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3340,7 +3374,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3363,6 +3397,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -3479,6 +3522,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rend" version = "0.4.1" @@ -3617,7 +3669,7 @@ dependencies = [ "bytes", "num-traits", "postgres", - "rand", + "rand 0.8.5", "rkyv", "serde", "serde_json", @@ -4147,7 +4199,7 @@ dependencies = [ "log", "nonempty", "once_cell", - "rand", + "rand 0.8.5", "scoped-tls", "spacetimedb-bindings-macro", "spacetimedb-bindings-sys", @@ -4169,7 +4221,7 @@ dependencies = [ "lazy_static", "log", "mimalloc", - "rand", + "rand 0.8.5", "regex", "rusqlite", "serde", @@ -4178,9 +4230,10 @@ dependencies = [ "spacetimedb-core", "spacetimedb-lib", "spacetimedb-primitives", + "spacetimedb-sats", "spacetimedb-standalone", "spacetimedb-testing", - "tempfile", + "tempdir", "tokio", "tracing-subscriber", "walkdir", @@ -4227,6 +4280,7 @@ dependencies = [ "itertools 0.11.0", "jsonwebtoken", "mimalloc", + "nonempty", "regex", "reqwest", "rustyline", @@ -4267,7 +4321,7 @@ dependencies = [ "log", "mime", "prometheus", - "rand", + "rand 0.8.5", "regex", "serde", "serde_json", @@ -4328,7 +4382,7 @@ dependencies = [ "proptest-derive", "prost", "prost-build", - "rand", + "rand 0.8.5", "rayon-core", "regex", "rocksdb", @@ -4393,11 +4447,10 @@ dependencies = [ "itertools 0.11.0", "proptest", "proptest-derive", - "rand", + "rand 0.8.5", "serde", "serde_json", "serde_with", - "sha3", "spacetimedb-bindings-macro", "spacetimedb-primitives", "spacetimedb-sats", @@ -4435,7 +4488,7 @@ dependencies = [ "nonempty", "proptest", "proptest-derive", - "rand", + "rand 0.8.5", "serde", "sha3", "smallvec", @@ -4462,7 +4515,7 @@ dependencies = [ "lazy_static", "log", "prost", - "rand", + "rand 0.8.5", "spacetimedb-client-api-messages", "spacetimedb-lib", "spacetimedb-sats", @@ -4505,7 +4558,7 @@ dependencies = [ "lazy_static", "log", "prost", - "rand", + "rand 0.8.5", "serde_json", "serial_test", "spacetimedb-cli", @@ -4835,6 +4888,16 @@ version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.8.0" @@ -5029,7 +5092,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand", + "rand 0.8.5", "socket2 0.5.4", "tokio", "tokio-util", @@ -5300,7 +5363,7 @@ dependencies = [ "httparse", "log", "native-tls", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -5933,7 +5996,7 @@ dependencies = [ "memfd", "memoffset 0.9.0", "paste", - "rand", + "rand 0.8.5", "rustix", "sptr", "wasm-encoder", diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml index a39c86f1faf..62bb519139f 100644 --- a/crates/bench/Cargo.toml +++ b/crates/bench/Cargo.toml @@ -22,27 +22,28 @@ bench = false [dependencies] spacetimedb-lib = { path = "../lib" } spacetimedb-core = { path = "../core" } +spacetimedb-sats= { path = "../sats" } spacetimedb-standalone = { path = "../standalone" } spacetimedb-client-api = { path = "../client-api" } spacetimedb-testing = { path = "../testing" } spacetimedb-primitives = { path = "../primitives" } ahash.workspace = true -log.workspace = true -clap.workspace = true -rusqlite.workspace = true -criterion.workspace = true -tempfile.workspace = true -rand.workspace = true -tokio.workspace = true -serde_json.workspace = true -serde.workspace = true anyhow.workspace = true anymap.workspace = true byte-unit.workspace = true +clap.workspace = true +criterion.workspace = true futures.workspace = true -tracing-subscriber.workspace = true lazy_static.workspace = true -walkdir.workspace = true -regex.workspace = true +log.workspace = true mimalloc.workspace = true +rand.workspace = true +regex.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +tempdir.workspace = true +tokio.workspace = true +tracing-subscriber.workspace = true +walkdir.workspace = true diff --git a/crates/bench/src/spacetime_module.rs b/crates/bench/src/spacetime_module.rs index 4a01572cf66..26cd31292ff 100644 --- a/crates/bench/src/spacetime_module.rs +++ b/crates/bench/src/spacetime_module.rs @@ -201,7 +201,7 @@ impl BenchDatabase for SpacetimeModule { } } -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct TableId { pascal_case: String, snake_case: String, diff --git a/crates/bench/src/spacetime_raw.rs b/crates/bench/src/spacetime_raw.rs index 5865efac019..df85e19f60b 100644 --- a/crates/bench/src/spacetime_raw.rs +++ b/crates/bench/src/spacetime_raw.rs @@ -9,7 +9,7 @@ use spacetimedb_lib::sats::db::def::{IndexDef, TableDef}; use spacetimedb_lib::sats::AlgebraicValue; use spacetimedb_primitives::{ColId, TableId}; use std::hint::black_box; -use tempfile::TempDir; +use tempdir::TempDir; pub type DbResult = (RelationalDB, TempDir, u32); @@ -28,7 +28,7 @@ impl BenchDatabase for SpacetimeRaw { where Self: Sized, { - let temp_dir = TempDir::with_prefix("stdb_test")?; + let temp_dir = TempDir::new("stdb_test")?; let db = open_db(temp_dir.path(), in_memory, fsync)?; Ok(SpacetimeRaw { @@ -46,13 +46,14 @@ impl BenchDatabase for SpacetimeRaw { match index_strategy { IndexStrategy::Unique => { self.db - .create_index(tx, IndexDef::new("id".to_string(), table_id, 0.into(), true))?; + .create_index(tx, table_id, IndexDef::new("id".to_string(), table_id, 0.into(), true))?; } IndexStrategy::NonUnique => (), IndexStrategy::MultiIndex => { for (i, column) in T::product_type().elements.iter().enumerate() { self.db.create_index( tx, + table_id, IndexDef::new(column.name.clone().unwrap(), table_id, i.into(), false), )?; } diff --git a/crates/bench/src/sqlite.rs b/crates/bench/src/sqlite.rs index 6a5cde106e9..d6167a09eb9 100644 --- a/crates/bench/src/sqlite.rs +++ b/crates/bench/src/sqlite.rs @@ -12,7 +12,7 @@ use std::{ hint::black_box, sync::{Arc, RwLock}, }; -use tempfile::TempDir; +use tempdir::TempDir; /// SQLite benchmark harness. pub struct SQLite { @@ -30,7 +30,7 @@ impl BenchDatabase for SQLite { where Self: Sized, { - let temp_dir = TempDir::with_prefix("sqlite_test")?; + let temp_dir = TempDir::new("sqlite_test")?; let db = if in_memory { Connection::open_in_memory()? } else { diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index c01be68639b..11543008088 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -18,13 +18,15 @@ use bitflags::Flags; use module::{derive_deserialize, derive_satstype, derive_serialize}; use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, TokenStreamExt}; -use spacetimedb_primitives::ColumnIndexAttribute; use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned; use syn::{ parse_quote, BinOp, Expr, ExprBinary, ExprLit, ExprUnary, FnArg, Ident, ItemFn, ItemStruct, Member, Token, Type, UnOp, }; +#[path = "../../sats/src/db/attr.rs"] +mod attr; +use attr::ColumnAttribute; mod sym { /// A symbol known at compile-time against @@ -433,7 +435,7 @@ fn gen_reducer(original_function: ItemFn, reducer_name: &str, extra: ReducerExtr struct Column<'a> { index: u8, field: &'a module::SatsField<'a>, - attr: ColumnIndexAttribute, + attr: ColumnAttribute, } fn spacetimedb_table(item: TokenStream) -> syn::Result { @@ -527,19 +529,19 @@ fn spacetimedb_tabletype_impl(item: syn::DeriveInput) -> syn::Result (ColumnIndexAttribute::UNIQUE, span), - ColumnAttr::Autoinc(span) => (ColumnIndexAttribute::AUTO_INC, span), - ColumnAttr::Primarykey(span) => (ColumnIndexAttribute::PRIMARY_KEY, span), + ColumnAttr::Unique(span) => (ColumnAttribute::UNIQUE, span), + ColumnAttr::Autoinc(span) => (ColumnAttribute::AUTO_INC, span), + ColumnAttr::Primarykey(span) => (ColumnAttribute::PRIMARY_KEY, span), }; // do those attributes intersect (not counting the INDEXED bit which is present in all attributes)? // this will check that no two attributes both have UNIQUE, AUTOINC or PRIMARY_KEY bits set if !(col_attr & extra_col_attr) - .difference(ColumnIndexAttribute::INDEXED) + .difference(ColumnAttribute::INDEXED) .is_empty() { return Err(duplicate(span)); @@ -547,7 +549,7 @@ fn spacetimedb_tabletype_impl(item: syn::DeriveInput) -> syn::Result syn::Result, Vec<_>) = columns - .iter() - .partition(|x| x.attr.contains(ColumnIndexAttribute::UNIQUE)); + let (unique_columns, nonunique_columns): (Vec<_>, Vec<_>) = + columns.iter().partition(|x| x.attr.contains(ColumnAttribute::UNIQUE)); let has_unique = !unique_columns.is_empty(); @@ -696,7 +697,7 @@ fn spacetimedb_tabletype_impl(item: syn::DeriveInput) -> syn::Result syn::Result] = &[#(#indexes),*]; type InsertResult = #insert_result; diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index e97c6266934..a0ad7a29b20 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -13,14 +13,15 @@ mod timestamp; use crate::sats::db::def::IndexType; use spacetimedb_lib::buffer::{BufReader, BufWriter, Cursor, DecodeError}; use spacetimedb_lib::sats::{impl_deserialize, impl_serialize, impl_st}; +pub use spacetimedb_lib::ser::Serialize; use spacetimedb_lib::{bsatn, PrimaryKey, ProductType, ProductValue}; -use spacetimedb_primitives::ColumnIndexAttribute; use std::cell::RefCell; use std::marker::PhantomData; use std::slice::from_ref; use std::{fmt, panic}; use sys::{Buffer, BufferIter}; +use crate::sats::db::attr::ColumnAttribute; pub use log; pub use sats::SpacetimeType; pub use spacetimedb_bindings_macro::{duration, query, spacetimedb, TableType}; @@ -28,7 +29,6 @@ pub use spacetimedb_bindings_sys as sys; pub use spacetimedb_lib; pub use spacetimedb_lib::de::{Deserialize, DeserializeOwned}; pub use spacetimedb_lib::sats; -pub use spacetimedb_lib::ser::Serialize; pub use spacetimedb_lib::Address; pub use spacetimedb_lib::AlgebraicValue; pub use spacetimedb_lib::Identity; @@ -345,7 +345,7 @@ impl Iterator for TableIter { /// Additionally, the type knows its own table name, its column attributes, and indices. pub trait TableType: SpacetimeType + DeserializeOwned + Serialize { const TABLE_NAME: &'static str; - const COLUMN_ATTRS: &'static [ColumnIndexAttribute]; + const COLUMN_ATTRS: &'static [ColumnAttribute]; const INDEXES: &'static [IndexDef<'static>]; type InsertResult: sealed::InsertResult; diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index a3cf9b37798..421741187c2 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,23 +1,26 @@ #![deny(unsafe_op_in_unsafe_fn)] +use nonempty::NonEmpty; use std::any::TypeId; use std::collections::{btree_map, BTreeMap}; use std::fmt; use std::marker::PhantomData; use std::sync::Mutex; use std::time::Duration; +use sys::Buffer; use crate::sats::db::auth::{StAccess, StTableType}; use crate::timestamp::with_timestamp_set; use crate::{sys, ReducerContext, ScheduleToken, SpacetimeType, TableType, Timestamp}; -pub use once_cell::sync::{Lazy, OnceCell}; use spacetimedb_lib::de::{self, Deserialize, SeqProductAccess}; use spacetimedb_lib::sats::typespace::TypespaceBuilder; use spacetimedb_lib::sats::{impl_deserialize, impl_serialize, AlgebraicType, AlgebraicTypeRef, ProductTypeElement}; use spacetimedb_lib::ser::{Serialize, SerializeSeqProduct}; use spacetimedb_lib::{bsatn, Address, Identity, MiscModuleExport, ModuleDef, ReducerDef, TableDef, TypeAlias}; use spacetimedb_primitives::TableId; -use sys::Buffer; + +use crate::sats::db::def::{IndexDef, AUTO_TABLE_ID}; +pub use once_cell::sync::{Lazy, OnceCell}; /// The `sender` invokes `reducer` at `timestamp` and provides it with the given `args`. /// @@ -400,11 +403,21 @@ pub fn register_reftype() { pub fn register_table() { register_describer(|module| { let data = *T::make_type(module).as_ref().unwrap(); + + let indexes = T::COLUMN_ATTRS.iter().zip(T::INDEXES.iter()).map(|(attr, idx)| { + IndexDef::new_cols( + idx.name.into(), + AUTO_TABLE_ID, + attr.has_unique(), + NonEmpty::from_slice(idx.col_ids).unwrap_or_default().map(Into::into), + ) + }); + let schema = TableDef { name: T::TABLE_NAME.into(), data, column_attrs: T::COLUMN_ATTRS.to_owned(), - indexes: T::INDEXES.iter().copied().map(Into::into).collect(), + indexes: indexes.collect(), table_type: StTableType::User, table_access: StAccess::for_name(T::TABLE_NAME), }; @@ -412,16 +425,6 @@ pub fn register_table() { }) } -impl From> for spacetimedb_lib::IndexDef { - fn from(index: crate::IndexDef<'_>) -> spacetimedb_lib::IndexDef { - spacetimedb_lib::IndexDef { - name: index.name.to_owned(), - index_type: index.ty, - cols: index.col_ids.to_owned(), - } - } -} - /// Registers a describer for the reducer `I` with arguments `A`. pub fn register_reducer<'a, A: Args<'a>, T, I: ReducerInfo>(_: impl Reducer<'a, A, T>) { register_describer(|module| { diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 2258e464f0a..28550fb2734 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -36,6 +36,7 @@ is-terminal.workspace = true itertools.workspace = true indicatif.workspace = true jsonwebtoken.workspace = true +nonempty.workspace = true mimalloc.workspace = true regex.workspace = true reqwest.workspace = true diff --git a/crates/cli/src/subcommands/generate/csharp.rs b/crates/cli/src/subcommands/generate/csharp.rs index 45495bb186c..7167e3de0e8 100644 --- a/crates/cli/src/subcommands/generate/csharp.rs +++ b/crates/cli/src/subcommands/generate/csharp.rs @@ -3,7 +3,7 @@ use super::util::fmt_fn; use std::fmt::{self, Write}; use convert_case::{Case, Casing}; -use spacetimedb_lib::sats::db::attr::ColumnIndexAttribute; +use spacetimedb_lib::sats::db::attr::ColumnAttribute; use spacetimedb_lib::sats::{ AlgebraicType, AlgebraicType::Builtin, AlgebraicTypeRef, ArrayType, BuiltinType, MapType, ProductType, SumType, }; @@ -565,7 +565,7 @@ fn autogen_csharp_product_table_common( ctx: &GenCtx, name: &str, product_type: &ProductType, - column_attrs: Option<&[ColumnIndexAttribute]>, + column_attrs: Option<&[ColumnAttribute]>, namespace: &str, ) -> String { let mut output = CodeIndenter::new(String::new()); @@ -923,7 +923,7 @@ fn autogen_csharp_access_funcs_for_struct( struct_name_pascal_case: &str, product_type: &ProductType, table_name: &str, - column_attrs: &[ColumnIndexAttribute], + column_attrs: &[ColumnAttribute], ) -> bool { let (unique, nonunique) = column_attrs .iter() diff --git a/crates/cli/src/subcommands/generate/python.rs b/crates/cli/src/subcommands/generate/python.rs index 627280ded24..c3aa0a13b8c 100644 --- a/crates/cli/src/subcommands/generate/python.rs +++ b/crates/cli/src/subcommands/generate/python.rs @@ -1,9 +1,9 @@ use super::util::fmt_fn; use convert_case::{Case, Casing}; -use spacetimedb_lib::sats::db::attr::ColumnIndexAttribute; +use spacetimedb_lib::sats::db::attr::ColumnAttribute; use spacetimedb_lib::{ - sats::{AlgebraicType::Builtin, AlgebraicTypeRef, ArrayType, BuiltinType, MapType}, + sats::{AlgebraicTypeRef, ArrayType, BuiltinType, MapType}, AlgebraicType, ProductType, ProductTypeElement, ReducerDef, SumType, TableDef, }; use std::fmt::{self, Write}; @@ -160,7 +160,7 @@ fn generate_imports(ctx: &GenCtx, elements: &Vec, imports: & fn _generate_imports(ctx: &GenCtx, ty: &AlgebraicType, imports: &mut Vec) { match ty { - Builtin(b) => match b { + AlgebraicType::Builtin(b) => match b { BuiltinType::Array(ArrayType { elem_ty }) => _generate_imports(ctx, elem_ty, imports), BuiltinType::Map(map_type) => { _generate_imports(ctx, &map_type.key_ty, imports); @@ -188,7 +188,7 @@ fn autogen_python_product_table_common( ctx: &GenCtx, name: &str, product_type: &ProductType, - column_attrs: Option<&[ColumnIndexAttribute]>, + column_attrs: Option<&[ColumnAttribute]>, ) -> String { let is_table = column_attrs.is_some(); @@ -389,7 +389,7 @@ fn autogen_python_product_table_common( AlgebraicType::Product(_) => { reducer_args.push(format!("self.{python_field_name}")); } - Builtin(_) => { + AlgebraicType::Builtin(_) => { reducer_args.push(format!("self.{python_field_name}")); } AlgebraicType::Ref(type_ref) => { diff --git a/crates/cli/src/subcommands/generate/rust.rs b/crates/cli/src/subcommands/generate/rust.rs index 339a587a621..fae4f0402f0 100644 --- a/crates/cli/src/subcommands/generate/rust.rs +++ b/crates/cli/src/subcommands/generate/rust.rs @@ -1,7 +1,7 @@ use super::code_indenter::CodeIndenter; use super::{GenCtx, GenItem}; use convert_case::{Case, Casing}; -use spacetimedb_lib::sats::db::attr::ColumnIndexAttribute; +use spacetimedb_lib::sats::db::attr::ColumnAttribute; use spacetimedb_lib::sats::{ AlgebraicType, AlgebraicTypeRef, ArrayType, BuiltinType, MapType, ProductType, ProductTypeElement, SumType, SumTypeVariant, @@ -489,7 +489,7 @@ fn print_table_filter_methods( out: &mut Indenter, table_type_name: &str, elements: &[ProductTypeElement], - attrs: &[ColumnIndexAttribute], + attrs: &[ColumnAttribute], ) { write!(out, "impl {} ", table_type_name).unwrap(); out.delimited_block( diff --git a/crates/cli/src/subcommands/generate/typescript.rs b/crates/cli/src/subcommands/generate/typescript.rs index b8eba1d4447..e1d5df2fefb 100644 --- a/crates/cli/src/subcommands/generate/typescript.rs +++ b/crates/cli/src/subcommands/generate/typescript.rs @@ -3,10 +3,10 @@ use super::util::fmt_fn; use std::fmt::{self, Write}; use convert_case::{Case, Casing}; -use spacetimedb_lib::sats::db::attr::ColumnIndexAttribute; +use spacetimedb_lib::sats::db::attr::ColumnAttribute; use spacetimedb_lib::sats::{ - AlgebraicType, AlgebraicType::Builtin, AlgebraicTypeRef, ArrayType, BuiltinType, MapType, ProductType, - ProductTypeElement, SumType, SumTypeVariant, + AlgebraicType, AlgebraicTypeRef, ArrayType, BuiltinType, MapType, ProductType, ProductTypeElement, SumType, + SumTypeVariant, }; use spacetimedb_lib::{ReducerDef, TableDef}; @@ -142,6 +142,7 @@ fn convert_type<'a>( } AlgebraicType::Sum(sum_type) => { if let Some(inner_ty) = sum_type.as_option() { + use AlgebraicType::Builtin; match inner_ty { Builtin(ty) => match ty { BuiltinType::Bool @@ -315,8 +316,8 @@ fn serialize_type<'a>( } } AlgebraicType::Builtin(BuiltinType::Array(ArrayType { elem_ty })) => match &**elem_ty { - Builtin(BuiltinType::U8) => write!(f, "Array.from({value})"), - Builtin(_) => write!(f, "{value}"), + AlgebraicType::Builtin(BuiltinType::U8) => write!(f, "Array.from({value})"), + AlgebraicType::Builtin(_) => write!(f, "{value}"), t => write!(f, "{value}.map(el => {})", serialize_type(ctx, t, "el", prefix)), }, AlgebraicType::Builtin(_) => write!(f, "{value}"), @@ -595,7 +596,7 @@ fn generate_imports_variants( fn _generate_imports(ctx: &GenCtx, ty: &AlgebraicType, imports: &mut Vec, prefix: Option<&str>) { match ty { - Builtin(b) => match b { + AlgebraicType::Builtin(b) => match b { BuiltinType::Array(ArrayType { elem_ty }) => _generate_imports(ctx, elem_ty, imports, prefix), BuiltinType::Map(map_type) => { _generate_imports(ctx, &map_type.key_ty, imports, prefix); @@ -629,7 +630,7 @@ fn autogen_typescript_product_table_common( ctx: &GenCtx, name: &str, product_type: &ProductType, - column_attrs: Option<&[ColumnIndexAttribute]>, + column_attrs: Option<&[ColumnAttribute]>, ) -> String { let mut output = CodeIndenter::new(String::new()); @@ -953,7 +954,7 @@ fn autogen_typescript_access_funcs_for_struct( struct_name_pascal_case: &str, product_type: &ProductType, table_name: &str, - column_attrs: &[ColumnIndexAttribute], + column_attrs: &[ColumnAttribute], ) { let (unique, nonunique) = column_attrs .iter() diff --git a/crates/core/src/db/commit_log.rs b/crates/core/src/db/commit_log.rs index 7b165b11c26..714a86750d0 100644 --- a/crates/core/src/db/commit_log.rs +++ b/crates/core/src/db/commit_log.rs @@ -1,3 +1,7 @@ +use anyhow::Context; +use std::io; +use std::sync::{Arc, Mutex, MutexGuard}; + use super::{ datastore::traits::{MutTxDatastore, TxData}, message_log::{self, MessageLog}, @@ -16,14 +20,8 @@ use crate::{ error::DBError, execution_context::ExecutionContext, }; - -use anyhow::Context; use spacetimedb_sats::hash::{hash_bytes, Hash}; - use spacetimedb_sats::DataKey; -use std::io; -use std::sync::Arc; -use std::sync::{Mutex, MutexGuard}; #[derive(Clone)] pub struct CommitLog { @@ -248,7 +246,7 @@ impl CommitLogView { /// the transactions in a [`Commit`]. /// /// The iterator attempts to read each large object in turn, yielding an - /// [`io::Error`] with kind [`io::ErrorKind::NotFound`] if the object was + /// [`io::Error`] with constraints [`io::ErrorKind::NotFound`] if the object was /// not found. // // TODO(kim): We probably want a more efficient way to stream the contents diff --git a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs index 8aef6b05d0f..75e98486f49 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs @@ -2,23 +2,37 @@ mod btree_index; mod sequence; mod table; +use nonempty::NonEmpty; +use parking_lot::{lock_api::ArcMutexGuard, Mutex, RawMutex}; + use self::{ btree_index::{BTreeIndex, BTreeIndexRangeIter}, sequence::Sequence, table::Table, }; +use anyhow::anyhow; +use std::time::{Duration, Instant}; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet, HashMap}, + ops::{Deref, RangeBounds}, + sync::Arc, + vec, +}; + use super::{ system_tables::{ - self, StColumnRow, StIndexRow, StModuleRow, StSequenceRow, StTableRow, INDEX_ID_SEQUENCE_ID, - SEQUENCE_ID_SEQUENCE_ID, ST_COLUMNS_ID, ST_COLUMNS_ROW_TYPE, ST_INDEXES_ID, ST_INDEX_ROW_TYPE, ST_MODULE_ID, - ST_SEQUENCES_ID, ST_SEQUENCE_ROW_TYPE, ST_TABLES_ID, ST_TABLE_ROW_TYPE, TABLE_ID_SEQUENCE_ID, WASM_MODULE, + StColumnRow, StIndexRow, StSequenceRow, StTableRow, ST_COLUMNS_ID, ST_COLUMNS_ROW_TYPE, ST_INDEXES_ID, + ST_INDEX_ROW_TYPE, ST_SEQUENCES_ID, ST_SEQUENCE_ROW_TYPE, ST_TABLES_ID, ST_TABLE_ROW_TYPE, }, traits::{self, DataRow, MutTx, MutTxDatastore, TxData, TxDatastore}, }; +use crate::db::datastore::system_tables; use crate::db::datastore::system_tables::{ st_constraints_schema, st_module_schema, table_name_is_system, StColumnFields, StConstraintRow, StIndexFields, - StSequenceFields, StTableFields, SystemTables, CONSTRAINT_ID_SEQUENCE_ID, ST_CONSTRAINTS_ID, - ST_CONSTRAINT_ROW_TYPE, ST_MODULE_ROW_TYPE, + StModuleRow, StSequenceFields, StTableFields, SystemTables, CONSTRAINT_ID_SEQUENCE_ID, INDEX_ID_SEQUENCE_ID, + SEQUENCE_ID_SEQUENCE_ID, ST_CONSTRAINTS_ID, ST_CONSTRAINT_ROW_TYPE, ST_MODULE_ID, ST_MODULE_ROW_TYPE, + TABLE_ID_SEQUENCE_ID, WASM_MODULE, }; use crate::db::db_metrics::DB_METRICS; use crate::{ @@ -31,27 +45,15 @@ use crate::{ error::{DBError, TableError}, execution_context::ExecutionContext, }; -use anyhow::anyhow; -use nonempty::NonEmpty; -use parking_lot::{lock_api::ArcMutexGuard, Mutex, RawMutex}; use spacetimedb_lib::Address; use spacetimedb_primitives::{ColId, IndexId, SequenceId, TableId}; -use spacetimedb_sats::db::def::{ColumnSchema, IndexDef, IndexSchema, IndexType, SequenceDef, TableDef, TableSchema}; +use spacetimedb_sats::data_key::{DataKey, ToDataKey}; +use spacetimedb_sats::db::def::*; use spacetimedb_sats::hash::Hash; +use spacetimedb_sats::relation::RelValue; use spacetimedb_sats::{ - data_key::ToDataKey, db::auth::{StAccess, StTableType}, - relation::RelValue, - DataKey, -}; -use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductType, ProductValue}; -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet, HashMap}, - ops::{Deref, RangeBounds}, - sync::Arc, - time::{Duration, Instant}, - vec, + AlgebraicType, AlgebraicValue, ProductType, ProductValue, }; use thiserror::Error; @@ -75,6 +77,8 @@ pub enum SequenceError { NotInteger { col: String, found: AlgebraicType }, #[error("Sequence ID `{0}` still had no values left after allocation.")] UnableToAllocate(SequenceId), + #[error("Autoinc constraint on table {0:?} spans more than one column: {1:?}")] + MultiColumnAutoInc(TableId, NonEmpty), } const SEQUENCE_PREALLOCATION_AMOUNT: i128 = 4_096; @@ -468,7 +472,7 @@ impl Inner { let row = StConstraintRow { constraint_id: constraint.constraint_id, constraint_name: constraint.constraint_name.clone(), - kind: constraint.kind, + constraints: constraint.constraints, table_id, columns: constraint.columns, }; @@ -486,7 +490,7 @@ impl Inner { let index_id = IndexId(constraint.constraint_id.0); //Check if add an index: - match constraint.kind { + match constraint.constraints { x if x.has_unique() => IndexSchema { index_id, table_id, @@ -503,7 +507,7 @@ impl Inner { is_unique: false, index_type: IndexType::BTree, }, - x => panic!("Adding constraint of kind `{x:?}` is not supported yet."), + x => panic!("Adding constraint of constraints `{x:?}` is not supported yet."), } })); @@ -603,7 +607,7 @@ impl Inner { let rows = self.iter_by_col_eq(&ctx, &table_id, col_id, value)?; let ids_to_delete = rows.map(|row| RowId(*row.id())).collect::>(); if ids_to_delete.is_empty() { - return Err(TableError::IdNotFound(table_id, table_id.0).into()); + return Err(TableError::IdNotFound(table_id, col_id.0).into()); } self.delete(&table_id, ids_to_delete); @@ -865,7 +869,8 @@ impl Inner { let row = rows .first() - .ok_or_else(|| TableError::IdNotFound(table_id, table_id.0))?; + .ok_or_else(|| TableError::IdNotFound(ST_TABLES_ID, table_id.0))?; + let el = StTableRow::try_from(row.view())?; let table_name = el.table_name.to_owned(); let table_id = el.table_id; @@ -1023,6 +1028,7 @@ impl Inner { self.create_index_internal(index_id, index)?; log::trace!("INDEX CREATED: id = {}", index_id); + Ok(index_id) } @@ -2362,20 +2368,15 @@ impl traits::MutProgrammable for Locking { Ok(()) } } - #[cfg(test)] mod tests { use super::*; + use itertools::Itertools; - use crate::db::datastore::system_tables::{StConstraintRow, ST_CONSTRAINTS_ID}; use crate::db::datastore::Result; use crate::error::IndexError; - use itertools::Itertools; - use nonempty::NonEmpty; use spacetimedb_lib::error::ResultTest; - use spacetimedb_sats::db::auth::{StAccess, StTableType}; - use spacetimedb_sats::db::def::{ColumnDef, Constraints}; - use spacetimedb_sats::{product, AlgebraicType, AlgebraicValue, ProductValue}; + use spacetimedb_sats::product; /// Utility to query the system tables and return their concrete table row pub struct SystemTableQuery<'a> { @@ -2708,12 +2709,12 @@ mod tests { ColRow { table: 4, pos: 0, name: "constraint_id", ty: AlgebraicType::U32, autoinc: true }, ColRow { table: 4, pos: 1, name: "constraint_name", ty: AlgebraicType::String, autoinc: false }, - ColRow { table: 4, pos: 2, name: "kind", ty: AlgebraicType::U32, autoinc: false }, + ColRow { table: 4, pos: 2, name: "constraints", ty: AlgebraicType::U32, autoinc: false }, ColRow { table: 4, pos: 3, name: "table_id", ty: AlgebraicType::U32, autoinc: false }, ColRow { table: 4, pos: 4, name: "columns", ty: AlgebraicType::array(AlgebraicType::U32), autoinc: false }, ColRow { table: 5, pos: 0, name: "program_hash", ty: AlgebraicType::array(AlgebraicType::U8), autoinc: false }, - ColRow { table: 5, pos: 1, name: "kind", ty: AlgebraicType::U8, autoinc: false }, + ColRow { table: 5, pos: 1, name: "constraints", ty: AlgebraicType::U8, autoinc: false }, ColRow { table: 5, pos: 2, name: "epoch", ty: AlgebraicType::U128, autoinc: false }, ])); #[rustfmt::skip] @@ -2736,7 +2737,7 @@ mod tests { assert_eq!(query.scan_st_constraints()?, vec![StConstraintRow { constraint_id: 5.into(), constraint_name: "ct_columns_table_id".to_string(), - kind: Constraints::indexed(), + constraints: Constraints::indexed(), table_id: 1.into(), columns: col(0), }]); diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index 1cad80e76d0..41ab373abfc 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -2,15 +2,14 @@ use crate::error::{DBError, TableError}; use core::fmt; use nonempty::NonEmpty; use once_cell::sync::Lazy; -use spacetimedb_primitives::{ColId, ConstraintId, IndexId, SequenceId, TableId}; + +use spacetimedb_primitives::*; use spacetimedb_sats::db::auth::{StAccess, StTableType}; -use spacetimedb_sats::db::def::{ - ColumnSchema, ConstraintSchema, Constraints, IndexSchema, IndexType, SequenceSchema, TableSchema, -}; +use spacetimedb_sats::db::def::*; use spacetimedb_sats::hash::Hash; +use spacetimedb_sats::product_value::InvalidFieldError; use spacetimedb_sats::{ - impl_deserialize, impl_serialize, product, product_value::InvalidFieldError, AlgebraicType, AlgebraicValue, - ArrayValue, ProductType, ProductValue, + impl_deserialize, impl_serialize, product, AlgebraicType, AlgebraicValue, ArrayValue, ProductType, ProductValue, }; /// The static ID of the table that defines tables @@ -71,7 +70,7 @@ impl SystemTables { pub(crate) fn total_constraints_indexes() -> usize { Self::tables() .iter() - .flat_map(|x| x.constraints.iter().filter(|x| x.kind != Constraints::unset())) + .flat_map(|x| x.constraints.iter().filter(|x| x.constraints != Constraints::unset())) .count() } @@ -169,14 +168,14 @@ st_fields_enum!( st_fields_enum!(enum StConstraintFields { "constraint_id", ConstraintId = 0, "constraint_name", ConstraintName = 1, - "kind", Kind = 2, + "constraints", Constraints = 2, "table_id", TableId = 3, "columns", Columns = 4, }); // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StModuleFields { "program_hash", ProgramHash = 0, - "kind", Kind = 1, + "constraints", Kind = 1, "epoch", Epoch = 2, }); @@ -297,10 +296,10 @@ pub fn st_columns_schema() -> TableSchema { constraints: vec![ConstraintSchema { constraint_id: ST_CONSTRAINT_ID_INDEX_HACK.0.into(), constraint_name: "ct_columns_table_id".to_string(), - kind: Constraints::indexed(), + constraints: Constraints::indexed(), table_id: ST_COLUMNS_ID, //TODO: Change to multi-columns when PR for it land: StColumnFields::ColId as u32 - columns: NonEmpty::new(StColumnFields::TableId.col_id()), + columns: StColumnFields::TableId.into(), }], table_type: StTableType::System, table_access: StAccess::Public, @@ -475,9 +474,9 @@ pub static ST_SEQUENCE_ROW_TYPE: Lazy = /// System Table [ST_CONSTRAINTS_NAME] /// -/// | constraint_id | constraint_name | kind | table_id | columns | -/// |---------------|-------------------- -|------|-------|------------| -/// | 1 | "unique_customer_id" | 1 | 100 | [1, 4] | +/// | constraint_id | constraint_name | constraints | table_id | columns | +/// |---------------|-------------------- -|-------------|-------|------------| +/// | 1 | "unique_customer_id" | 1 | 100 | [1, 4] | pub(crate) fn st_constraints_schema() -> TableSchema { TableSchema { table_id: ST_CONSTRAINTS_ID, @@ -499,8 +498,8 @@ pub(crate) fn st_constraints_schema() -> TableSchema { }, ColumnSchema { table_id: ST_CONSTRAINTS_ID, - col_id: StConstraintFields::Kind.col_id(), - col_name: StConstraintFields::Kind.col_name(), + col_id: StConstraintFields::Constraints.col_id(), + col_name: StConstraintFields::Constraints.col_name(), col_type: AlgebraicType::U32, is_autoinc: false, }, @@ -543,10 +542,10 @@ pub static ST_CONSTRAINT_ROW_TYPE: Lazy = /// SpacetimeDB module associated with the database: /// /// * `program_hash` is the [`Hash`] of the raw bytes of the (compiled) module. -/// * `kind` is the [`ModuleKind`] (currently always [`WASM_MODULE`]). +/// * `constraints` is the [`ModuleKind`] (currently always [`WASM_MODULE`]). /// * `epoch` is a _fencing token_ used to protect against concurrent updates. /// -/// | program_hash | kind | epoch | +/// | program_hash | constraints | epoch | /// |---------------------|----------|-------| /// | [250, 207, 5, ...] | 0 | 42 | pub(crate) fn st_module_schema() -> TableSchema { @@ -735,15 +734,15 @@ impl StIndexRow<&str> { } fn to_cols(row: &ProductValue, col_pos: ColId, col_name: &'static str) -> Result, DBError> { - let index = col_pos.idx(); - let cols = row.field_as_array(index, Some(col_name))?; + let col_pos = col_pos.idx(); + let cols = row.field_as_array(col_pos, Some(col_name))?; if let ArrayValue::U32(x) = &cols { let x: Vec<_> = x.iter().map(|x| ColId::from(*x)).collect(); Ok(NonEmpty::from_slice(&x).unwrap()) } else { Err(InvalidFieldError { name: Some(col_name), - index, + col_pos: col_pos.into(), } .into()) } @@ -757,11 +756,12 @@ impl<'a> TryFrom<&'a ProductValue> for StIndexRow<&'a str> { let index_name = row.field_as_str(StIndexFields::IndexName.col_idx(), None)?; let index_type = row.field_as_u8(StIndexFields::IndexType.col_idx(), None)?; let index_type = IndexType::try_from(index_type).map_err(|_| InvalidFieldError { - index: StIndexFields::IndexType.col_idx(), + col_pos: StIndexFields::IndexType.col_id(), name: Some(StIndexFields::IndexType.name()), })?; let columns = to_cols(row, StIndexFields::Columns.col_id(), StIndexFields::Columns.name())?; let is_unique = row.field_as_bool(StIndexFields::IsUnique.col_idx(), None)?; + Ok(StIndexRow { index_id, table_id, @@ -878,7 +878,7 @@ impl From> for SequenceSchema { pub struct StConstraintRow> { pub(crate) constraint_id: ConstraintId, pub(crate) constraint_name: Name, - pub(crate) kind: Constraints, + pub(crate) constraints: Constraints, pub(crate) table_id: TableId, pub(crate) columns: NonEmpty, } @@ -888,7 +888,7 @@ impl StConstraintRow<&str> { StConstraintRow { constraint_id: self.constraint_id, constraint_name: self.constraint_name.to_string(), - kind: self.kind, + constraints: self.constraints, table_id: self.table_id, columns: self.columns.clone(), } @@ -902,8 +902,8 @@ impl<'a> TryFrom<&'a ProductValue> for StConstraintRow<&'a str> { .field_as_u32(StConstraintFields::ConstraintId.col_idx(), None)? .into(); let constraint_name = row.field_as_str(StConstraintFields::ConstraintName.col_idx(), None)?; - let kind = row.field_as_u8(StConstraintFields::Kind.col_idx(), None)?; - let kind = Constraints::try_from(kind).expect("Fail to decode Constraints"); + let constraints = row.field_as_u8(StConstraintFields::Constraints.col_idx(), None)?; + let constraints = Constraints::try_from(constraints).expect("Fail to decode Constraints"); let table_id = row.field_as_u32(StConstraintFields::TableId.col_idx(), None)?.into(); let columns = to_cols( row, @@ -914,7 +914,7 @@ impl<'a> TryFrom<&'a ProductValue> for StConstraintRow<&'a str> { Ok(StConstraintRow { constraint_id, constraint_name, - kind, + constraints, table_id, columns, }) @@ -928,14 +928,14 @@ impl From> for ProductValue { product![ x.constraint_id, x.constraint_name, - x.kind.bits(), + x.constraints.bits(), x.table_id, ArrayValue::from(cols) ] } } -/// Indicates the kind of module the `program_hash` of a [`StModuleRow`] +/// Indicates the constraints of module the `program_hash` of a [`StModuleRow`] /// describes. /// /// More or less a placeholder to allow for future non-WASM hosts without @@ -945,7 +945,7 @@ pub struct ModuleKind(u8); /// The [`ModuleKind`] of WASM-based modules. /// -/// This is currently the only known kind. +/// This is currently the only known constraints. pub const WASM_MODULE: ModuleKind = ModuleKind(0); impl_serialize!([] ModuleKind, (self, ser) => self.0.serialize(ser)); diff --git a/crates/core/src/db/datastore/traits.rs b/crates/core/src/db/datastore/traits.rs index 2c4605436cb..2bb5e505e1e 100644 --- a/crates/core/src/db/datastore/traits.rs +++ b/crates/core/src/db/datastore/traits.rs @@ -223,7 +223,7 @@ pub trait MutTxDatastore: TxDatastore + MutTx { /// Describes a programmable [`TxDatastore`]. /// -/// A programmable datastore is one which has a program of some kind associated +/// A programmable datastore is one which has a program of some constraints associated /// with it. pub trait Programmable: TxDatastore { /// Retrieve the [`Hash`] of the program currently associated with the @@ -254,7 +254,7 @@ pub trait MutProgrammable: MutTxDatastore { mod tests { use nonempty::NonEmpty; use spacetimedb_primitives::ColId; - use spacetimedb_sats::db::attr::ColumnIndexAttribute; + use spacetimedb_sats::db::attr::ColumnAttribute; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::{IndexType, AUTO_TABLE_ID}; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductType, ProductTypeElement, Typespace}; @@ -266,18 +266,20 @@ mod tests { let lib_table_def = spacetimedb_lib::TableDef { name: "Person".into(), data: AlgebraicTypeRef(0), - column_attrs: vec![ColumnIndexAttribute::IDENTITY, ColumnIndexAttribute::UNSET], + column_attrs: vec![ColumnAttribute::IDENTITY, ColumnAttribute::UNSET], indexes: vec![ - spacetimedb_lib::IndexDef { - name: "id_and_name".into(), - index_type: IndexType::BTree, - cols: [0, 1].into(), - }, - spacetimedb_lib::IndexDef { - name: "just_name".into(), - index_type: IndexType::BTree, - cols: [1].into(), - }, + IndexDef::new_cols( + "id_and_name".into(), + AUTO_TABLE_ID, + false, + NonEmpty::from_slice(&[0.into(), 1.into()]).unwrap(), + ), + IndexDef::new_cols( + "just_name".into(), + AUTO_TABLE_ID, + false, + NonEmpty::from_slice(&[1.into()]).unwrap(), + ), ], table_type: StTableType::User, table_access: StAccess::Public, diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 5c02361a69e..942afd2640e 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -1,42 +1,36 @@ +use fs2::FileExt; +use nonempty::NonEmpty; +use std::borrow::Cow; +use std::fs::{create_dir_all, File}; +use std::io; +use std::ops::RangeBounds; +use std::path::Path; +use std::sync::{Arc, Mutex}; + use super::commit_log::{CommitLog, CommitLogView}; -use super::datastore::locking_tx_datastore::{DataRef, Iter, IterByColEq, IterByColRange, Locking, MutTxId, RowId}; -use super::datastore::traits::{DataRow, MutProgrammable, MutTx, MutTxDatastore, Programmable, TxData}; +use super::datastore::locking_tx_datastore::{DataRef, Iter, IterByColEq, IterByColRange, MutTxId, RowId}; +use super::datastore::traits::{MutProgrammable, MutTx, MutTxDatastore, Programmable, TxData}; use super::message_log::MessageLog; use super::ostorage::memory_object_db::MemoryObjectDB; use super::relational_operators::Relation; use crate::address::Address; +use crate::db::commit_log; +use crate::db::datastore::traits::DataRow; use crate::db::db_metrics::DB_METRICS; use crate::db::messages::commit::Commit; use crate::db::ostorage::hashmap_object_db::HashMapObjectDB; use crate::db::ostorage::ObjectDB; -use crate::error::{DBError, DatabaseError, IndexError, LogReplayError, TableError}; +use crate::error::{DBError, DatabaseError, IndexError, TableError}; use crate::execution_context::ExecutionContext; use crate::hash::Hash; -use fs2::FileExt; -use nonempty::NonEmpty; use spacetimedb_lib::PrimaryKey; -use spacetimedb_primitives::{ColId, ColumnIndexAttribute, IndexId, SequenceId, TableId}; +use spacetimedb_primitives::*; use spacetimedb_sats::data_key::ToDataKey; -use spacetimedb_sats::db::def::{IndexDef, SequenceDef, TableDef, TableSchema}; +use spacetimedb_sats::db::attr::ColumnAttribute; +use spacetimedb_sats::db::def::*; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductType, ProductValue}; -use std::borrow::Cow; -use std::fs::{create_dir_all, File}; -use std::io; -use std::ops::RangeBounds; -use std::path::Path; -use std::sync::{Arc, Mutex}; -pub const ST_TABLES_NAME: &str = "st_table"; -pub const ST_COLUMNS_NAME: &str = "st_columns"; -pub const ST_SEQUENCES_NAME: &str = "st_sequence"; -pub const ST_INDEXES_NAME: &str = "st_indexes"; - -/// The static ID of the table that defines tables -pub const ST_TABLES_ID: TableId = TableId(0); -/// The static ID of the table that defines columns -pub const ST_COLUMNS_ID: TableId = TableId(1); -/// The ID that we can start use to generate user tables that will not conflict with the bootstrapped ones. -pub const ST_TABLE_ID_START: TableId = TableId(2); +use super::datastore::locking_tx_datastore::Locking; #[derive(Clone)] pub struct RelationalDB { @@ -512,7 +506,7 @@ impl RelationalDB { tx: &mut MutTxId, table_id: TableId, cols: &NonEmpty, - ) -> Result { + ) -> Result { let table = self.inner.schema_for_table_mut_tx(tx, table_id)?; let columns = table.project_not_empty(cols)?; // Verify we don't have more than 1 auto_inc in the list of columns @@ -526,15 +520,15 @@ impl RelationalDB { ))); }; let unique_index = table.indexes.iter().find(|x| &x.cols == cols).map(|x| x.is_unique); - let mut attr = ColumnIndexAttribute::UNSET; + let mut attr = ColumnAttribute::UNSET; if is_autoinc { - attr |= ColumnIndexAttribute::AUTO_INC; + attr |= ColumnAttribute::AUTO_INC; } if let Some(is_unique) = unique_index { attr |= if is_unique { - ColumnIndexAttribute::UNIQUE + ColumnAttribute::UNIQUE } else { - ColumnIndexAttribute::INDEXED + ColumnAttribute::INDEXED }; } Ok(attr) @@ -556,7 +550,7 @@ impl RelationalDB { /// /// NOTE: It loads the data from the table into it before returning #[tracing::instrument(skip(self, tx, index), fields(index=index.name))] - pub fn create_index(&self, tx: &mut MutTxId, index: IndexDef) -> Result { + pub fn create_index(&self, tx: &mut MutTxId, table_id: TableId, index: IndexDef) -> Result { self.inner.create_index_mut_tx(tx, index) } @@ -667,7 +661,12 @@ impl RelationalDB { /// Add a [Sequence] into the database instance, generates a stable [SequenceId] for it that will persist on restart. #[tracing::instrument(skip(self, tx, seq), fields(seq=seq.sequence_name))] - pub fn create_sequence(&self, tx: &mut MutTxId, seq: SequenceDef) -> Result { + pub fn create_sequence( + &mut self, + tx: &mut MutTxId, + table_id: TableId, + seq: SequenceDef, + ) -> Result { self.inner.create_sequence_mut_tx(tx, seq) } @@ -752,35 +751,14 @@ pub(crate) mod tests_utils { mod tests { #![allow(clippy::disallowed_macros)] - use nonempty::NonEmpty; + use super::*; + use crate::db::datastore::system_tables::{ + StIndexRow, StSequenceRow, StTableRow, ST_INDEXES_ID, ST_SEQUENCES_ID, ST_TABLES_ID, + }; + use crate::db::relational_db::tests_utils::make_test_db; use spacetimedb_lib::error::ResultTest; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::db::auth::{StAccess, StTableType}; - use spacetimedb_sats::db::def::{ColumnDef, IndexDef, IndexType, TableDef}; - use std::fs::File; - use std::io::{self, Seek, SeekFrom, Write}; - use std::ops::Range; - use std::path::Path; - use std::sync::{Arc, Mutex}; - use tempfile::TempDir; - - use super::RelationalDB; - use crate::address::Address; - use crate::db::datastore::locking_tx_datastore::IterByColEq; - use crate::db::datastore::system_tables::StIndexRow; - use crate::db::datastore::system_tables::StSequenceRow; - use crate::db::datastore::system_tables::StTableRow; - use crate::db::datastore::system_tables::ST_INDEXES_ID; - use crate::db::datastore::system_tables::ST_SEQUENCES_ID; - use crate::db::message_log::{MessageLog, SegmentView}; - use crate::db::ostorage::sled_object_db::SledObjectDB; - use crate::db::ostorage::ObjectDB; - use crate::db::relational_db::make_default_ostorage; - use crate::db::relational_db::tests_utils::make_test_db; - use crate::db::relational_db::{open_db, ST_TABLES_ID}; - use crate::error::{DBError, DatabaseError, IndexError, LogReplayError}; - use crate::execution_context::ExecutionContext; - use spacetimedb_lib::{AlgebraicType, AlgebraicValue, ProductType}; use spacetimedb_sats::product; fn column(name: &str, ty: AlgebraicType) -> ColumnDef { diff --git a/crates/core/src/db/update.rs b/crates/core/src/db/update.rs index 32ad06a6e22..7a255462dae 100644 --- a/crates/core/src/db/update.rs +++ b/crates/core/src/db/update.rs @@ -57,7 +57,7 @@ pub fn update_database( for index_def in indexes_to_create { system_logger.info(&format!("Creating index `{}`", index_def.name)); - stdb.create_index(tx, index_def)?; + stdb.create_index(tx, index_def.table_id, index_def)?; } } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 6aa405aa3c4..fad3e85cc9e 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -5,15 +5,14 @@ use spacetimedb_lib::{bsatn, ProductValue}; use std::ops::DerefMut; use std::sync::Arc; +use super::scheduler::{ScheduleError, ScheduledReducerId, Scheduler}; +use super::timestamp::Timestamp; use crate::database_instance_context::DatabaseInstanceContext; use crate::database_logger::{BacktraceProvider, LogLevel, Record}; use crate::db::datastore::locking_tx_datastore::{MutTxId, RowId}; use crate::error::{IndexError, NodesError}; use crate::execution_context::ExecutionContext; use crate::util::ResultInspectExt; - -use super::scheduler::{ScheduleError, ScheduledReducerId, Scheduler}; -use super::timestamp::Timestamp; use crate::vm::DbProgram; use spacetimedb_lib::filter::CmpArgs; use spacetimedb_lib::identity::AuthCtx; @@ -22,7 +21,7 @@ use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::buffer::BufWriter; use spacetimedb_sats::db::def::{IndexDef, IndexType}; use spacetimedb_sats::relation::{FieldExpr, FieldName}; -use spacetimedb_sats::{ProductType, Typespace}; +use spacetimedb_sats::{bsatn, ProductType, ProductValue, Typespace}; use spacetimedb_vm::expr::{Code, ColumnOp}; #[derive(Clone)] @@ -251,7 +250,7 @@ impl InstanceEnv { index_type, }; - stdb.create_index(tx, index)?; + stdb.create_index(tx, table_id, index)?; Ok(()) } diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 0125ae94c61..ef8a8c081a1 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -1,3 +1,14 @@ +use std::time::Duration; + +use anyhow::Context; +use bytes::Bytes; +use bytestring::ByteString; +use spacetimedb_lib::de::serde::SeedWrapper; +use spacetimedb_lib::de::DeserializeSeed; +use spacetimedb_lib::{bsatn, Identity}; +use spacetimedb_lib::{ProductValue, ReducerDef}; +use spacetimedb_sats::WithTypespace; + mod host_controller; pub(crate) mod module_host; pub mod scheduler; @@ -7,9 +18,6 @@ pub mod instance_env; mod timestamp; mod wasm_common; -use anyhow::Context; -use bytes::Bytes; -use bytestring::ByteString; use derive_more::Display; use enum_map::Enum; pub use host_controller::{ @@ -17,12 +25,6 @@ pub use host_controller::{ }; pub use module_host::{ModuleHost, NoSuchModule}; pub use module_host::{UpdateDatabaseResult, UpdateDatabaseSuccess}; -use spacetimedb_lib::de::serde::SeedWrapper; -use spacetimedb_lib::de::DeserializeSeed; -use spacetimedb_lib::{bsatn, Identity}; -use spacetimedb_lib::{ProductValue, ReducerDef}; -use spacetimedb_sats::WithTypespace; -use std::time::Duration; pub use timestamp::Timestamp; #[derive(Debug)] diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 180fabbe24d..b895c2f7777 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1,3 +1,13 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use base64::{engine::general_purpose::STANDARD as BASE_64_STD, Engine as _}; +use futures::{Future, FutureExt}; +use indexmap::IndexMap; +use tokio::sync::oneshot; + use super::host_controller::HostThreadpool; use super::{ArgsTuple, EnergyDiff, InvalidReducerArguments, ReducerArgs, ReducerCallResult, ReducerId, Timestamp}; use crate::client::ClientConnectionSender; @@ -16,18 +26,10 @@ use crate::util::lending_pool::{Closed, LendingPool, LentResource, PoolClosed, W use crate::util::notify_once::NotifyOnce; use crate::util::prometheus_handle::{GaugeInc, IntGaugeExt}; use crate::worker_metrics::WORKER_METRICS; -use base64::{engine::general_purpose::STANDARD as BASE_64_STD, Engine as _}; -use futures::{Future, FutureExt}; -use indexmap::IndexMap; use spacetimedb_lib::{Address, ReducerDef, TableDef}; use spacetimedb_primitives::TableId; use spacetimedb_sats::relation::MemTable; use spacetimedb_sats::{ProductValue, Typespace, WithTypespace}; -use std::collections::HashMap; -use std::fmt; -use std::sync::{Arc, Weak}; -use std::time::Duration; -use tokio::sync::oneshot; #[derive(Debug, Default, Clone)] pub struct DatabaseUpdate { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 631277acf00..53f6861b0b3 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -6,9 +6,10 @@ use std::time::Duration; use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::{bsatn, Address, ModuleDef}; -use spacetimedb_sats::db::def::TableDef; use spacetimedb_vm::expr::CrudExpr; +use super::instrumentation::CallTimes; +use super::*; use crate::database_instance_context::DatabaseInstanceContext; use crate::database_logger::{LogLevel, Record, SystemLogger}; use crate::db::datastore::locking_tx_datastore::MutTxId; @@ -29,9 +30,7 @@ use crate::sql; use crate::subscription::module_subscription_actor::{ModuleSubscriptionManager, SubscriptionEventSender}; use crate::util::{const_unwrap, ResultInspectExt}; use crate::worker_metrics::WORKER_METRICS; - -use super::instrumentation::CallTimes; -use super::*; +use spacetimedb_sats::db::def::TableDef; pub trait WasmModule: Send + 'static { type Instance: WasmInstance; diff --git a/crates/core/src/host/wasmer/wasm_instance_env.rs b/crates/core/src/host/wasmer/wasm_instance_env.rs index ff37d131ffc..c3c52eed195 100644 --- a/crates/core/src/host/wasmer/wasm_instance_env.rs +++ b/crates/core/src/host/wasmer/wasm_instance_env.rs @@ -538,7 +538,7 @@ impl WasmInstanceEnv { let name = Self::read_string(&caller, mem, name, name_len)?; // Query the table id. - Ok(caller.data().instance_env.get_table_id(name)?.0) + Ok(caller.data().instance_env.get_table_id(name)?.into()) }) } diff --git a/crates/core/src/sql/ast.rs b/crates/core/src/sql/ast.rs index dbc1d338cec..dcceb697c9d 100644 --- a/crates/core/src/sql/ast.rs +++ b/crates/core/src/sql/ast.rs @@ -1,12 +1,7 @@ -use crate::db::datastore::locking_tx_datastore::MutTxId; -use crate::db::datastore::traits::MutTxDatastore; -use crate::db::relational_db::RelationalDB; -use crate::error::{DBError, PlanError}; -use spacetimedb_primitives::ColumnIndexAttribute; use spacetimedb_sats::db::auth::{StAccess, StTableType}; -use spacetimedb_sats::db::def::{ColumnDefMeta, ProductTypeMeta, TableSchema}; +use spacetimedb_sats::db::def::{ConstraintFlags, Constraints, TableSchema}; +use spacetimedb_sats::db::def::{FieldDef, ProductTypeMeta}; use spacetimedb_sats::db::error::RelationError; -use spacetimedb_sats::relation::{extract_table_field, FieldExpr, FieldName}; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductTypeElement}; use spacetimedb_vm::errors::ErrorVm; use spacetimedb_vm::expr::{ColumnOp, DbType, Expr}; @@ -22,6 +17,12 @@ use sqlparser::parser::Parser; use std::borrow::Cow; use std::collections::HashMap; +use crate::db::datastore::locking_tx_datastore::MutTxId; +use crate::db::datastore::traits::MutTxDatastore; +use crate::db::relational_db::RelationalDB; +use crate::error::{DBError, PlanError}; +use spacetimedb_sats::relation::{extract_table_field, FieldExpr, FieldName}; + /// Simplify to detect features of the syntax we don't support yet /// Because we use [PostgreSqlDialect] in the compiler step it already protect against features /// that are not in the standard SQL-92 but still need to check for completeness @@ -122,12 +123,6 @@ pub enum Join { Inner { rhs: TableSchema, on: OnExpr }, } -#[derive(Clone)] -pub struct FromField { - pub field: FieldName, - pub column: ColumnDefMeta, -} - /// The list of tables in `... FROM table1 [JOIN table2] ...` pub struct From { pub root: TableSchema, @@ -178,13 +173,18 @@ impl From { /// Returns all the fields matching `f` as a `Vec`, /// including the ones inside the joins. - pub fn find_field(&self, f: &str) -> Result, RelationError> { + pub fn find_field(&self, f: &str) -> Result, RelationError> { let field = extract_table_field(f)?; - let fields = self.iter_tables().filter_map(|t| { - let f = t.normalize_field(&field); - t.get_column_by_field(&f).map(|column| FromField { - field: f, - column: column.into(), + let fields = self.iter_tables().flat_map(|t| { + t.columns.iter().filter_map(|column| { + if column.col_name == field.field { + Some(FieldDef { + column: column.clone(), + table_name: field.table.unwrap_or(&t.table_name).to_string(), + }) + } else { + None + } }) }); @@ -193,7 +193,7 @@ impl From { /// Checks if the field `named` matches exactly once in all the tables /// including the ones inside the joins - pub fn resolve_field(&self, named: &str) -> Result { + pub fn resolve_field(&self, named: &str) -> Result { let fields = self.find_field(named)?; match fields.len() { @@ -208,7 +208,7 @@ impl From { 1 => Ok(fields[0].clone()), _ => Err(PlanError::AmbiguousField { field: named.into(), - found: fields.iter().map(|x| x.field.clone()).collect(), + found: fields.into_iter().map(Into::into).collect(), }), } } @@ -252,12 +252,12 @@ fn extract_field(table: &From, of: &SqlExpr) -> Result { let f = table.resolve_field(&x.value)?; - Ok(Some(f.column.column)) + Ok(Some(f.into())) } SqlExpr::CompoundIdentifier(ident) => { let col_name = compound_ident(ident); let f = table.resolve_field(&col_name)?; - Ok(Some(f.column.column)) + Ok(Some(f.into())) } _ => Ok(None), } @@ -290,10 +290,10 @@ fn infer_number(field: Option<&ProductTypeElement>, value: &str, is_long: bool) /// Compiles a [SqlExpr] expression into a [ColumnOp] fn compile_expr_value(table: &From, field: Option<&ProductTypeElement>, of: SqlExpr) -> Result { Ok(ColumnOp::Field(match of { - SqlExpr::Identifier(name) => FieldExpr::Name(table.resolve_field(&name.value)?.field), + SqlExpr::Identifier(name) => FieldExpr::Name(table.resolve_field(&name.value)?.into()), SqlExpr::CompoundIdentifier(ident) => { let col_name = compound_ident(&ident); - table.resolve_field(&col_name)?.field.into() + FieldExpr::Name(table.resolve_field(&col_name)?.into()) } SqlExpr::Value(x) => FieldExpr::Value(match x { Value::Number(value, is_long) => infer_number(field, &value, is_long)?, @@ -753,9 +753,9 @@ fn column_def_type(named: &String, is_null: bool, data_type: &DataType) -> Resul Ok(if is_null { AlgebraicType::option(ty) } else { ty }) } -/// Extract the column attributes into [ColumnIndexAttribute] -fn compile_column_option(col: &SqlColumnDef) -> Result<(bool, ColumnIndexAttribute), PlanError> { - let mut attr = ColumnIndexAttribute::UNSET; +/// Extract the column attributes into [ColumnAttribute] +fn compile_column_option(col: &SqlColumnDef) -> Result<(bool, Constraints), PlanError> { + let mut attr = Constraints::unset(); let mut is_null = false; for x in &col.options { @@ -767,11 +767,11 @@ fn compile_column_option(col: &SqlColumnDef) -> Result<(bool, ColumnIndexAttribu is_null = false; } ColumnOption::Unique { is_primary } => { - attr = if *is_primary { - ColumnIndexAttribute::PRIMARY_KEY + attr = attr.push(if *is_primary { + ConstraintFlags::PRIMARY_KEY } else { - ColumnIndexAttribute::UNIQUE - }; + ConstraintFlags::UNIQUE + }); } ColumnOption::Generated { generated_as, @@ -782,7 +782,7 @@ fn compile_column_option(col: &SqlColumnDef) -> Result<(bool, ColumnIndexAttribu match generated_as { GeneratedAs::ByDefault => { - attr |= ColumnIndexAttribute::IDENTITY; + attr = attr.push(ConstraintFlags::IDENTITY); } x => { return Err(PlanError::Unsupported { diff --git a/crates/core/src/sql/compiler.rs b/crates/core/src/sql/compiler.rs index 6d54f72ba73..89443af1cf3 100644 --- a/crates/core/src/sql/compiler.rs +++ b/crates/core/src/sql/compiler.rs @@ -37,7 +37,7 @@ fn expr_for_projection(table: &From, of: Expr) -> Result { Expr::Ident(x) => { let f = table.resolve_field(&x)?; - Ok(FieldExpr::Name(f.field)) + Ok(FieldExpr::Name(f.into())) } Expr::Value(x) => Ok(FieldExpr::Value(x)), x => unreachable!("Wrong expression in SQL query {:?}", x), @@ -346,12 +346,12 @@ mod tests { ) -> TableId { if let Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound, upper_bound, }) = op { - assert_eq!(col_id, col, "Columns don't match"); + assert_eq!(columns, col.into(), "Columns don't match"); assert_eq!(lower_bound, low_bound, "Lower bound don't match"); assert_eq!(upper_bound, up_bound, "Upper bound don't match"); table.table_id diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 6ef9572708a..1f7247aee52 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,6 +1,6 @@ use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_lib::{ProductType, ProductValue}; use spacetimedb_sats::relation::MemTable; -use spacetimedb_sats::{ProductType, ProductValue}; use spacetimedb_vm::eval::run_ast; use spacetimedb_vm::expr::{CodeResult, CrudExpr, Expr}; use tracing::info; @@ -101,8 +101,8 @@ pub fn run(db: &RelationalDB, tx: &mut MutTxId, sql_text: &str, auth: AuthCtx) - #[cfg(test)] pub(crate) mod tests { use super::*; + use crate::db::datastore::system_tables::{ST_TABLES_ID, ST_TABLES_NAME}; use crate::db::relational_db::tests_utils::make_test_db; - use crate::db::relational_db::{ST_TABLES_ID, ST_TABLES_NAME}; use crate::vm::tests::create_table_with_rows; use itertools::Itertools; use spacetimedb_lib::error::ResultTest; diff --git a/crates/core/src/subscription/query.rs b/crates/core/src/subscription/query.rs index 8f06af24bb1..d0faef062c4 100644 --- a/crates/core/src/subscription/query.rs +++ b/crates/core/src/subscription/query.rs @@ -7,9 +7,9 @@ use crate::sql::execute::execute_single_sql; use crate::subscription::subscription::QuerySet; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_sats::relation::{Column, FieldName, MemTable, RelValue}; -use spacetimedb_sats::{AlgebraicType, DataKey}; +use spacetimedb_sats::AlgebraicType; +use spacetimedb_sats::DataKey; use spacetimedb_vm::expr::{self, Crud, CrudExpr, DbType, QueryExpr, SourceExpr}; - pub const SUBSCRIBE_TO_ALL_QUERY: &str = "SELECT * FROM *"; pub enum QueryDef { @@ -166,10 +166,11 @@ mod tests { use itertools::Itertools; use spacetimedb_lib::error::ResultTest; use spacetimedb_lib::Identity; - use spacetimedb_primitives::*; + use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::data_key::ToDataKey; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::*; + use spacetimedb_sats::relation::FieldName; use spacetimedb_sats::{product, ProductType, ProductValue}; use spacetimedb_vm::dsl::{db_table, mem_table, scalar}; use spacetimedb_vm::operator::OpCmp; @@ -179,7 +180,7 @@ mod tests { tx: &mut MutTxId, name: &str, schema: &[(&str, AlgebraicType)], - indexes: &[(u32, &str)], + indexes: &[(ColId, &str)], ) -> ResultTest { let table_name = name.to_string(); let table_type = StTableType::User; @@ -196,7 +197,7 @@ mod tests { let indexes = indexes .iter() - .map(|(col_id, index_name)| IndexDef::new(index_name.to_string(), 0.into(), ColId(*col_id), false)) + .map(|(col_id, index_name)| IndexDef::new(index_name.to_string(), 0.into(), *col_id, false)) .collect_vec(); let schema = TableDef { @@ -438,7 +439,7 @@ mod tests { // Create table [test] with index on [b] let schema = &[("a", AlgebraicType::U64), ("b", AlgebraicType::U64)]; - let indexes = &[(1, "b")]; + let indexes = &[(1.into(), "b")]; let table_id = create_table(&db, &mut tx, "test", schema, indexes)?; let sql = "select * from test where b = 3"; @@ -495,7 +496,7 @@ mod tests { // Create table [lhs] with index on [id] let schema = &[("id", AlgebraicType::I32), ("x", AlgebraicType::I32)]; - let indexes = &[(0, "id")]; + let indexes = &[(0.into(), "id")]; let lhs_id = create_table(&db, &mut tx, "lhs", schema, indexes)?; // Create table [rhs] with no indexes @@ -887,7 +888,11 @@ mod tests { ("timestamp", AlgebraicType::U64), ("dimension", AlgebraicType::U32), ]; - let indexes = &[(0, "entity_id"), (1, "location_x"), (2, "location_z")]; + let indexes = &[ + (0.into(), "entity_id"), + (1.into(), "location_x"), + (2.into(), "location_z"), + ]; create_table(&db, &mut tx, "MobileEntityState", schema, indexes)?; // Create table [EnemyState] @@ -898,7 +903,7 @@ mod tests { ("type", AlgebraicType::I32), ("direction", AlgebraicType::I32), ]; - let indexes = &[(0, "entity_id")]; + let indexes = &[(0.into(), "entity_id")]; create_table(&db, &mut tx, "EnemyState", schema, indexes)?; let sql_insert = "\ @@ -985,12 +990,12 @@ mod tests { // Create table [lhs] with indexes on [id] and [x] let schema = &[("id", AlgebraicType::U64), ("x", AlgebraicType::I32)]; - let indexes = &[(0, "id"), (1, "x")]; + let indexes = &[(ColId(0), "id"), (ColId(1), "x")]; create_table(&db, &mut tx, "lhs", schema, indexes)?; // Create table [rhs] with indexes on [id] and [y] let schema = &[("id", AlgebraicType::U64), ("y", AlgebraicType::I32)]; - let indexes = &[(0, "id"), (1, "y")]; + let indexes = &[(ColId(0), "id"), (ColId(1), "y")]; create_table(&db, &mut tx, "rhs", schema, indexes)?; let auth = AuthCtx::for_testing(); diff --git a/crates/core/src/subscription/subscription.rs b/crates/core/src/subscription/subscription.rs index 1062b509cfc..301c074c851 100644 --- a/crates/core/src/subscription/subscription.rs +++ b/crates/core/src/subscription/subscription.rs @@ -25,12 +25,6 @@ use anyhow::Context; use derive_more::{Deref, DerefMut, From, IntoIterator}; -use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::PrimaryKey; -use spacetimedb_sats::db::auth::{StAccess, StTableType}; -use spacetimedb_sats::relation::{DbTable, MemTable, RelValue}; -use spacetimedb_sats::{AlgebraicValue, DataKey, ProductValue}; -use spacetimedb_vm::expr::{self, IndexJoin, QueryExpr, SourceExpr}; use std::collections::{btree_set, BTreeSet, HashMap, HashSet}; use std::ops::Deref; @@ -42,6 +36,12 @@ use crate::{ db::relational_db::RelationalDB, host::module_host::{DatabaseTableUpdate, DatabaseUpdate, TableOp}, }; +use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_lib::PrimaryKey; +use spacetimedb_sats::db::auth::{StAccess, StTableType}; +use spacetimedb_sats::relation::{DbTable, MemTable, RelValue}; +use spacetimedb_sats::{AlgebraicValue, DataKey, ProductValue}; +use spacetimedb_vm::expr::{self, IndexJoin, QueryExpr, SourceExpr}; use super::query; diff --git a/crates/core/src/vm.rs b/crates/core/src/vm.rs index d51b819479d..f80d93631d2 100644 --- a/crates/core/src/vm.rs +++ b/crates/core/src/vm.rs @@ -1,15 +1,21 @@ //! The [DbProgram] that execute arbitrary queries & code against the database. +use std::collections::HashMap; +use std::ops::RangeBounds; + +use itertools::Itertools; +use tracing::debug; + use crate::db::cursor::{CatalogCursor, IndexCursor, TableCursor}; use crate::db::datastore::locking_tx_datastore::{IterByColEq, MutTxId}; use crate::db::relational_db::RelationalDB; use crate::execution_context::ExecutionContext; -use itertools::Itertools; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::{ColumnDef, IndexDef, ProductTypeMeta, TableDef}; -use spacetimedb_sats::relation::{DbTable, FieldExpr, FieldName, Relation}; -use spacetimedb_sats::relation::{Header, MemTable, RelIter, RelValue, RowCount, Table}; +use spacetimedb_sats::relation::{ + DbTable, FieldExpr, FieldName, Header, MemTable, RelIter, RelValue, Relation, RowCount, Table, +}; use spacetimedb_sats::{AlgebraicValue, ProductValue}; use spacetimedb_vm::env::EnvDb; use spacetimedb_vm::errors::ErrorVm; @@ -17,9 +23,6 @@ use spacetimedb_vm::eval::IterRows; use spacetimedb_vm::expr::*; use spacetimedb_vm::program::{ProgramRef, ProgramVm}; use spacetimedb_vm::rel_ops::RelOps; -use std::collections::HashMap; -use std::ops::RangeBounds; -use tracing::debug; //TODO: This is partially duplicated from the `vm` crate to avoid borrow checker issues //and pull all that crate in core. Will be revisited after trait refactor @@ -37,10 +40,14 @@ pub fn build_query<'a>( result = match op { Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound, upper_bound, - }) if db_table => iter_by_col_range(ctx, stdb, tx, table, col_id, (lower_bound, upper_bound))?, + }) if db_table => { + assert_eq!(columns.len(), 1, "Only support single column IndexScan"); + let col_id = columns.head; + iter_by_col_range(ctx, stdb, tx, table, col_id, (lower_bound, upper_bound))? + } Query::IndexScan(index_scan) => { let header = result.head().clone(); let cmp: ColumnOp = index_scan.into(); @@ -362,6 +369,9 @@ impl<'db, 'tx> DbProgram<'db, 'tx> { self.db.drop_sequence(self.tx, id)?; } } + DbType::Constraint => { + todo!("Will be finished in PR#267") + } } Ok(Code::Pass) @@ -530,15 +540,15 @@ pub(crate) mod tests { use crate::db::datastore::system_tables::{ st_columns_schema, st_indexes_schema, st_sequences_schema, st_table_schema, StColumnFields, StColumnRow, StIndexFields, StIndexRow, StSequenceFields, StSequenceRow, StTableFields, StTableRow, ST_COLUMNS_ID, - ST_SEQUENCES_ID, ST_TABLES_ID, + ST_COLUMNS_NAME, ST_INDEXES_NAME, ST_SEQUENCES_ID, ST_SEQUENCES_NAME, ST_TABLES_ID, ST_TABLES_NAME, }; use crate::db::relational_db::tests_utils::make_test_db; - use crate::db::relational_db::{ST_COLUMNS_NAME, ST_INDEXES_NAME, ST_SEQUENCES_NAME, ST_TABLES_NAME}; use crate::execution_context::ExecutionContext; use nonempty::NonEmpty; use spacetimedb_lib::error::ResultTest; use spacetimedb_primitives::TableId; - use spacetimedb_sats::db::def::IndexType; + use spacetimedb_sats::db::auth::{StAccess, StTableType}; + use spacetimedb_sats::db::def::{ColumnDef, IndexDef, IndexType}; use spacetimedb_sats::relation::{DbTable, FieldName}; use spacetimedb_sats::{product, AlgebraicType, ProductType, ProductValue}; use spacetimedb_vm::dsl::*; @@ -687,7 +697,7 @@ pub(crate) mod tests { .with_select_cmp( OpCmp::Eq, FieldName::named(ST_COLUMNS_NAME, StColumnFields::TableId.name()), - scalar(ST_COLUMNS_ID.0), + scalar(ST_COLUMNS_ID), ) .with_select_cmp( OpCmp::Eq, @@ -728,7 +738,7 @@ pub(crate) mod tests { let mut tx = db.begin_tx(); let index = IndexDef::new("idx_1".into(), table_id, 0.into(), true); - let index_id = db.create_index(&mut tx, index)?; + let index_id = db.create_index(&mut tx, table_id, index)?; let p = &mut DbProgram::new(&ctx, &db, &mut tx, AuthCtx::for_testing()); diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index af03ee2f487..5a619406bf2 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -35,7 +35,6 @@ hex.workspace = true itertools.workspace = true serde = { workspace = true, optional = true } serde_with = {workspace = true, optional = true } -sha3.workspace = true thiserror.workspace = true tracing.workspace = true diff --git a/crates/lib/src/identity.rs b/crates/lib/src/identity.rs index ff7786afb19..f6ab907e9db 100644 --- a/crates/lib/src/identity.rs +++ b/crates/lib/src/identity.rs @@ -1,8 +1,8 @@ +use std::{fmt, str::FromStr}; + use spacetimedb_bindings_macro::{Deserialize, Serialize}; use spacetimedb_sats::hex::HexString; use spacetimedb_sats::{hash, impl_st, AlgebraicType}; -use std::fmt; -use std::str::FromStr; #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct AuthCtx { diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 7483e243d56..f8db502a0eb 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -1,7 +1,7 @@ use anyhow::Context; -use spacetimedb_primitives::{ColId, ColumnIndexAttribute}; +use spacetimedb_primitives::ColId; use spacetimedb_sats::db::auth::{StAccess, StTableType}; -use spacetimedb_sats::db::def::{ColumnDef, IndexType, AUTO_TABLE_ID}; +use spacetimedb_sats::db::def::{ColumnDef, IndexDef, IndexType, AUTO_TABLE_ID}; use spacetimedb_sats::{impl_serialize, WithTypespace}; use std::iter; @@ -85,13 +85,6 @@ impl std::fmt::Display for VersionTuple { extern crate self as spacetimedb_lib; -#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord, de::Deserialize, ser::Serialize)] -pub struct IndexDef { - pub name: String, - pub index_type: IndexType, - pub cols: Vec, -} - //WARNING: Change this structure(or any of their members) is an ABI change. #[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord, de::Deserialize, ser::Serialize)] #[sats(crate = crate)] @@ -99,7 +92,7 @@ pub struct TableDef { pub name: String, /// data should always point to a ProductType in the typespace pub data: sats::AlgebraicTypeRef, - pub column_attrs: Vec, + pub column_attrs: Vec, pub indexes: Vec, pub table_type: StTableType, pub table_access: StAccess, @@ -131,7 +124,7 @@ impl TableDef { let index_for_column = table.indexes.iter().find(|index| { // Ignore multi-column indexes - matches!(*index.cols, [index_col_id] if index_col_id as usize == col_id) + matches!(index.cols.split_first(), (index_col_id,[]) if index_col_id.idx() == col_id) }); // If there's an index defined for this column already, use it, @@ -163,17 +156,12 @@ impl TableDef { // Multi-column indexes cannot be unique (yet), so just add them. let multi_col_indexes = table.indexes.iter().filter_map(|index| { - if let [a, b, rest @ ..] = &*index.cols { + if let (a, [b, rest @ ..]) = index.cols.split_first() { let idx = spacetimedb_sats::db::def::IndexDef::new_cols( index.name.clone(), AUTO_TABLE_ID, false, - ( - ColId::from(*a), - iter::once(ColId::from(*b)) - .chain(rest.iter().copied().map(Into::into)) - .collect(), - ), + (*a, iter::once(*b).chain(rest.iter().copied()).collect()), ); Some(idx) diff --git a/crates/sats/src/db/attr.rs b/crates/sats/src/db/attr.rs index 57c3604010d..8b7b739f4cd 100644 --- a/crates/sats/src/db/attr.rs +++ b/crates/sats/src/db/attr.rs @@ -1,9 +1,86 @@ -use crate::{de, impl_deserialize, impl_serialize}; -pub use spacetimedb_primitives::ColumnIndexAttribute; +// This file is `included` also in `crates/bindings-macro/src/lib.rs` to avoid duplicating +// So can't import things from `sats` crate +#![allow(dead_code)] +use bitflags::bitflags; -impl_deserialize!([] ColumnIndexAttribute, de => - Self::from_bits(de.deserialize_u8()?) - .ok_or_else(|| de::Error::custom("invalid bitflags for `ColumnIndexAttribute`")) -); +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Eq, PartialEq)] +pub enum AttributeKind { + UNSET, + /// Index no unique + INDEXED, + /// Auto Increment + AUTO_INC, + /// Index unique + UNIQUE, + /// Unique + AutoInc + IDENTITY, + /// Primary key column (implies Unique) + PRIMARY_KEY, + /// PrimaryKey + AutoInc + PRIMARY_KEY_AUTO, + /// PrimaryKey + Identity + PRIMARY_KEY_IDENTITY, +} -impl_serialize!([] ColumnIndexAttribute, (self, ser) => ser.serialize_u8(self.bits())); +// This indeed is only used for defining columns + constraints AND/OR auto_inc, +// and is distinct to `Constraints` in `sats/db/def.rs` +bitflags! { + #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] + pub struct ColumnAttribute: u8 { + const UNSET = Self::empty().bits(); + /// Index no unique + const INDEXED = 0b0001; + /// Generate the next [Sequence] + const AUTO_INC = 0b0010; + /// Index unique + const UNIQUE = Self::INDEXED.bits() | 0b0100; + /// Unique + AutoInc + const IDENTITY = Self::UNIQUE.bits() | Self::AUTO_INC.bits(); + /// Primary key column (implies Unique) + const PRIMARY_KEY = Self::UNIQUE.bits() | 0b1000; + /// PrimaryKey + AutoInc + const PRIMARY_KEY_AUTO = Self::PRIMARY_KEY.bits() | Self::AUTO_INC.bits(); + /// PrimaryKey + Identity + const PRIMARY_KEY_IDENTITY = Self::PRIMARY_KEY.bits() | Self::IDENTITY.bits() ; + } +} + +impl ColumnAttribute { + /// Checks if either 'IDENTITY' or 'PRIMARY_KEY_AUTO' constraints are set because the imply the use of + /// auto increment sequence. + pub const fn has_autoinc(&self) -> bool { + self.contains(ColumnAttribute::IDENTITY) + || self.contains(ColumnAttribute::PRIMARY_KEY_AUTO) + || self.contains(ColumnAttribute::AUTO_INC) + } + + pub const fn has_unique(&self) -> bool { + self.contains(ColumnAttribute::UNIQUE) + } + + pub const fn has_primary(&self) -> bool { + self.contains(ColumnAttribute::IDENTITY) + || self.contains(ColumnAttribute::PRIMARY_KEY) + || self.contains(ColumnAttribute::PRIMARY_KEY_AUTO) + } + + /// Returns the [ColumnAttribute] of constraints as an enum variant. + /// + /// NOTE: This represent the higher possible representation of a constraints, so for example + /// `IDENTITY` imply that is `INDEXED, UNIQUE` + pub fn kind(&self) -> AttributeKind { + match self { + x if x == &ColumnAttribute::UNSET => AttributeKind::UNSET, + x if x == &ColumnAttribute::INDEXED => AttributeKind::INDEXED, + x if x == &ColumnAttribute::UNIQUE => AttributeKind::UNIQUE, + x if x == &ColumnAttribute::AUTO_INC => AttributeKind::AUTO_INC, + x if x == &ColumnAttribute::IDENTITY => AttributeKind::IDENTITY, + x if x == &ColumnAttribute::PRIMARY_KEY => AttributeKind::PRIMARY_KEY, + x if x == &ColumnAttribute::PRIMARY_KEY_AUTO => AttributeKind::PRIMARY_KEY_AUTO, + x if x == &ColumnAttribute::PRIMARY_KEY_IDENTITY => AttributeKind::PRIMARY_KEY_IDENTITY, + x => unreachable!("Unexpected value {x:?}"), + } + } +} diff --git a/crates/sats/src/db/def.rs b/crates/sats/src/db/def.rs index e82d5f1426d..f30d70e5698 100644 --- a/crates/sats/src/db/def.rs +++ b/crates/sats/src/db/def.rs @@ -1,3 +1,7 @@ +use derive_more::Display; +use nonempty::NonEmpty; +use spacetimedb_primitives::*; + use crate::db::auth::{StAccess, StTableType}; use crate::de::BasicVecVisitor; use crate::product_value::InvalidFieldError; @@ -5,9 +9,6 @@ use crate::relation::{Column, DbTable, FieldName, FieldOnly, Header, TableField} use crate::ser::SerializeArray; use crate::{de, impl_deserialize, impl_serialize, ser, AlgebraicValue, ProductValue}; use crate::{AlgebraicType, ProductType, ProductTypeElement}; -use derive_more::Display; -use nonempty::NonEmpty; -use spacetimedb_primitives::{AttributeKind, ColId, ColumnIndexAttribute, ConstraintId, IndexId, SequenceId, TableId}; /// The default preallocation amount for sequences. pub const SEQUENCE_PREALLOCATION_AMOUNT: i128 = 4_096; @@ -332,24 +333,118 @@ impl From<&ColumnSchema> for ProductTypeElement { #[derive(Clone)] pub struct ColumnDefMeta { pub column: ProductTypeElement, - pub attr: ColumnIndexAttribute, + pub attr: Constraints, pub pos: usize, } -impl From<&ColumnSchema> for ColumnDefMeta { - fn from(value: &ColumnSchema) -> Self { +/// Describe the columns + meta attributes +/// TODO(cloutiertyler): This type should be deprecated and replaced with +/// ColumnDef or ColumnSchema where appropriate +#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)] +pub struct ProductTypeMeta { + pub columns: ProductType, + pub attr: Vec, +} + +impl ProductTypeMeta { + pub fn new(columns: ProductType) -> Self { Self { - column: ProductTypeElement::from(value), - // TODO(cloutiertyler): !!! This is not correct !!! We do not have the information regarding constraints here. - // We should remove this field from the ColumnDef struct. - attr: if value.is_autoinc { - ColumnIndexAttribute::AUTO_INC - } else { - ColumnIndexAttribute::UNSET - }, - pos: value.col_id.idx(), + attr: vec![Constraints::unset(); columns.elements.len()], + columns, } } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + attr: Vec::with_capacity(capacity), + columns: ProductType::new(Vec::with_capacity(capacity)), + } + } + + pub fn clear(&mut self) { + self.columns.elements.clear(); + self.attr.clear(); + } + + pub fn push(&mut self, name: &str, ty: AlgebraicType, attr: Constraints) { + self.columns + .elements + .push(ProductTypeElement::new(ty, Some(name.to_string()))); + self.attr.push(attr); + } + + /// Removes the data at position `index` and returns it. + /// + /// # Panics + /// + /// If `index` is out of bounds. + pub fn remove(&mut self, index: usize) -> (ProductTypeElement, Constraints) { + (self.columns.elements.remove(index), self.attr.remove(index)) + } + + /// Return mutable references to the data at position `index`, or `None` if + /// the index is out of bounds. + pub fn get_mut(&mut self, index: usize) -> Option<(&mut ProductTypeElement, &mut Constraints)> { + self.columns + .elements + .get_mut(index) + .and_then(|pte| self.attr.get_mut(index).map(|attr| (pte, attr))) + } + + pub fn with_attributes(iter: impl Iterator) -> Self { + let mut columns = Vec::new(); + let mut attrs = Vec::new(); + for (col, attr) in iter { + columns.push(col); + attrs.push(attr); + } + Self { + attr: attrs, + columns: ProductType::new(columns), + } + } + + pub fn len(&self) -> usize { + self.columns.elements.len() + } + + pub fn is_empty(&self) -> bool { + self.columns.elements.is_empty() + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.columns + .elements + .iter() + .zip(self.attr.iter()) + .enumerate() + .map(|(pos, (column, attr))| ColumnDefMeta { + column: column.clone(), + attr: *attr, + pos, + }) + } + + pub fn with_defaults<'a>( + &'a self, + row: &'a mut ProductValue, + ) -> impl Iterator + 'a { + self.iter() + .zip(row.elements.iter_mut()) + .filter(|(col, _)| col.attr.has_autoinc()) + } +} + +impl From for ProductTypeMeta { + fn from(value: ProductType) -> Self { + ProductTypeMeta::new(value) + } +} + +impl From for ProductType { + fn from(value: ProductTypeMeta) -> Self { + value.columns + } } /// This type is just the [ColumnSchema] without the autoinc fields @@ -374,7 +469,7 @@ impl From for ColumnDef { pub struct ConstraintSchema { pub constraint_id: ConstraintId, pub constraint_name: String, - pub kind: Constraints, + pub constraints: Constraints, pub table_id: TableId, pub columns: NonEmpty, } @@ -444,7 +539,7 @@ impl TableSchema { columns .map(|pos| { self.get_column(usize::from(pos)).ok_or(InvalidFieldError { - index: pos.idx(), + col_pos: pos, name: None, }) }) @@ -570,13 +665,7 @@ impl From<&TableSchema> for TableDef { impl From for TableDef { fn from(value: TableSchema) -> Self { - Self { - table_name: value.table_name, - columns: value.columns.into_iter().map(Into::into).collect(), - indexes: value.indexes.into_iter().map(Into::into).collect(), - table_type: value.table_type, - table_access: value.table_access, - } + (&value).into() } } @@ -605,113 +694,3 @@ impl From for ProductTypeElement { ProductTypeElement::new(value.column.col_type, Some(f.to_string())) } } - -/// Describe the columns + meta attributes -/// TODO(cloutiertyler): This type should be deprecated and replaced with -/// ColumnDef or ColumnSchema where appropriate -#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)] -pub struct ProductTypeMeta { - pub columns: ProductType, - pub attr: Vec, -} - -impl ProductTypeMeta { - pub fn new(columns: ProductType) -> Self { - Self { - attr: vec![ColumnIndexAttribute::UNSET; columns.elements.len()], - columns, - } - } - - pub fn with_capacity(capacity: usize) -> Self { - Self { - attr: Vec::with_capacity(capacity), - columns: ProductType::new(Vec::with_capacity(capacity)), - } - } - - pub fn clear(&mut self) { - self.columns.elements.clear(); - self.attr.clear(); - } - - pub fn push(&mut self, name: &str, ty: AlgebraicType, attr: ColumnIndexAttribute) { - self.columns - .elements - .push(ProductTypeElement::new(ty, Some(name.to_string()))); - self.attr.push(attr); - } - - /// Removes the data at position `index` and returns it. - /// - /// # Panics - /// - /// If `index` is out of bounds. - pub fn remove(&mut self, index: usize) -> (ProductTypeElement, ColumnIndexAttribute) { - (self.columns.elements.remove(index), self.attr.remove(index)) - } - - /// Return mutable references to the data at position `index`, or `None` if - /// the index is out of bounds. - pub fn get_mut(&mut self, index: usize) -> Option<(&mut ProductTypeElement, &mut ColumnIndexAttribute)> { - self.columns - .elements - .get_mut(index) - .and_then(|pte| self.attr.get_mut(index).map(|attr| (pte, attr))) - } - - pub fn with_attributes(iter: impl Iterator) -> Self { - let mut columns = Vec::new(); - let mut attrs = Vec::new(); - for (col, attr) in iter { - columns.push(col); - attrs.push(attr); - } - Self { - attr: attrs, - columns: ProductType::new(columns), - } - } - - pub fn len(&self) -> usize { - self.columns.elements.len() - } - - pub fn is_empty(&self) -> bool { - self.columns.elements.is_empty() - } - - pub fn iter(&self) -> impl Iterator + '_ { - self.columns - .elements - .iter() - .zip(self.attr.iter()) - .enumerate() - .map(|(pos, (column, attr))| ColumnDefMeta { - column: column.clone(), - attr: *attr, - pos, - }) - } - - pub fn with_defaults<'a>( - &'a self, - row: &'a mut ProductValue, - ) -> impl Iterator + 'a { - self.iter() - .zip(row.elements.iter_mut()) - .filter(|(col, _)| col.attr.has_autoinc()) - } -} - -impl From for ProductTypeMeta { - fn from(value: ProductType) -> Self { - ProductTypeMeta::new(value) - } -} - -impl From for ProductType { - fn from(value: ProductTypeMeta) -> Self { - value.columns - } -} diff --git a/crates/sats/src/db/mod.rs b/crates/sats/src/db/mod.rs index bee72d5d586..e321dbfc9f4 100644 --- a/crates/sats/src/db/mod.rs +++ b/crates/sats/src/db/mod.rs @@ -1,6 +1,21 @@ -//! Defines all the typed database objects & schemas. +//! Defines all the typed database objects & schemas +use crate::db::attr::ColumnAttribute; +use crate::{de, impl_deserialize, impl_serialize}; pub mod attr; pub mod auth; pub mod def; pub mod error; + +impl TryFrom for ColumnAttribute { + type Error = (); + + fn try_from(v: u8) -> Result { + Self::from_bits(v).ok_or(()) + } +} + +impl_deserialize!([] ColumnAttribute, de => Self::from_bits(de.deserialize_u8()?) + .ok_or_else(|| de::Error::custom("invalid bitflags for ColumnIndexAttribute"))); + +impl_serialize!([] ColumnAttribute, (self, ser) => ser.serialize_u8(self.bits())); diff --git a/crates/sats/src/lib.rs b/crates/sats/src/lib.rs index 5a45700326c..b9be642827b 100644 --- a/crates/sats/src/lib.rs +++ b/crates/sats/src/lib.rs @@ -33,7 +33,7 @@ pub use algebraic_value::{AlgebraicValue, F32, F64}; pub use array_type::ArrayType; pub use array_value::ArrayValue; pub use builtin_type::BuiltinType; -pub use data_key::DataKey; +pub use data_key::{DataKey, ToDataKey}; pub use map_type::MapType; pub use map_value::MapValue; pub use product_type::ProductType; diff --git a/crates/sats/src/product_value.rs b/crates/sats/src/product_value.rs index a131484a215..e7019f77426 100644 --- a/crates/sats/src/product_value.rs +++ b/crates/sats/src/product_value.rs @@ -64,10 +64,10 @@ impl crate::Value for ProductValue { /// An error that occurs when a field, of a product value, is accessed that doesn't exist. #[derive(thiserror::Error, Debug, Copy, Clone)] -#[error("Field at position {index} named: {name:?} not found or has an invalid type")] +#[error("Field at position {col_pos} named: {name:?} not found or has an invalid type")] pub struct InvalidFieldError { - /// The claimed index of the field within the product value. - pub index: usize, + /// The claimed col_pos of the field within the product value. + pub col_pos: ColId, /// The name of the field, if any. pub name: Option<&'static str>, } @@ -76,8 +76,11 @@ impl ProductValue { /// Borrow the value at field of `self` indentified by `index`. /// /// The `name` is non-functional and is only used for error-messages. - pub fn get_field(&self, index: usize, name: Option<&'static str>) -> Result<&AlgebraicValue, InvalidFieldError> { - self.elements.get(index).ok_or(InvalidFieldError { index, name }) + pub fn get_field(&self, col_pos: usize, name: Option<&'static str>) -> Result<&AlgebraicValue, InvalidFieldError> { + self.elements.get(col_pos).ok_or(InvalidFieldError { + col_pos: col_pos.into(), + name, + }) } /// This function is used to project fields based on the provided `indexes`. @@ -126,11 +129,14 @@ impl ProductValue { /// and then runs it through the function `f` which possibly returns a `T` derived from `value`. pub fn extract_field<'a, T>( &'a self, - index: usize, + col_pos: usize, name: Option<&'static str>, f: impl 'a + Fn(&'a AlgebraicValue) -> Option, ) -> Result { - f(self.get_field(index, name)?).ok_or(InvalidFieldError { index, name }) + f(self.get_field(col_pos, name)?).ok_or(InvalidFieldError { + col_pos: col_pos.into(), + name, + }) } /// Interprets the value at field of `self` identified by `index` as a `bool`. @@ -178,7 +184,7 @@ impl ProductValue { self.extract_field(index, named, |f| f.as_bytes()) } - /// Interprets the value at field of `self` identified by `index` as a array. + /// Interprets the value at field of `self` identified by `index` as an `ArrayValue`. pub fn field_as_array(&self, index: usize, named: Option<&'static str>) -> Result<&ArrayValue, InvalidFieldError> { self.extract_field(index, named, |f| f.as_array()) } diff --git a/crates/vm/src/expr.rs b/crates/vm/src/expr.rs index 3f2ec8e24b2..54f52d70beb 100644 --- a/crates/vm/src/expr.rs +++ b/crates/vm/src/expr.rs @@ -1,5 +1,14 @@ use derive_more::From; +use nonempty::NonEmpty; +use std::cmp::Ordering; +use std::collections::{HashMap, VecDeque}; +use std::fmt; +use std::ops::Bound; +use crate::errors::{ErrorKind, ErrorLang, ErrorType, ErrorVm}; +use crate::functions::{FunDef, Param}; +use crate::operator::{Op, OpCmp, OpLogic, OpQuery}; +use crate::types::Ty; use spacetimedb_lib::Identity; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::algebraic_type::AlgebraicType; @@ -12,15 +21,6 @@ use spacetimedb_sats::relation::{ }; use spacetimedb_sats::satn::Satn; use spacetimedb_sats::{ProductValue, Typespace, WithTypespace}; -use std::cmp::Ordering; -use std::collections::{HashMap, VecDeque}; -use std::fmt; -use std::ops::Bound; - -use crate::errors::{ErrorKind, ErrorLang, ErrorType, ErrorVm}; -use crate::functions::{FunDef, Param}; -use crate::operator::{Op, OpCmp, OpLogic, OpQuery}; -use crate::types::Ty; /// A `index` into the list of [Fun] pub type FunctionId = usize; @@ -229,8 +229,10 @@ impl From for Box { impl From for ColumnOp { fn from(value: IndexScan) -> Self { let table = value.table; - let col_id = value.col_id; - let field = table.head.fields[col_id.idx()].field.clone(); + let columns = value.columns; + assert_eq!(columns.len(), 1, "Not yet supported multi-column predicates"); + + let field = table.head.fields[usize::from(columns.head)].field.clone(); match (value.lower_bound, value.upper_bound) { // Inclusive lower bound => field >= value (Bound::Included(value), Bound::Unbounded) => ColumnOp::Cmp { @@ -260,13 +262,13 @@ impl From for ColumnOp { (lower_bound, upper_bound) => { let lhs = IndexScan { table: table.clone(), - col_id, + columns: columns.clone(), lower_bound, upper_bound: Bound::Unbounded, }; let rhs = IndexScan { table, - col_id, + columns, lower_bound: Bound::Unbounded, upper_bound, }; @@ -327,6 +329,15 @@ impl SourceExpr { } } +impl From for Table { + fn from(value: SourceExpr) -> Self { + match value { + SourceExpr::MemTable(t) => Table::MemTable(t), + SourceExpr::DbTable(t) => Table::DbTable(t), + } + } +} + impl Relation for SourceExpr { fn head(&self) -> &Header { match self { @@ -352,15 +363,6 @@ impl From for SourceExpr { } } -impl From for Table { - fn from(value: SourceExpr) -> Self { - match value { - SourceExpr::MemTable(t) => Table::MemTable(t), - SourceExpr::DbTable(t) => Table::DbTable(t), - } - } -} - impl From<&TableSchema> for SourceExpr { fn from(value: &TableSchema) -> Self { SourceExpr::DbTable(DbTable::new( @@ -420,6 +422,7 @@ pub enum DbType { Table, Index, Sequence, + Constraint, } #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] @@ -489,7 +492,7 @@ impl CrudExpr { #[derive(Debug, Clone, Eq, PartialEq)] pub struct IndexScan { pub table: DbTable, - pub col_id: ColId, + pub columns: NonEmpty, pub lower_bound: Bound, pub upper_bound: Bound, } @@ -536,7 +539,7 @@ impl Ord for IndexScan { return order; }; - let order = self.col_id.cmp(&other.col_id); + let order = self.columns.cmp(&other.columns); let Ordering::Equal = order else { return order; }; @@ -731,12 +734,12 @@ impl QueryExpr { // Generate an index scan for an equality predicate if this is the first operator. // Otherwise generate a select. // TODO: Replace these methods with a proper query optimization pass. - pub fn with_index_eq(mut self, table: DbTable, col_id: ColId, value: AlgebraicValue) -> Self { + pub fn with_index_eq(mut self, table: DbTable, columns: NonEmpty, value: AlgebraicValue) -> Self { // if this is the first operator in the list, generate index scan let Some(query) = self.query.pop() else { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Bound::Included(value.clone()), upper_bound: Bound::Included(value), })); @@ -755,14 +758,14 @@ impl QueryExpr { }, .. }) if table.table_id != rhs_table_id => { - self = self.with_index_eq(table, col_id, value); + self = self.with_index_eq(table, columns, value); self.query.push(query); self } // try to push below join's rhs Query::JoinInner(JoinExpr { rhs, col_lhs, col_rhs }) => { self.query.push(Query::JoinInner(JoinExpr { - rhs: rhs.with_index_eq(table, col_id, value), + rhs: rhs.with_index_eq(table, columns, value), col_lhs, col_rhs, })); @@ -775,7 +778,7 @@ impl QueryExpr { lhs: filter.into(), rhs: IndexScan { table, - col_id, + columns, lower_bound: Bound::Included(value.clone()), upper_bound: Bound::Included(value), } @@ -789,7 +792,7 @@ impl QueryExpr { self.query.push(Query::Select( IndexScan { table, - col_id, + columns, lower_bound: Bound::Included(value.clone()), upper_bound: Bound::Included(value), } @@ -806,7 +809,7 @@ impl QueryExpr { pub fn with_index_lower_bound( mut self, table: DbTable, - col_id: ColId, + columns: NonEmpty, value: AlgebraicValue, inclusive: bool, ) -> Self { @@ -814,7 +817,7 @@ impl QueryExpr { let Some(query) = self.query.pop() else { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Self::bound(value, inclusive), upper_bound: Bound::Unbounded, })); @@ -833,14 +836,14 @@ impl QueryExpr { }, .. }) if table.table_id != rhs_table_id => { - self = self.with_index_lower_bound(table, col_id, value, inclusive); + self = self.with_index_lower_bound(table, columns, value, inclusive); self.query.push(query); self } // try to push below join's rhs Query::JoinInner(JoinExpr { rhs, col_lhs, col_rhs }) => { self.query.push(Query::JoinInner(JoinExpr { - rhs: rhs.with_index_lower_bound(table, col_id, value, inclusive), + rhs: rhs.with_index_lower_bound(table, columns, value, inclusive), col_lhs, col_rhs, })); @@ -848,14 +851,14 @@ impl QueryExpr { } // merge with a preceding upper bounded index scan (inclusive) Query::IndexScan(IndexScan { - col_id: lhs_col_id, + columns: lhs_col_id, lower_bound: Bound::Unbounded, upper_bound: Bound::Included(upper), .. - }) if col_id == lhs_col_id => { + }) if columns == lhs_col_id => { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Self::bound(value, inclusive), upper_bound: Bound::Included(upper), })); @@ -863,14 +866,14 @@ impl QueryExpr { } // merge with a preceding upper bounded index scan (exclusive) Query::IndexScan(IndexScan { - col_id: lhs_col_id, + columns: lhs_col_id, lower_bound: Bound::Unbounded, upper_bound: Bound::Excluded(upper), .. - }) if col_id == lhs_col_id => { + }) if columns == lhs_col_id => { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Self::bound(value, inclusive), upper_bound: Bound::Excluded(upper), })); @@ -883,7 +886,7 @@ impl QueryExpr { lhs: filter.into(), rhs: IndexScan { table, - col_id, + columns, lower_bound: Self::bound(value, inclusive), upper_bound: Bound::Unbounded, } @@ -897,7 +900,7 @@ impl QueryExpr { self.query.push(Query::Select( IndexScan { table, - col_id, + columns, lower_bound: Self::bound(value, inclusive), upper_bound: Bound::Unbounded, } @@ -914,7 +917,7 @@ impl QueryExpr { pub fn with_index_upper_bound( mut self, table: DbTable, - col_id: ColId, + columns: NonEmpty, value: AlgebraicValue, inclusive: bool, ) -> Self { @@ -922,7 +925,7 @@ impl QueryExpr { let Some(query) = self.query.pop() else { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Bound::Unbounded, upper_bound: Self::bound(value, inclusive), })); @@ -941,14 +944,14 @@ impl QueryExpr { }, .. }) if table.table_id != rhs_table_id => { - self = self.with_index_upper_bound(table, col_id, value, inclusive); + self = self.with_index_upper_bound(table, columns, value, inclusive); self.query.push(query); self } // try to push below join's rhs Query::JoinInner(JoinExpr { rhs, col_lhs, col_rhs }) => { self.query.push(Query::JoinInner(JoinExpr { - rhs: rhs.with_index_upper_bound(table, col_id, value, inclusive), + rhs: rhs.with_index_upper_bound(table, columns, value, inclusive), col_lhs, col_rhs, })); @@ -956,14 +959,14 @@ impl QueryExpr { } // merge with a preceding lower bounded index scan (inclusive) Query::IndexScan(IndexScan { - col_id: lhs_col_id, + columns: lhs_col_id, lower_bound: Bound::Included(lower), upper_bound: Bound::Unbounded, .. - }) if col_id == lhs_col_id => { + }) if columns == lhs_col_id => { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Bound::Included(lower), upper_bound: Self::bound(value, inclusive), })); @@ -971,14 +974,14 @@ impl QueryExpr { } // merge with a preceding lower bounded index scan (inclusive) Query::IndexScan(IndexScan { - col_id: lhs_col_id, + columns: lhs_col_id, lower_bound: Bound::Excluded(lower), upper_bound: Bound::Unbounded, .. - }) if col_id == lhs_col_id => { + }) if columns == lhs_col_id => { self.query.push(Query::IndexScan(IndexScan { table, - col_id, + columns, lower_bound: Bound::Excluded(lower), upper_bound: Self::bound(value, inclusive), })); @@ -991,7 +994,7 @@ impl QueryExpr { lhs: filter.into(), rhs: IndexScan { table, - col_id, + columns, lower_bound: Bound::Unbounded, upper_bound: Self::bound(value, inclusive), } @@ -1005,7 +1008,7 @@ impl QueryExpr { self.query.push(Query::Select( IndexScan { table, - col_id, + columns, lower_bound: Bound::Unbounded, upper_bound: Self::bound(value, inclusive), } @@ -1185,7 +1188,7 @@ impl QueryExpr { match is_sargable(schema, op) { // found sargable equality condition for one of the table schemas Some(IndexArgument::Eq { col_id, value }) => { - q = q.with_index_eq(schema.into(), col_id, value); + q = q.with_index_eq(schema.into(), col_id.into(), value); continue 'outer; } // found sargable range condition for one of the table schemas @@ -1194,7 +1197,7 @@ impl QueryExpr { value, inclusive, }) => { - q = q.with_index_lower_bound(schema.into(), col_id, value, inclusive); + q = q.with_index_lower_bound(schema.into(), col_id.into(), value, inclusive); continue 'outer; } // found sargable range condition for one of the table schemas @@ -1203,7 +1206,7 @@ impl QueryExpr { value, inclusive, }) => { - q = q.with_index_upper_bound(schema.into(), col_id, value, inclusive); + q = q.with_index_upper_bound(schema.into(), col_id.into(), value, inclusive); continue 'outer; } None => {} @@ -1736,7 +1739,7 @@ mod tests { [ Query::IndexScan(IndexScan { table: db_table, - col_id: 42.into(), + columns: NonEmpty::new(42.into()), lower_bound: Bound::Included(22.into()), upper_bound: Bound::Unbounded, }), From ce3d040994fd89a1b35077c323e512c6eaea75f2 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Mon, 13 Nov 2023 20:20:05 +0100 Subject: [PATCH 2/5] fixup --- crates/bench/Cargo.toml | 2 +- crates/bindings-macro/src/lib.rs | 9 +- crates/bindings/src/rt.rs | 19 ++-- crates/cli/src/subcommands/generate/python.rs | 6 +- .../src/subcommands/generate/typescript.rs | 11 +-- crates/core/src/db/commit_log.rs | 7 +- .../db/datastore/locking_tx_datastore/mod.rs | 27 +++--- crates/core/src/db/datastore/system_tables.rs | 14 +-- crates/core/src/db/relational_db.rs | 9 +- crates/core/src/host/mod.rs | 10 +- crates/core/src/host/module_host.rs | 18 ++-- crates/core/src/sql/ast.rs | 23 ++--- crates/core/src/vm.rs | 9 +- crates/lib/src/identity.rs | 3 +- crates/lib/src/lib.rs | 18 ++-- crates/primitives/src/attr.rs | 16 ++-- crates/primitives/src/lib.rs | 2 +- crates/sats/src/db/attr.rs | 91 ++----------------- crates/sats/src/db/def.rs | 19 ++-- crates/sats/src/db/mod.rs | 17 +--- crates/sats/src/product_value.rs | 20 ++-- crates/vm/src/expr.rs | 36 ++++---- 22 files changed, 125 insertions(+), 261 deletions(-) diff --git a/crates/bench/Cargo.toml b/crates/bench/Cargo.toml index 62bb519139f..81fc3407748 100644 --- a/crates/bench/Cargo.toml +++ b/crates/bench/Cargo.toml @@ -22,7 +22,7 @@ bench = false [dependencies] spacetimedb-lib = { path = "../lib" } spacetimedb-core = { path = "../core" } -spacetimedb-sats= { path = "../sats" } +spacetimedb-sats = { path = "../sats" } spacetimedb-standalone = { path = "../standalone" } spacetimedb-client-api = { path = "../client-api" } spacetimedb-testing = { path = "../testing" } diff --git a/crates/bindings-macro/src/lib.rs b/crates/bindings-macro/src/lib.rs index 11543008088..86f1c5413bb 100644 --- a/crates/bindings-macro/src/lib.rs +++ b/crates/bindings-macro/src/lib.rs @@ -11,22 +11,19 @@ mod module; extern crate core; extern crate proc_macro; -use std::collections::HashMap; -use std::time::Duration; - use bitflags::Flags; use module::{derive_deserialize, derive_satstype, derive_serialize}; use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, quote_spanned, TokenStreamExt}; +use spacetimedb_primitives::ColumnAttribute; +use std::collections::HashMap; +use std::time::Duration; use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned; use syn::{ parse_quote, BinOp, Expr, ExprBinary, ExprLit, ExprUnary, FnArg, Ident, ItemFn, ItemStruct, Member, Token, Type, UnOp, }; -#[path = "../../sats/src/db/attr.rs"] -mod attr; -use attr::ColumnAttribute; mod sym { /// A symbol known at compile-time against diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 421741187c2..0abbb564c58 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,25 +1,24 @@ #![deny(unsafe_op_in_unsafe_fn)] -use nonempty::NonEmpty; -use std::any::TypeId; -use std::collections::{btree_map, BTreeMap}; -use std::fmt; -use std::marker::PhantomData; -use std::sync::Mutex; -use std::time::Duration; -use sys::Buffer; - use crate::sats::db::auth::{StAccess, StTableType}; +use crate::sats::db::def::{IndexDef, AUTO_TABLE_ID}; use crate::timestamp::with_timestamp_set; use crate::{sys, ReducerContext, ScheduleToken, SpacetimeType, TableType, Timestamp}; +use nonempty::NonEmpty; use spacetimedb_lib::de::{self, Deserialize, SeqProductAccess}; use spacetimedb_lib::sats::typespace::TypespaceBuilder; use spacetimedb_lib::sats::{impl_deserialize, impl_serialize, AlgebraicType, AlgebraicTypeRef, ProductTypeElement}; use spacetimedb_lib::ser::{Serialize, SerializeSeqProduct}; use spacetimedb_lib::{bsatn, Address, Identity, MiscModuleExport, ModuleDef, ReducerDef, TableDef, TypeAlias}; use spacetimedb_primitives::TableId; +use std::any::TypeId; +use std::collections::{btree_map, BTreeMap}; +use std::fmt; +use std::marker::PhantomData; +use std::sync::Mutex; +use std::time::Duration; +use sys::Buffer; -use crate::sats::db::def::{IndexDef, AUTO_TABLE_ID}; pub use once_cell::sync::{Lazy, OnceCell}; /// The `sender` invokes `reducer` at `timestamp` and provides it with the given `args`. diff --git a/crates/cli/src/subcommands/generate/python.rs b/crates/cli/src/subcommands/generate/python.rs index c3aa0a13b8c..ad530c8009d 100644 --- a/crates/cli/src/subcommands/generate/python.rs +++ b/crates/cli/src/subcommands/generate/python.rs @@ -3,7 +3,7 @@ use super::util::fmt_fn; use convert_case::{Case, Casing}; use spacetimedb_lib::sats::db::attr::ColumnAttribute; use spacetimedb_lib::{ - sats::{AlgebraicTypeRef, ArrayType, BuiltinType, MapType}, + sats::{AlgebraicType::Builtin, AlgebraicTypeRef, ArrayType, BuiltinType, MapType}, AlgebraicType, ProductType, ProductTypeElement, ReducerDef, SumType, TableDef, }; use std::fmt::{self, Write}; @@ -160,7 +160,7 @@ fn generate_imports(ctx: &GenCtx, elements: &Vec, imports: & fn _generate_imports(ctx: &GenCtx, ty: &AlgebraicType, imports: &mut Vec) { match ty { - AlgebraicType::Builtin(b) => match b { + Builtin(b) => match b { BuiltinType::Array(ArrayType { elem_ty }) => _generate_imports(ctx, elem_ty, imports), BuiltinType::Map(map_type) => { _generate_imports(ctx, &map_type.key_ty, imports); @@ -389,7 +389,7 @@ fn autogen_python_product_table_common( AlgebraicType::Product(_) => { reducer_args.push(format!("self.{python_field_name}")); } - AlgebraicType::Builtin(_) => { + Builtin(_) => { reducer_args.push(format!("self.{python_field_name}")); } AlgebraicType::Ref(type_ref) => { diff --git a/crates/cli/src/subcommands/generate/typescript.rs b/crates/cli/src/subcommands/generate/typescript.rs index e1d5df2fefb..f0eb6a45c9f 100644 --- a/crates/cli/src/subcommands/generate/typescript.rs +++ b/crates/cli/src/subcommands/generate/typescript.rs @@ -5,8 +5,8 @@ use std::fmt::{self, Write}; use convert_case::{Case, Casing}; use spacetimedb_lib::sats::db::attr::ColumnAttribute; use spacetimedb_lib::sats::{ - AlgebraicType, AlgebraicTypeRef, ArrayType, BuiltinType, MapType, ProductType, ProductTypeElement, SumType, - SumTypeVariant, + AlgebraicType, AlgebraicType::Builtin, AlgebraicTypeRef, ArrayType, BuiltinType, MapType, ProductType, + ProductTypeElement, SumType, SumTypeVariant, }; use spacetimedb_lib::{ReducerDef, TableDef}; @@ -142,7 +142,6 @@ fn convert_type<'a>( } AlgebraicType::Sum(sum_type) => { if let Some(inner_ty) = sum_type.as_option() { - use AlgebraicType::Builtin; match inner_ty { Builtin(ty) => match ty { BuiltinType::Bool @@ -316,8 +315,8 @@ fn serialize_type<'a>( } } AlgebraicType::Builtin(BuiltinType::Array(ArrayType { elem_ty })) => match &**elem_ty { - AlgebraicType::Builtin(BuiltinType::U8) => write!(f, "Array.from({value})"), - AlgebraicType::Builtin(_) => write!(f, "{value}"), + Builtin(BuiltinType::U8) => write!(f, "Array.from({value})"), + Builtin(_) => write!(f, "{value}"), t => write!(f, "{value}.map(el => {})", serialize_type(ctx, t, "el", prefix)), }, AlgebraicType::Builtin(_) => write!(f, "{value}"), @@ -596,7 +595,7 @@ fn generate_imports_variants( fn _generate_imports(ctx: &GenCtx, ty: &AlgebraicType, imports: &mut Vec, prefix: Option<&str>) { match ty { - AlgebraicType::Builtin(b) => match b { + Builtin(b) => match b { BuiltinType::Array(ArrayType { elem_ty }) => _generate_imports(ctx, elem_ty, imports, prefix), BuiltinType::Map(map_type) => { _generate_imports(ctx, &map_type.key_ty, imports, prefix); diff --git a/crates/core/src/db/commit_log.rs b/crates/core/src/db/commit_log.rs index 714a86750d0..3a68a545c08 100644 --- a/crates/core/src/db/commit_log.rs +++ b/crates/core/src/db/commit_log.rs @@ -1,7 +1,3 @@ -use anyhow::Context; -use std::io; -use std::sync::{Arc, Mutex, MutexGuard}; - use super::{ datastore::traits::{MutTxDatastore, TxData}, message_log::{self, MessageLog}, @@ -20,8 +16,11 @@ use crate::{ error::DBError, execution_context::ExecutionContext, }; +use anyhow::Context; use spacetimedb_sats::hash::{hash_bytes, Hash}; use spacetimedb_sats::DataKey; +use std::io; +use std::sync::{Arc, Mutex, MutexGuard}; #[derive(Clone)] pub struct CommitLog { diff --git a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs index 75e98486f49..80900361423 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs @@ -2,24 +2,11 @@ mod btree_index; mod sequence; mod table; -use nonempty::NonEmpty; -use parking_lot::{lock_api::ArcMutexGuard, Mutex, RawMutex}; - use self::{ btree_index::{BTreeIndex, BTreeIndexRangeIter}, sequence::Sequence, table::Table, }; -use anyhow::anyhow; -use std::time::{Duration, Instant}; -use std::{ - borrow::Cow, - collections::{BTreeMap, BTreeSet, HashMap}, - ops::{Deref, RangeBounds}, - sync::Arc, - vec, -}; - use super::{ system_tables::{ StColumnRow, StIndexRow, StSequenceRow, StTableRow, ST_COLUMNS_ID, ST_COLUMNS_ROW_TYPE, ST_INDEXES_ID, @@ -45,6 +32,9 @@ use crate::{ error::{DBError, TableError}, execution_context::ExecutionContext, }; +use anyhow::anyhow; +use nonempty::NonEmpty; +use parking_lot::{lock_api::ArcMutexGuard, Mutex, RawMutex}; use spacetimedb_lib::Address; use spacetimedb_primitives::{ColId, IndexId, SequenceId, TableId}; use spacetimedb_sats::data_key::{DataKey, ToDataKey}; @@ -55,6 +45,14 @@ use spacetimedb_sats::{ db::auth::{StAccess, StTableType}, AlgebraicType, AlgebraicValue, ProductType, ProductValue, }; +use std::time::{Duration, Instant}; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet, HashMap}, + ops::{Deref, RangeBounds}, + sync::Arc, + vec, +}; use thiserror::Error; #[derive(Error, Debug, PartialEq, Eq)] @@ -2371,10 +2369,9 @@ impl traits::MutProgrammable for Locking { #[cfg(test)] mod tests { use super::*; - use itertools::Itertools; - use crate::db::datastore::Result; use crate::error::IndexError; + use itertools::Itertools; use spacetimedb_lib::error::ResultTest; use spacetimedb_sats::product; diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index 41ab373abfc..165658b4042 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -2,7 +2,6 @@ use crate::error::{DBError, TableError}; use core::fmt; use nonempty::NonEmpty; use once_cell::sync::Lazy; - use spacetimedb_primitives::*; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::*; @@ -734,17 +733,14 @@ impl StIndexRow<&str> { } fn to_cols(row: &ProductValue, col_pos: ColId, col_name: &'static str) -> Result, DBError> { - let col_pos = col_pos.idx(); - let cols = row.field_as_array(col_pos, Some(col_name))?; + let index = col_pos.idx(); + let name = Some(col_name); + let cols = row.field_as_array(index, name)?; if let ArrayValue::U32(x) = &cols { let x: Vec<_> = x.iter().map(|x| ColId::from(*x)).collect(); Ok(NonEmpty::from_slice(&x).unwrap()) } else { - Err(InvalidFieldError { - name: Some(col_name), - col_pos: col_pos.into(), - } - .into()) + Err(InvalidFieldError { name, index }.into()) } } @@ -756,7 +752,7 @@ impl<'a> TryFrom<&'a ProductValue> for StIndexRow<&'a str> { let index_name = row.field_as_str(StIndexFields::IndexName.col_idx(), None)?; let index_type = row.field_as_u8(StIndexFields::IndexType.col_idx(), None)?; let index_type = IndexType::try_from(index_type).map_err(|_| InvalidFieldError { - col_pos: StIndexFields::IndexType.col_id(), + index: StIndexFields::IndexType.col_idx(), name: Some(StIndexFields::IndexType.name()), })?; let columns = to_cols(row, StIndexFields::Columns.col_id(), StIndexFields::Columns.name())?; diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 942afd2640e..e4f6a3cfe2e 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -8,6 +8,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use super::commit_log::{CommitLog, CommitLogView}; +use super::datastore::locking_tx_datastore::Locking; use super::datastore::locking_tx_datastore::{DataRef, Iter, IterByColEq, IterByColRange, MutTxId, RowId}; use super::datastore::traits::{MutProgrammable, MutTx, MutTxDatastore, Programmable, TxData}; use super::message_log::MessageLog; @@ -24,14 +25,11 @@ use crate::error::{DBError, DatabaseError, IndexError, TableError}; use crate::execution_context::ExecutionContext; use crate::hash::Hash; use spacetimedb_lib::PrimaryKey; -use spacetimedb_primitives::*; +use spacetimedb_primitives::{ColId, ColumnAttribute, IndexId, SequenceId, TableId}; use spacetimedb_sats::data_key::ToDataKey; -use spacetimedb_sats::db::attr::ColumnAttribute; -use spacetimedb_sats::db::def::*; +use spacetimedb_sats::db::def::{IndexDef, SequenceDef, TableDef, TableSchema}; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductType, ProductValue}; -use super::datastore::locking_tx_datastore::Locking; - #[derive(Clone)] pub struct RelationalDB { // TODO(cloutiertyler): This should not be public @@ -759,6 +757,7 @@ mod tests { use spacetimedb_lib::error::ResultTest; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::db::auth::{StAccess, StTableType}; + use spacetimedb_sats::db::def::{ColumnDef, IndexType}; use spacetimedb_sats::product; fn column(name: &str, ty: AlgebraicType) -> ColumnDef { diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index ef8a8c081a1..959977dc2b7 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -1,13 +1,13 @@ -use std::time::Duration; - use anyhow::Context; use bytes::Bytes; use bytestring::ByteString; +use derive_more::Display; +use enum_map::Enum; use spacetimedb_lib::de::serde::SeedWrapper; use spacetimedb_lib::de::DeserializeSeed; -use spacetimedb_lib::{bsatn, Identity}; -use spacetimedb_lib::{ProductValue, ReducerDef}; +use spacetimedb_lib::{bsatn, Identity, ProductValue, ReducerDef}; use spacetimedb_sats::WithTypespace; +use std::time::Duration; mod host_controller; pub(crate) mod module_host; @@ -18,8 +18,6 @@ pub mod instance_env; mod timestamp; mod wasm_common; -use derive_more::Display; -use enum_map::Enum; pub use host_controller::{ DescribedEntityType, EnergyDiff, EnergyQuanta, HostController, ReducerCallResult, ReducerOutcome, UpdateOutcome, }; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index b895c2f7777..180fabbe24d 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1,13 +1,3 @@ -use std::collections::HashMap; -use std::fmt; -use std::sync::{Arc, Weak}; -use std::time::Duration; - -use base64::{engine::general_purpose::STANDARD as BASE_64_STD, Engine as _}; -use futures::{Future, FutureExt}; -use indexmap::IndexMap; -use tokio::sync::oneshot; - use super::host_controller::HostThreadpool; use super::{ArgsTuple, EnergyDiff, InvalidReducerArguments, ReducerArgs, ReducerCallResult, ReducerId, Timestamp}; use crate::client::ClientConnectionSender; @@ -26,10 +16,18 @@ use crate::util::lending_pool::{Closed, LendingPool, LentResource, PoolClosed, W use crate::util::notify_once::NotifyOnce; use crate::util::prometheus_handle::{GaugeInc, IntGaugeExt}; use crate::worker_metrics::WORKER_METRICS; +use base64::{engine::general_purpose::STANDARD as BASE_64_STD, Engine as _}; +use futures::{Future, FutureExt}; +use indexmap::IndexMap; use spacetimedb_lib::{Address, ReducerDef, TableDef}; use spacetimedb_primitives::TableId; use spacetimedb_sats::relation::MemTable; use spacetimedb_sats::{ProductValue, Typespace, WithTypespace}; +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, Weak}; +use std::time::Duration; +use tokio::sync::oneshot; #[derive(Debug, Default, Clone)] pub struct DatabaseUpdate { diff --git a/crates/core/src/sql/ast.rs b/crates/core/src/sql/ast.rs index dcceb697c9d..72955ce3530 100644 --- a/crates/core/src/sql/ast.rs +++ b/crates/core/src/sql/ast.rs @@ -1,7 +1,12 @@ +use crate::db::datastore::locking_tx_datastore::MutTxId; +use crate::db::datastore::traits::MutTxDatastore; +use crate::db::relational_db::RelationalDB; +use crate::error::{DBError, PlanError}; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::{ConstraintFlags, Constraints, TableSchema}; use spacetimedb_sats::db::def::{FieldDef, ProductTypeMeta}; use spacetimedb_sats::db::error::RelationError; +use spacetimedb_sats::relation::{extract_table_field, FieldExpr, FieldName}; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductTypeElement}; use spacetimedb_vm::errors::ErrorVm; use spacetimedb_vm::expr::{ColumnOp, DbType, Expr}; @@ -17,12 +22,6 @@ use sqlparser::parser::Parser; use std::borrow::Cow; use std::collections::HashMap; -use crate::db::datastore::locking_tx_datastore::MutTxId; -use crate::db::datastore::traits::MutTxDatastore; -use crate::db::relational_db::RelationalDB; -use crate::error::{DBError, PlanError}; -use spacetimedb_sats::relation::{extract_table_field, FieldExpr, FieldName}; - /// Simplify to detect features of the syntax we don't support yet /// Because we use [PostgreSqlDialect] in the compiler step it already protect against features /// that are not in the standard SQL-92 but still need to check for completeness @@ -177,14 +176,10 @@ impl From { let field = extract_table_field(f)?; let fields = self.iter_tables().flat_map(|t| { t.columns.iter().filter_map(|column| { - if column.col_name == field.field { - Some(FieldDef { - column: column.clone(), - table_name: field.table.unwrap_or(&t.table_name).to_string(), - }) - } else { - None - } + (column.col_name == field.field).then(|| FieldDef { + column: column.clone(), + table_name: field.table.unwrap_or(&t.table_name).to_string(), + }) }) }); diff --git a/crates/core/src/vm.rs b/crates/core/src/vm.rs index f80d93631d2..3295b524be6 100644 --- a/crates/core/src/vm.rs +++ b/crates/core/src/vm.rs @@ -1,14 +1,10 @@ //! The [DbProgram] that execute arbitrary queries & code against the database. -use std::collections::HashMap; -use std::ops::RangeBounds; - -use itertools::Itertools; -use tracing::debug; use crate::db::cursor::{CatalogCursor, IndexCursor, TableCursor}; use crate::db::datastore::locking_tx_datastore::{IterByColEq, MutTxId}; use crate::db::relational_db::RelationalDB; use crate::execution_context::ExecutionContext; +use itertools::Itertools; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::db::auth::{StAccess, StTableType}; @@ -23,6 +19,9 @@ use spacetimedb_vm::eval::IterRows; use spacetimedb_vm::expr::*; use spacetimedb_vm::program::{ProgramRef, ProgramVm}; use spacetimedb_vm::rel_ops::RelOps; +use std::collections::HashMap; +use std::ops::RangeBounds; +use tracing::debug; //TODO: This is partially duplicated from the `vm` crate to avoid borrow checker issues //and pull all that crate in core. Will be revisited after trait refactor diff --git a/crates/lib/src/identity.rs b/crates/lib/src/identity.rs index f6ab907e9db..3c5bd8819a5 100644 --- a/crates/lib/src/identity.rs +++ b/crates/lib/src/identity.rs @@ -1,8 +1,7 @@ -use std::{fmt, str::FromStr}; - use spacetimedb_bindings_macro::{Deserialize, Serialize}; use spacetimedb_sats::hex::HexString; use spacetimedb_sats::{hash, impl_st, AlgebraicType}; +use std::{fmt, str::FromStr}; #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct AuthCtx { diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index f8db502a0eb..c8055ad8516 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -1,9 +1,9 @@ use anyhow::Context; use spacetimedb_primitives::ColId; +use spacetimedb_sats::db::attr::ColumnAttribute; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::{ColumnDef, IndexDef, IndexType, AUTO_TABLE_ID}; use spacetimedb_sats::{impl_serialize, WithTypespace}; -use std::iter; pub mod address; pub mod filter; @@ -124,7 +124,7 @@ impl TableDef { let index_for_column = table.indexes.iter().find(|index| { // Ignore multi-column indexes - matches!(index.cols.split_first(), (index_col_id,[]) if index_col_id.idx() == col_id) + index.cols.tail.is_empty() && index.cols.head.idx() == col_id }); // If there's an index defined for this column already, use it, @@ -156,18 +156,14 @@ impl TableDef { // Multi-column indexes cannot be unique (yet), so just add them. let multi_col_indexes = table.indexes.iter().filter_map(|index| { - if let (a, [b, rest @ ..]) = index.cols.split_first() { - let idx = spacetimedb_sats::db::def::IndexDef::new_cols( + (index.cols.len() > 1).then(|| { + spacetimedb_sats::db::def::IndexDef::new_cols( index.name.clone(), AUTO_TABLE_ID, false, - (*a, iter::once(*b).chain(rest.iter().copied()).collect()), - ); - - Some(idx) - } else { - None - } + index.cols.clone(), + ) + }) }); indexes.extend(multi_col_indexes); diff --git a/crates/primitives/src/attr.rs b/crates/primitives/src/attr.rs index 8f1e0d88008..64ab3c85bf4 100644 --- a/crates/primitives/src/attr.rs +++ b/crates/primitives/src/attr.rs @@ -24,7 +24,7 @@ pub enum AttributeKind { // and is distinct to `Constraints` in `sats/db/def.rs` bitflags! { #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] - pub struct ColumnIndexAttribute: u8 { + pub struct ColumnAttribute: u8 { const UNSET = Self::empty().bits(); /// Index no unique const INDEXED = 0b0001; @@ -43,7 +43,7 @@ bitflags! { } } -impl TryFrom for ColumnIndexAttribute { +impl TryFrom for ColumnAttribute { type Error = (); fn try_from(v: u8) -> Result { @@ -51,23 +51,19 @@ impl TryFrom for ColumnIndexAttribute { } } -impl ColumnIndexAttribute { +impl ColumnAttribute { /// Checks if either 'IDENTITY' or 'PRIMARY_KEY_AUTO' constraints are set because the imply the use of /// auto increment sequence. pub const fn has_autoinc(&self) -> bool { - self.contains(ColumnIndexAttribute::IDENTITY) - || self.contains(ColumnIndexAttribute::PRIMARY_KEY_AUTO) - || self.contains(ColumnIndexAttribute::AUTO_INC) + self.contains(Self::IDENTITY) || self.contains(Self::PRIMARY_KEY_AUTO) || self.contains(Self::AUTO_INC) } pub const fn has_unique(&self) -> bool { - self.contains(ColumnIndexAttribute::UNIQUE) + self.contains(Self::UNIQUE) } pub const fn has_primary(&self) -> bool { - self.contains(ColumnIndexAttribute::IDENTITY) - || self.contains(ColumnIndexAttribute::PRIMARY_KEY) - || self.contains(ColumnIndexAttribute::PRIMARY_KEY_AUTO) + self.contains(Self::IDENTITY) || self.contains(Self::PRIMARY_KEY) || self.contains(Self::PRIMARY_KEY_AUTO) } /// Returns the [ColumnIndexAttribute] of constraints as an enum variant. diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index d1210f3e00f..ded834789ac 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -3,5 +3,5 @@ mod attr; mod ids; -pub use attr::{AttributeKind, ColumnIndexAttribute}; +pub use attr::{AttributeKind, ColumnAttribute}; pub use ids::{ColId, ConstraintId, IndexId, SequenceId, TableId}; diff --git a/crates/sats/src/db/attr.rs b/crates/sats/src/db/attr.rs index 8b7b739f4cd..2b46195e260 100644 --- a/crates/sats/src/db/attr.rs +++ b/crates/sats/src/db/attr.rs @@ -1,86 +1,9 @@ -// This file is `included` also in `crates/bindings-macro/src/lib.rs` to avoid duplicating -// So can't import things from `sats` crate -#![allow(dead_code)] -use bitflags::bitflags; +use crate::{de, impl_deserialize, impl_serialize}; +pub use spacetimedb_primitives::ColumnAttribute; -#[allow(non_camel_case_types)] -#[allow(clippy::upper_case_acronyms)] -#[derive(Eq, PartialEq)] -pub enum AttributeKind { - UNSET, - /// Index no unique - INDEXED, - /// Auto Increment - AUTO_INC, - /// Index unique - UNIQUE, - /// Unique + AutoInc - IDENTITY, - /// Primary key column (implies Unique) - PRIMARY_KEY, - /// PrimaryKey + AutoInc - PRIMARY_KEY_AUTO, - /// PrimaryKey + Identity - PRIMARY_KEY_IDENTITY, -} +impl_deserialize!([] ColumnAttribute, de => + Self::from_bits(de.deserialize_u8()?) + .ok_or_else(|| de::Error::custom("invalid bitflags for `ColumnAttribute`")) +); -// This indeed is only used for defining columns + constraints AND/OR auto_inc, -// and is distinct to `Constraints` in `sats/db/def.rs` -bitflags! { - #[derive(Debug, Clone, Copy, Eq, PartialEq, PartialOrd, Ord)] - pub struct ColumnAttribute: u8 { - const UNSET = Self::empty().bits(); - /// Index no unique - const INDEXED = 0b0001; - /// Generate the next [Sequence] - const AUTO_INC = 0b0010; - /// Index unique - const UNIQUE = Self::INDEXED.bits() | 0b0100; - /// Unique + AutoInc - const IDENTITY = Self::UNIQUE.bits() | Self::AUTO_INC.bits(); - /// Primary key column (implies Unique) - const PRIMARY_KEY = Self::UNIQUE.bits() | 0b1000; - /// PrimaryKey + AutoInc - const PRIMARY_KEY_AUTO = Self::PRIMARY_KEY.bits() | Self::AUTO_INC.bits(); - /// PrimaryKey + Identity - const PRIMARY_KEY_IDENTITY = Self::PRIMARY_KEY.bits() | Self::IDENTITY.bits() ; - } -} - -impl ColumnAttribute { - /// Checks if either 'IDENTITY' or 'PRIMARY_KEY_AUTO' constraints are set because the imply the use of - /// auto increment sequence. - pub const fn has_autoinc(&self) -> bool { - self.contains(ColumnAttribute::IDENTITY) - || self.contains(ColumnAttribute::PRIMARY_KEY_AUTO) - || self.contains(ColumnAttribute::AUTO_INC) - } - - pub const fn has_unique(&self) -> bool { - self.contains(ColumnAttribute::UNIQUE) - } - - pub const fn has_primary(&self) -> bool { - self.contains(ColumnAttribute::IDENTITY) - || self.contains(ColumnAttribute::PRIMARY_KEY) - || self.contains(ColumnAttribute::PRIMARY_KEY_AUTO) - } - - /// Returns the [ColumnAttribute] of constraints as an enum variant. - /// - /// NOTE: This represent the higher possible representation of a constraints, so for example - /// `IDENTITY` imply that is `INDEXED, UNIQUE` - pub fn kind(&self) -> AttributeKind { - match self { - x if x == &ColumnAttribute::UNSET => AttributeKind::UNSET, - x if x == &ColumnAttribute::INDEXED => AttributeKind::INDEXED, - x if x == &ColumnAttribute::UNIQUE => AttributeKind::UNIQUE, - x if x == &ColumnAttribute::AUTO_INC => AttributeKind::AUTO_INC, - x if x == &ColumnAttribute::IDENTITY => AttributeKind::IDENTITY, - x if x == &ColumnAttribute::PRIMARY_KEY => AttributeKind::PRIMARY_KEY, - x if x == &ColumnAttribute::PRIMARY_KEY_AUTO => AttributeKind::PRIMARY_KEY_AUTO, - x if x == &ColumnAttribute::PRIMARY_KEY_IDENTITY => AttributeKind::PRIMARY_KEY_IDENTITY, - x => unreachable!("Unexpected value {x:?}"), - } - } -} +impl_serialize!([] ColumnAttribute, (self, ser) => ser.serialize_u8(self.bits())); diff --git a/crates/sats/src/db/def.rs b/crates/sats/src/db/def.rs index f30d70e5698..056714ca1c0 100644 --- a/crates/sats/src/db/def.rs +++ b/crates/sats/src/db/def.rs @@ -1,7 +1,3 @@ -use derive_more::Display; -use nonempty::NonEmpty; -use spacetimedb_primitives::*; - use crate::db::auth::{StAccess, StTableType}; use crate::de::BasicVecVisitor; use crate::product_value::InvalidFieldError; @@ -9,6 +5,9 @@ use crate::relation::{Column, DbTable, FieldName, FieldOnly, Header, TableField} use crate::ser::SerializeArray; use crate::{de, impl_deserialize, impl_serialize, ser, AlgebraicValue, ProductValue}; use crate::{AlgebraicType, ProductType, ProductTypeElement}; +use derive_more::Display; +use nonempty::NonEmpty; +use spacetimedb_primitives::*; /// The default preallocation amount for sequences. pub const SEQUENCE_PREALLOCATION_AMOUNT: i128 = 4_096; @@ -169,10 +168,10 @@ impl TryFrom for Constraints { } } -impl TryFrom for Constraints { +impl TryFrom for Constraints { type Error = (); - fn try_from(value: ColumnIndexAttribute) -> Result { + fn try_from(value: ColumnAttribute) -> Result { Ok(match value.kind() { AttributeKind::UNSET => Self::unset(), AttributeKind::INDEXED => Self::indexed(), @@ -289,7 +288,7 @@ impl IndexDef { } } - pub fn new_cols>>(name: String, table_id: TableId, is_unique: bool, cols: Col) -> Self { + pub fn new_cols(name: String, table_id: TableId, is_unique: bool, cols: impl Into>) -> Self { Self { cols: cols.into(), name, @@ -538,10 +537,8 @@ impl TableSchema { pub fn project(&self, columns: impl Iterator) -> Result, InvalidFieldError> { columns .map(|pos| { - self.get_column(usize::from(pos)).ok_or(InvalidFieldError { - col_pos: pos, - name: None, - }) + let index = pos.idx(); + self.get_column(index).ok_or(InvalidFieldError { index, name: None }) }) .collect() } diff --git a/crates/sats/src/db/mod.rs b/crates/sats/src/db/mod.rs index e321dbfc9f4..bee72d5d586 100644 --- a/crates/sats/src/db/mod.rs +++ b/crates/sats/src/db/mod.rs @@ -1,21 +1,6 @@ -//! Defines all the typed database objects & schemas -use crate::db::attr::ColumnAttribute; -use crate::{de, impl_deserialize, impl_serialize}; +//! Defines all the typed database objects & schemas. pub mod attr; pub mod auth; pub mod def; pub mod error; - -impl TryFrom for ColumnAttribute { - type Error = (); - - fn try_from(v: u8) -> Result { - Self::from_bits(v).ok_or(()) - } -} - -impl_deserialize!([] ColumnAttribute, de => Self::from_bits(de.deserialize_u8()?) - .ok_or_else(|| de::Error::custom("invalid bitflags for ColumnIndexAttribute"))); - -impl_serialize!([] ColumnAttribute, (self, ser) => ser.serialize_u8(self.bits())); diff --git a/crates/sats/src/product_value.rs b/crates/sats/src/product_value.rs index e7019f77426..58a956b5a2a 100644 --- a/crates/sats/src/product_value.rs +++ b/crates/sats/src/product_value.rs @@ -64,10 +64,10 @@ impl crate::Value for ProductValue { /// An error that occurs when a field, of a product value, is accessed that doesn't exist. #[derive(thiserror::Error, Debug, Copy, Clone)] -#[error("Field at position {col_pos} named: {name:?} not found or has an invalid type")] +#[error("Field at position {index} named: {name:?} not found or has an invalid type")] pub struct InvalidFieldError { - /// The claimed col_pos of the field within the product value. - pub col_pos: ColId, + /// The claimed index of the field within the product value. + pub index: usize, /// The name of the field, if any. pub name: Option<&'static str>, } @@ -76,11 +76,8 @@ impl ProductValue { /// Borrow the value at field of `self` indentified by `index`. /// /// The `name` is non-functional and is only used for error-messages. - pub fn get_field(&self, col_pos: usize, name: Option<&'static str>) -> Result<&AlgebraicValue, InvalidFieldError> { - self.elements.get(col_pos).ok_or(InvalidFieldError { - col_pos: col_pos.into(), - name, - }) + pub fn get_field(&self, index: usize, name: Option<&'static str>) -> Result<&AlgebraicValue, InvalidFieldError> { + self.elements.get(index).ok_or(InvalidFieldError { index, name }) } /// This function is used to project fields based on the provided `indexes`. @@ -129,14 +126,11 @@ impl ProductValue { /// and then runs it through the function `f` which possibly returns a `T` derived from `value`. pub fn extract_field<'a, T>( &'a self, - col_pos: usize, + index: usize, name: Option<&'static str>, f: impl 'a + Fn(&'a AlgebraicValue) -> Option, ) -> Result { - f(self.get_field(col_pos, name)?).ok_or(InvalidFieldError { - col_pos: col_pos.into(), - name, - }) + f(self.get_field(index, name)?).ok_or(InvalidFieldError { index, name }) } /// Interprets the value at field of `self` identified by `index` as a `bool`. diff --git a/crates/vm/src/expr.rs b/crates/vm/src/expr.rs index 54f52d70beb..963a9c7552c 100644 --- a/crates/vm/src/expr.rs +++ b/crates/vm/src/expr.rs @@ -1,14 +1,9 @@ -use derive_more::From; -use nonempty::NonEmpty; -use std::cmp::Ordering; -use std::collections::{HashMap, VecDeque}; -use std::fmt; -use std::ops::Bound; - use crate::errors::{ErrorKind, ErrorLang, ErrorType, ErrorVm}; use crate::functions::{FunDef, Param}; use crate::operator::{Op, OpCmp, OpLogic, OpQuery}; use crate::types::Ty; +use derive_more::From; +use nonempty::NonEmpty; use spacetimedb_lib::Identity; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::algebraic_type::AlgebraicType; @@ -19,8 +14,11 @@ use spacetimedb_sats::db::error::AuthError; use spacetimedb_sats::relation::{ Column, DbTable, FieldExpr, FieldName, Header, MemTable, RelValueRef, Relation, RowCount, Table, }; -use spacetimedb_sats::satn::Satn; -use spacetimedb_sats::{ProductValue, Typespace, WithTypespace}; +use spacetimedb_sats::{satn::Satn, ProductValue, Typespace, WithTypespace}; +use std::cmp::Ordering; +use std::collections::{HashMap, VecDeque}; +use std::fmt; +use std::ops::Bound; /// A `index` into the list of [Fun] pub type FunctionId = usize; @@ -230,7 +228,7 @@ impl From for ColumnOp { fn from(value: IndexScan) -> Self { let table = value.table; let columns = value.columns; - assert_eq!(columns.len(), 1, "Not yet supported multi-column predicates"); + assert_eq!(columns.len(), 1, "multi-column predicates are not yet supported"); let field = table.head.fields[usize::from(columns.head)].field.clone(); match (value.lower_bound, value.upper_bound) { @@ -329,15 +327,6 @@ impl SourceExpr { } } -impl From for Table { - fn from(value: SourceExpr) -> Self { - match value { - SourceExpr::MemTable(t) => Table::MemTable(t), - SourceExpr::DbTable(t) => Table::DbTable(t), - } - } -} - impl Relation for SourceExpr { fn head(&self) -> &Header { match self { @@ -363,6 +352,15 @@ impl From
for SourceExpr { } } +impl From for Table { + fn from(value: SourceExpr) -> Self { + match value { + SourceExpr::MemTable(t) => Table::MemTable(t), + SourceExpr::DbTable(t) => Table::DbTable(t), + } + } +} + impl From<&TableSchema> for SourceExpr { fn from(value: &TableSchema) -> Self { SourceExpr::DbTable(DbTable::new( From 877a1ecbd86dea46ee0b1146829a17676ca67089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Alejandro=20Montoya=20Corte=CC=81s?= Date: Tue, 14 Nov 2023 10:54:49 -0500 Subject: [PATCH 3/5] Fix stray constraints text on refactor --- crates/bench/src/spacetime_raw.rs | 3 +-- crates/core/src/db/commit_log.rs | 2 +- crates/core/src/db/datastore/locking_tx_datastore/mod.rs | 4 ++-- crates/core/src/db/datastore/system_tables.rs | 8 ++++---- crates/core/src/db/datastore/traits.rs | 2 +- crates/core/src/db/relational_db.rs | 2 +- crates/core/src/db/update.rs | 2 +- crates/core/src/host/instance_env.rs | 2 +- crates/core/src/vm.rs | 2 +- 9 files changed, 13 insertions(+), 14 deletions(-) diff --git a/crates/bench/src/spacetime_raw.rs b/crates/bench/src/spacetime_raw.rs index df85e19f60b..607f6a61cb9 100644 --- a/crates/bench/src/spacetime_raw.rs +++ b/crates/bench/src/spacetime_raw.rs @@ -46,14 +46,13 @@ impl BenchDatabase for SpacetimeRaw { match index_strategy { IndexStrategy::Unique => { self.db - .create_index(tx, table_id, IndexDef::new("id".to_string(), table_id, 0.into(), true))?; + .create_index(tx, IndexDef::new("id".to_string(), table_id, 0.into(), true))?; } IndexStrategy::NonUnique => (), IndexStrategy::MultiIndex => { for (i, column) in T::product_type().elements.iter().enumerate() { self.db.create_index( tx, - table_id, IndexDef::new(column.name.clone().unwrap(), table_id, i.into(), false), )?; } diff --git a/crates/core/src/db/commit_log.rs b/crates/core/src/db/commit_log.rs index 3a68a545c08..e17d2107435 100644 --- a/crates/core/src/db/commit_log.rs +++ b/crates/core/src/db/commit_log.rs @@ -245,7 +245,7 @@ impl CommitLogView { /// the transactions in a [`Commit`]. /// /// The iterator attempts to read each large object in turn, yielding an - /// [`io::Error`] with constraints [`io::ErrorKind::NotFound`] if the object was + /// [`io::Error`] with kind [`io::ErrorKind::NotFound`] if the object was /// not found. // // TODO(kim): We probably want a more efficient way to stream the contents diff --git a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs index 80900361423..80538f75ef3 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs @@ -505,7 +505,7 @@ impl Inner { is_unique: false, index_type: IndexType::BTree, }, - x => panic!("Adding constraint of constraints `{x:?}` is not supported yet."), + x => panic!("Adding constraint of kind `{x:?}` is not supported yet."), } })); @@ -2711,7 +2711,7 @@ mod tests { ColRow { table: 4, pos: 4, name: "columns", ty: AlgebraicType::array(AlgebraicType::U32), autoinc: false }, ColRow { table: 5, pos: 0, name: "program_hash", ty: AlgebraicType::array(AlgebraicType::U8), autoinc: false }, - ColRow { table: 5, pos: 1, name: "constraints", ty: AlgebraicType::U8, autoinc: false }, + ColRow { table: 5, pos: 1, name: "kind", ty: AlgebraicType::U8, autoinc: false }, ColRow { table: 5, pos: 2, name: "epoch", ty: AlgebraicType::U128, autoinc: false }, ])); #[rustfmt::skip] diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index 165658b4042..789e9b9725e 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -174,7 +174,7 @@ st_fields_enum!(enum StConstraintFields { // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StModuleFields { "program_hash", ProgramHash = 0, - "constraints", Kind = 1, + "kind", Kind = 1, "epoch", Epoch = 2, }); @@ -544,7 +544,7 @@ pub static ST_CONSTRAINT_ROW_TYPE: Lazy = /// * `constraints` is the [`ModuleKind`] (currently always [`WASM_MODULE`]). /// * `epoch` is a _fencing token_ used to protect against concurrent updates. /// -/// | program_hash | constraints | epoch | +/// | program_hash | kind | epoch | /// |---------------------|----------|-------| /// | [250, 207, 5, ...] | 0 | 42 | pub(crate) fn st_module_schema() -> TableSchema { @@ -931,7 +931,7 @@ impl From> for ProductValue { } } -/// Indicates the constraints of module the `program_hash` of a [`StModuleRow`] +/// Indicates the kind of module the `program_hash` of a [`StModuleRow`] /// describes. /// /// More or less a placeholder to allow for future non-WASM hosts without @@ -941,7 +941,7 @@ pub struct ModuleKind(u8); /// The [`ModuleKind`] of WASM-based modules. /// -/// This is currently the only known constraints. +/// This is currently the only known kind. pub const WASM_MODULE: ModuleKind = ModuleKind(0); impl_serialize!([] ModuleKind, (self, ser) => self.0.serialize(ser)); diff --git a/crates/core/src/db/datastore/traits.rs b/crates/core/src/db/datastore/traits.rs index 2bb5e505e1e..c6431c6857b 100644 --- a/crates/core/src/db/datastore/traits.rs +++ b/crates/core/src/db/datastore/traits.rs @@ -223,7 +223,7 @@ pub trait MutTxDatastore: TxDatastore + MutTx { /// Describes a programmable [`TxDatastore`]. /// -/// A programmable datastore is one which has a program of some constraints associated +/// A programmable datastore is one which has a program of some kind associated /// with it. pub trait Programmable: TxDatastore { /// Retrieve the [`Hash`] of the program currently associated with the diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index e4f6a3cfe2e..37cbb76b3d7 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -548,7 +548,7 @@ impl RelationalDB { /// /// NOTE: It loads the data from the table into it before returning #[tracing::instrument(skip(self, tx, index), fields(index=index.name))] - pub fn create_index(&self, tx: &mut MutTxId, table_id: TableId, index: IndexDef) -> Result { + pub fn create_index(&self, tx: &mut MutTxId, index: IndexDef) -> Result { self.inner.create_index_mut_tx(tx, index) } diff --git a/crates/core/src/db/update.rs b/crates/core/src/db/update.rs index 7a255462dae..32ad06a6e22 100644 --- a/crates/core/src/db/update.rs +++ b/crates/core/src/db/update.rs @@ -57,7 +57,7 @@ pub fn update_database( for index_def in indexes_to_create { system_logger.info(&format!("Creating index `{}`", index_def.name)); - stdb.create_index(tx, index_def.table_id, index_def)?; + stdb.create_index(tx, index_def)?; } } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index fad3e85cc9e..0f8b3a06520 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -250,7 +250,7 @@ impl InstanceEnv { index_type, }; - stdb.create_index(tx, table_id, index)?; + stdb.create_index(tx, index)?; Ok(()) } diff --git a/crates/core/src/vm.rs b/crates/core/src/vm.rs index 3295b524be6..ebf3646089b 100644 --- a/crates/core/src/vm.rs +++ b/crates/core/src/vm.rs @@ -737,7 +737,7 @@ pub(crate) mod tests { let mut tx = db.begin_tx(); let index = IndexDef::new("idx_1".into(), table_id, 0.into(), true); - let index_id = db.create_index(&mut tx, table_id, index)?; + let index_id = db.create_index(&mut tx, index)?; let p = &mut DbProgram::new(&ctx, &db, &mut tx, AuthCtx::for_testing()); From b11669da5dcda33929b0a9c1740ff714bbeb9cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Alejandro=20Montoya=20Corte=CC=81s?= Date: Tue, 14 Nov 2023 15:57:56 -0500 Subject: [PATCH 4/5] Fix conflict --- crates/bindings/src/lib.rs | 2 +- crates/core/src/host/instance_env.rs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index a0ad7a29b20..d7f133e3756 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -13,7 +13,6 @@ mod timestamp; use crate::sats::db::def::IndexType; use spacetimedb_lib::buffer::{BufReader, BufWriter, Cursor, DecodeError}; use spacetimedb_lib::sats::{impl_deserialize, impl_serialize, impl_st}; -pub use spacetimedb_lib::ser::Serialize; use spacetimedb_lib::{bsatn, PrimaryKey, ProductType, ProductValue}; use std::cell::RefCell; use std::marker::PhantomData; @@ -29,6 +28,7 @@ pub use spacetimedb_bindings_sys as sys; pub use spacetimedb_lib; pub use spacetimedb_lib::de::{Deserialize, DeserializeOwned}; pub use spacetimedb_lib::sats; +pub use spacetimedb_lib::ser::Serialize; pub use spacetimedb_lib::Address; pub use spacetimedb_lib::AlgebraicValue; pub use spacetimedb_lib::Identity; diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 0f8b3a06520..99f5bd09573 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,7 +1,6 @@ use nonempty::NonEmpty; use parking_lot::{Mutex, MutexGuard}; use smallvec::SmallVec; -use spacetimedb_lib::{bsatn, ProductValue}; use std::ops::DerefMut; use std::sync::Arc; From 051d6f638cf5ee2ad9acb3ff17c41234e755073b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mario=20Alejandro=20Montoya=20Corte=CC=81s?= Date: Wed, 15 Nov 2023 10:02:34 -0500 Subject: [PATCH 5/5] Merge --- crates/core/src/db/relational_db.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 37cbb76b3d7..3ce5c531c53 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -15,13 +15,12 @@ use super::message_log::MessageLog; use super::ostorage::memory_object_db::MemoryObjectDB; use super::relational_operators::Relation; use crate::address::Address; -use crate::db::commit_log; use crate::db::datastore::traits::DataRow; use crate::db::db_metrics::DB_METRICS; use crate::db::messages::commit::Commit; use crate::db::ostorage::hashmap_object_db::HashMapObjectDB; use crate::db::ostorage::ObjectDB; -use crate::error::{DBError, DatabaseError, IndexError, TableError}; +use crate::error::{DBError, DatabaseError, IndexError, LogReplayError, TableError}; use crate::execution_context::ExecutionContext; use crate::hash::Hash; use spacetimedb_lib::PrimaryKey; @@ -753,12 +752,17 @@ mod tests { use crate::db::datastore::system_tables::{ StIndexRow, StSequenceRow, StTableRow, ST_INDEXES_ID, ST_SEQUENCES_ID, ST_TABLES_ID, }; + use crate::db::message_log::SegmentView; + use crate::db::ostorage::sled_object_db::SledObjectDB; use crate::db::relational_db::tests_utils::make_test_db; use spacetimedb_lib::error::ResultTest; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::db::auth::{StAccess, StTableType}; use spacetimedb_sats::db::def::{ColumnDef, IndexType}; use spacetimedb_sats::product; + use std::io::{Seek, SeekFrom, Write}; + use std::ops::Range; + use tempfile::TempDir; fn column(name: &str, ty: AlgebraicType) -> ColumnDef { ColumnDef {