From 9aecc256de48dd210f6be201f6688c1fba5e0b62 Mon Sep 17 00:00:00 2001 From: Mario Montoya Date: Mon, 25 Sep 2023 11:32:29 -0500 Subject: [PATCH] Adding multi-column indexes to the DB (#243) * Initial draft for multi-column indexes * Fix index seek for multi-columns * Restore use of ColId * Fix compilation * Merge * Addressing some PR comments * Clippy * Clarify usage of AlgebraicValue for indexes * . --- Cargo.lock | 8 + Cargo.toml | 1 + crates/core/Cargo.toml | 1 + .../locking_tx_datastore/btree_index.rs | 47 +++- .../db/datastore/locking_tx_datastore/mod.rs | 213 ++++++++++-------- .../datastore/locking_tx_datastore/table.rs | 17 +- crates/core/src/db/datastore/system_tables.rs | 46 ++-- crates/core/src/db/datastore/traits.rs | 31 ++- crates/core/src/db/relational_db.rs | 44 ++-- crates/core/src/error.rs | 8 +- crates/core/src/host/instance_env.rs | 35 ++- crates/core/src/host/wasm_common.rs | 2 +- .../src/host/wasm_common/module_host_actor.rs | 8 +- .../core/src/host/wasmer/wasm_instance_env.rs | 9 +- .../src/messages/instance_db_trace_log.rs | 2 +- crates/core/src/sql/compiler.rs | 20 +- crates/core/src/vm.rs | 5 +- crates/lib/src/table.rs | 9 - crates/sats/Cargo.toml | 1 + crates/sats/src/builtin_value.rs | 12 + crates/sats/src/product_value.rs | 55 ++++- 21 files changed, 365 insertions(+), 209 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01e7921f965..a0fa633867f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2508,6 +2508,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonempty" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeaf4ad7403de93e699c191202f017118df734d3850b01e13a3a8b2e6953d3c9" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -4292,6 +4298,7 @@ dependencies = [ "jsonwebtoken", "lazy_static", "log", + "nonempty", "once_cell", "openssl", "parking_lot 0.12.1", @@ -4386,6 +4393,7 @@ dependencies = [ "enum-as-inner", "hex", "itertools", + "nonempty", "proptest", "rand 0.8.5", "serde", diff --git a/Cargo.toml b/Cargo.toml index 87f5300a929..880af797363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ itertools = "0.10.5" jsonwebtoken = { version = "8.1.0" } lazy_static = "1.4.0" log = "0.4.17" +nonempty = "0.8.1" once_cell = "1.16" parking_lot = { version = "0.12.1", features = ["send_guard", "arc_lock"] } pin-project-lite = "0.2.9" diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 42711043494..703f0b3a554 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -47,6 +47,7 @@ itertools.workspace = true jsonwebtoken.workspace = true lazy_static.workspace = true log.workspace = true +nonempty.workspace = true once_cell.workspace = true openssl.workspace = true parking_lot.workspace = true diff --git a/crates/core/src/db/datastore/locking_tx_datastore/btree_index.rs b/crates/core/src/db/datastore/locking_tx_datastore/btree_index.rs index 48ae4d650b1..6ba705c0bf4 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/btree_index.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/btree_index.rs @@ -3,6 +3,7 @@ use crate::{ db::datastore::traits::{IndexId, IndexSchema}, error::DBError, }; +use nonempty::NonEmpty; use spacetimedb_lib::{data_key::ToDataKey, DataKey}; use spacetimedb_sats::{AlgebraicValue, ProductValue}; use std::{ @@ -10,6 +11,26 @@ use std::{ ops::{Bound, RangeBounds}, }; +/// ## Index Key Composition +/// +/// [IndexKey] use an [AlgebraicValue] to optimize for the common case of *single columns* as key. +/// +/// See [ProductValue::project] for the logic. +/// +/// ### SQL Examples +/// +/// To illustrate the concept of single and multiple column indexes, consider the following SQL examples: +/// +/// ```sql +/// CREATE INDEX a ON t1 (column_i32); -- Creating a single column index, a common case. +/// CREATE INDEX b ON t1 (column_i32, column_i32); -- Creating a multiple column index for more complex requirements. +/// ``` +/// Will be on memory: +/// +/// ```rust,ignore +/// [AlgebraicValue::I32(0)] = Row(ProductValue(...)) +/// [AlgebraicValue::Product(AlgebraicValue::I32(0), AlgebraicValue::I32(1))] = Row(ProductValue(...)) +/// ``` #[derive(Clone, Eq, PartialEq, Ord, PartialOrd)] struct IndexKey { value: AlgebraicValue, @@ -57,28 +78,33 @@ impl Iterator for BTreeIndexRangeIter<'_> { pub(crate) struct BTreeIndex { pub(crate) index_id: IndexId, pub(crate) table_id: u32, - pub(crate) col_id: u32, + pub(crate) cols: NonEmpty, pub(crate) name: String, pub(crate) is_unique: bool, idx: BTreeSet, } impl BTreeIndex { - pub(crate) fn new(index_id: IndexId, table_id: u32, col_id: u32, name: String, is_unique: bool) -> Self { + pub(crate) fn new(index_id: IndexId, table_id: u32, cols: NonEmpty, name: String, is_unique: bool) -> Self { Self { index_id, table_id, - col_id, + cols, name, is_unique, idx: BTreeSet::new(), } } + pub(crate) fn get_fields(&self, row: &ProductValue) -> Result { + let fields = row.project_not_empty(&self.cols)?; + Ok(fields) + } + #[tracing::instrument(skip_all)] pub(crate) fn insert(&mut self, row: &ProductValue) -> Result<(), DBError> { - let col_value = row.get_field(self.col_id as usize, None)?; - let key = IndexKey::from_row(col_value, row.to_data_key()); + let col_value = self.get_fields(row)?; + let key = IndexKey::from_row(&col_value, row.to_data_key()); self.idx.insert(key); Ok(()) } @@ -92,8 +118,8 @@ impl BTreeIndex { #[tracing::instrument(skip_all)] pub(crate) fn violates_unique_constraint(&self, row: &ProductValue) -> bool { if self.is_unique { - let col_value = row.get_field(self.col_id as usize, None).unwrap(); - return self.contains_any(col_value); + let col_value = self.get_fields(row).unwrap(); + return self.contains_any(&col_value); } false } @@ -101,10 +127,9 @@ impl BTreeIndex { #[tracing::instrument(skip_all)] pub(crate) fn get_rows_that_violate_unique_constraint<'a>( &'a self, - row: &'a ProductValue, + row: &'a AlgebraicValue, ) -> Option> { - self.is_unique - .then(|| self.seek(row.get_field(self.col_id as usize, None).unwrap())) + self.is_unique.then(|| self.seek(row)) } /// Returns `true` if the [BTreeIndex] contains a value for the specified `value`. @@ -150,7 +175,7 @@ impl From<&BTreeIndex> for IndexSchema { IndexSchema { index_id: x.index_id.0, table_id: x.table_id, - col_id: x.col_id, + cols: x.cols.clone(), is_unique: x.is_unique, index_name: x.name.clone(), } diff --git a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs index 45d70491685..48784ab709f 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/mod.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/mod.rs @@ -7,6 +7,14 @@ use self::{ sequence::Sequence, table::Table, }; +use nonempty::NonEmpty; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + ops::RangeBounds, + sync::Arc, + vec, +}; + use super::{ system_tables::{ StColumnRow, StIndexRow, StSequenceRow, StTableRow, INDEX_ID_SEQUENCE_ID, SEQUENCE_ID_SEQUENCE_ID, @@ -47,12 +55,6 @@ use spacetimedb_lib::{ use spacetimedb_sats::{ AlgebraicType, AlgebraicValue, BuiltinType, BuiltinValue, ProductType, ProductTypeElement, ProductValue, }; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap}, - ops::RangeBounds, - sync::Arc, - vec, -}; use thiserror::Error; #[derive(Error, Debug, PartialEq, Eq)] @@ -172,7 +174,7 @@ impl CommittedState { // Add all newly created indexes to the committed state for (_, index) in table.indexes { - if !commit_table.indexes.contains_key(&ColId(index.col_id)) { + if !commit_table.indexes.contains_key(&index.cols.clone().map(ColId)) { commit_table.insert_index(index); } } @@ -206,11 +208,11 @@ impl CommittedState { pub fn index_seek<'a>( &'a self, table_id: &TableId, - col_id: &ColId, + cols: NonEmpty, range: &impl RangeBounds, ) -> Option> { if let Some(table) = self.tables.get(table_id) { - table.index_seek(*col_id, range) + table.index_seek(cols, range) } else { None } @@ -309,7 +311,7 @@ impl TxState { self.delete_tables.entry(table_id).or_insert_with(BTreeSet::new) } - /// When there's an index on `col_id`, + /// When there's an index on `cols`, /// returns an iterator over the [BTreeIndex] that yields all the `RowId`s /// that match the specified `value` in the indexed column. /// @@ -320,10 +322,10 @@ impl TxState { pub fn index_seek<'a>( &'a self, table_id: &TableId, - col_id: &ColId, + cols: NonEmpty, range: &impl RangeBounds, ) -> Option> { - self.insert_tables.get(table_id)?.index_seek(*col_id, range) + self.insert_tables.get(table_id)?.index_seek(cols, range) } } @@ -466,14 +468,14 @@ impl Inner { x if x.is_unique() => IndexSchema { index_id: constraint.constraint_id, table_id, - col_id: col_id.col_id, + cols: NonEmpty::new(col_id.col_id), index_name: format!("idx_{}", &constraint.constraint_name), is_unique: true, }, x if x.is_indexed() => IndexSchema { index_id: constraint.constraint_id, table_id, - col_id: col_id.col_id, + cols: NonEmpty::new(col_id.col_id), index_name: format!("idx_{}", &constraint.constraint_name), is_unique: false, }, @@ -492,7 +494,7 @@ impl Inner { let row = StIndexRow { index_id: index.index_id, table_id, - col_id: index.col_id, + cols: index.cols.clone(), index_name: &index.index_name, is_unique: index.is_unique, }; @@ -541,12 +543,12 @@ impl Inner { let mut index = BTreeIndex::new( IndexId(index_row.index_id), index_row.table_id, - index_row.col_id, + index_row.cols.clone(), index_row.index_name.into(), index_row.is_unique, ); index.build_from_rows(table.scan_rows())?; - table.indexes.insert(ColId(index_row.col_id), index); + table.indexes.insert(index_row.cols.map(ColId), index); } Ok(()) } @@ -868,7 +870,7 @@ impl Inner { let el = StIndexRow::try_from(row)?; let index_schema = IndexSchema { table_id: el.table_id, - col_id: el.col_id, + cols: el.cols, index_name: el.index_name.into(), is_unique: el.is_unique, index_id: el.index_id, @@ -963,10 +965,10 @@ impl Inner { fn create_index(&mut self, index: IndexDef) -> super::Result { log::trace!( - "INDEX CREATING: {} for table: {} and col: {}", + "INDEX CREATING: {} for table: {} and col(s): {:?}", index.name, index.table_id, - index.col_id + index.cols ); // Insert the index row into st_indexes @@ -975,7 +977,7 @@ impl Inner { let row = StIndexRow { index_id: 0, // Autogen'd table_id: index.table_id, - col_id: index.col_id, + cols: index.cols.clone(), index_name: &index.name, is_unique: index.is_unique, }; @@ -988,10 +990,10 @@ impl Inner { self.create_index_internal(IndexId(index_id), &index)?; log::trace!( - "INDEX CREATED: {} for table: {} and col: {}", + "INDEX CREATED: {} for table: {} and col(s): {:?}", index.name, index.table_id, - index.col_id + index.cols ); Ok(IndexId(index_id)) } @@ -1026,7 +1028,7 @@ impl Inner { let mut insert_index = BTreeIndex::new( index_id, index.table_id, - index.col_id, + index.cols.clone(), index.name.to_string(), index.is_unique, ); @@ -1039,13 +1041,13 @@ impl Inner { insert_table.schema.indexes.push(IndexSchema { table_id: index.table_id, - col_id: index.col_id, + cols: index.cols.clone(), index_name: index.name.to_string(), is_unique: index.is_unique, index_id: index_id.0, }); - insert_table.indexes.insert(ColId(index.col_id), insert_index); + insert_table.indexes.insert(index.cols.clone().map(ColId), insert_index); Ok(()) } @@ -1073,12 +1075,12 @@ impl Inner { let mut cols = vec![]; for index in table.indexes.values_mut() { if index.index_id == *index_id { - cols.push(index.col_id); + cols.push(index.cols.clone()); } } for col in cols { - table.indexes.remove(&ColId(col)); - table.schema.indexes.retain(|x| x.col_id != col); + table.indexes.remove(&col.clone().map(ColId)); + table.schema.indexes.retain(|x| x.cols != col); } } if let Some(insert_table) = self @@ -1090,12 +1092,12 @@ impl Inner { let mut cols = vec![]; for index in insert_table.indexes.values_mut() { if index.index_id == *index_id { - cols.push(index.col_id); + cols.push(index.cols.clone()); } } for col in cols { - insert_table.indexes.remove(&ColId(col)); - insert_table.schema.indexes.retain(|x| x.col_id != col); + insert_table.indexes.remove(&col.clone().map(ColId)); + insert_table.schema.indexes.retain(|x| x.cols != col); } } } @@ -1247,13 +1249,13 @@ impl Inner { indexes: committed_table .indexes .iter() - .map(|(col_id, index)| { + .map(|(cols, index)| { ( - *col_id, + cols.clone(), BTreeIndex::new( index.index_id, index.table_id, - index.col_id, + index.cols.clone(), index.name.clone(), index.is_unique, ), @@ -1269,40 +1271,53 @@ impl Inner { // Check unique constraints for index in insert_table.indexes.values() { if index.violates_unique_constraint(&row) { - let value = row.get_field(index.col_id as usize, None).unwrap(); + let value = row.project_not_empty(&index.cols).unwrap(); return Err(IndexError::UniqueConstraintViolation { constraint_name: index.name.clone(), table_name: insert_table.schema.table_name.clone(), - col_name: insert_table.schema.columns[index.col_id as usize].col_name.clone(), - value: value.clone(), + col_names: index + .cols + .iter() + .map(|&x| insert_table.schema.columns[x as usize].col_name.clone()) + .collect(), + value, } .into()); } } if let Some(table) = self.committed_state.tables.get_mut(&table_id) { for index in table.indexes.values() { - let Some(violators) = index.get_rows_that_violate_unique_constraint(&row) else { + let value = index.get_fields(&row)?; + let Some(violators) = index.get_rows_that_violate_unique_constraint(&value) else { continue; }; for row_id in violators { if let Some(delete_table) = self.tx_state.as_ref().unwrap().delete_tables.get(&table_id) { if !delete_table.contains(&row_id) { - let value = row.get_field(index.col_id as usize, None).unwrap(); + let value = row.project_not_empty(&index.cols)?; return Err(IndexError::UniqueConstraintViolation { constraint_name: index.name.clone(), table_name: table.schema.table_name.clone(), - col_name: table.schema.columns[index.col_id as usize].col_name.clone(), - value: value.clone(), + col_names: index + .cols + .iter() + .map(|&x| insert_table.schema.columns[x as usize].col_name.clone()) + .collect(), + value, } .into()); } } else { - let value = row.get_field(index.col_id as usize, None).unwrap(); + let value = row.project_not_empty(&index.cols)?; return Err(IndexError::UniqueConstraintViolation { constraint_name: index.name.clone(), table_name: table.schema.table_name.clone(), - col_name: table.schema.columns[index.col_id as usize].col_name.clone(), - value: value.clone(), + col_names: index + .cols + .iter() + .map(|&x| insert_table.schema.columns[x as usize].col_name.clone()) + .collect(), + value, } .into()); } @@ -1498,7 +1513,7 @@ impl Inner { if let Some(inserted_rows) = self .tx_state .as_ref() - .and_then(|tx_state| tx_state.index_seek(table_id, col_id, &range)) + .and_then(|tx_state| tx_state.index_seek(table_id, NonEmpty::new(*col_id), &range)) { // The current transaction has modified this table, and the table is indexed. let tx_state = self.tx_state.as_ref().unwrap(); @@ -1506,13 +1521,18 @@ impl Inner { table_id: *table_id, tx_state, inserted_rows, - committed_rows: self.committed_state.index_seek(table_id, col_id, &range), + committed_rows: self + .committed_state + .index_seek(table_id, NonEmpty::new(*col_id), &range), committed_state: &self.committed_state, })) } else { // Either the current transaction has not modified this table, or the table is not // indexed. - match self.committed_state.index_seek(table_id, col_id, &range) { + match self + .committed_state + .index_seek(table_id, NonEmpty::new(*col_id), &range) + { //If we don't have `self.tx_state` yet is likely we are running the bootstrap process Some(committed_rows) => match self.tx_state.as_ref() { None => Ok(IterByColRange::Scan(ScanIterByColRange { @@ -2101,6 +2121,7 @@ mod tests { error::{DBError, IndexError}, }; use itertools::Itertools; + use nonempty::NonEmpty; use spacetimedb_lib::{ auth::{StAccess, StTableType}, error::ResultTest, @@ -2135,13 +2156,13 @@ mod tests { indexes: vec![ IndexDef { table_id: 0, // Ignored - col_id: 0, + cols: NonEmpty::new(0), name: "id_idx".into(), is_unique: true, }, IndexDef { table_id: 0, // Ignored - col_id: 1, + cols: NonEmpty::new(1), name: "name_idx".into(), is_unique: true, }, @@ -2203,7 +2224,7 @@ mod tests { StColumnRow { table_id: 3, col_id: 0, col_name: "index_id".to_string(), col_type: AlgebraicType::U32, is_autoinc: true }, StColumnRow { table_id: 3, col_id: 1, col_name: "table_id".to_string(), col_type: AlgebraicType::U32, is_autoinc: false }, - StColumnRow { table_id: 3, col_id: 2, col_name: "col_id".to_string(), col_type: AlgebraicType::U32, is_autoinc: false }, + StColumnRow { table_id: 3, col_id: 2, col_name: "cols".to_string(), col_type: AlgebraicType::array(AlgebraicType::U32), is_autoinc: false }, StColumnRow { table_id: 3, col_id: 3, col_name: "index_name".to_string(), col_type: AlgebraicType::String, is_autoinc: false }, StColumnRow { table_id: 3, col_id: 4, col_name: "is_unique".to_string(), col_type: AlgebraicType::Bool, is_autoinc: false }, @@ -2223,12 +2244,12 @@ mod tests { assert_eq!( index_rows, vec![ - StIndexRow { index_id: 0, table_id: 0, col_id: 0, index_name: "table_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 1, table_id: 3, col_id: 0, index_name: "index_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 2, table_id: 2, col_id: 0, index_name: "sequences_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 3, table_id: 0, col_id: 1, index_name: "table_name_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 4, table_id: 4, col_id: 0, index_name: "constraint_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 5, table_id: 1, col_id: 0, index_name: "idx_ct_columns_table_id".to_string(), is_unique: false } + StIndexRow { index_id: 0, table_id: 0, cols: NonEmpty::new(0), index_name: "table_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 1, table_id: 3, cols: NonEmpty::new(0), index_name: "index_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 2, table_id: 2, cols: NonEmpty::new(0), index_name: "sequences_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 3, table_id: 0, cols: NonEmpty::new(1), index_name: "table_name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 4, table_id: 4, cols: NonEmpty::new(0), index_name: "constraint_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 5, table_id: 1, cols: NonEmpty::new(0), index_name: "idx_ct_columns_table_id".to_string(), is_unique: false } ] ); let sequence_rows = datastore @@ -2375,8 +2396,8 @@ mod tests { ColumnSchema { table_id: 5, col_id: 2, col_name: "age".to_string(), col_type: AlgebraicType::U32, is_autoinc: false }, ], indexes: vec![ - IndexSchema { index_id: 6, table_id: 5, col_id: 0, index_name: "id_idx".to_string(), is_unique: true }, - IndexSchema { index_id: 7, table_id: 5, col_id: 1, index_name: "name_idx".to_string(), is_unique: true }, + IndexSchema { index_id: 6, table_id: 5, cols: NonEmpty::new(0), index_name: "id_idx".to_string(), is_unique: true }, + IndexSchema { index_id: 7, table_id: 5, cols: NonEmpty::new(1), index_name: "name_idx".to_string(), is_unique: true }, ], constraints: vec![], table_type: StTableType::User, @@ -2404,8 +2425,8 @@ mod tests { ColumnSchema { table_id: 5, col_id: 2, col_name: "age".to_string(), col_type: AlgebraicType::U32, is_autoinc: false }, ], indexes: vec![ - IndexSchema { index_id: 6, table_id: 5, col_id: 0, index_name: "id_idx".to_string(), is_unique: true }, - IndexSchema { index_id: 7, table_id: 5, col_id: 1, index_name: "name_idx".to_string(), is_unique: true }, + IndexSchema { index_id: 6, table_id: 5, cols: NonEmpty::new(0), index_name: "id_idx".to_string(), is_unique: true }, + IndexSchema { index_id: 7, table_id: 5, cols: NonEmpty::new(1), index_name: "name_idx".to_string(), is_unique: true }, ], constraints: vec![], table_type: StTableType::User, @@ -2444,7 +2465,7 @@ mod tests { &mut tx, IndexDef { table_id: 5, - col_id: 0, + cols: NonEmpty::new(0), name: "id_idx".into(), is_unique: true, }, @@ -2453,7 +2474,7 @@ mod tests { let expected_indexes = vec![IndexSchema { index_id: 8, table_id: 5, - col_id: 0, + cols: NonEmpty::new(0), index_name: "id_idx".into(), is_unique: true, }]; @@ -2697,7 +2718,7 @@ mod tests { Err(DBError::Index(IndexError::UniqueConstraintViolation { constraint_name: _, table_name: _, - col_name: _, + col_names: _, value: _, })) => (), _ => panic!("Expected an unique constraint violation error."), @@ -2736,7 +2757,7 @@ mod tests { Err(DBError::Index(IndexError::UniqueConstraintViolation { constraint_name: _, table_name: _, - col_name: _, + col_names: _, value: _, })) => (), _ => panic!("Expected an unique constraint violation error."), @@ -2805,7 +2826,7 @@ mod tests { datastore.commit_mut_tx(tx)?; let mut tx = datastore.begin_mut_tx(); let index_def = IndexDef { - col_id: 2, + cols: NonEmpty::new(2), name: "age_idx".to_string(), is_unique: true, table_id: table_id.0, @@ -2818,15 +2839,15 @@ mod tests { .collect::>(); #[rustfmt::skip] assert_eq!(index_rows, vec![ - StIndexRow { index_id: 0, table_id: 0, col_id: 0, index_name: "table_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 1, table_id: 3, col_id: 0, index_name: "index_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 2, table_id: 2, col_id: 0, index_name: "sequences_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 3, table_id: 0, col_id: 1, index_name: "table_name_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 4, table_id: 4, col_id: 0, index_name: "constraint_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 5, table_id: 1, col_id: 0, index_name: "idx_ct_columns_table_id".to_string(), is_unique: false }, - StIndexRow { index_id: 6, table_id: 5, col_id: 0, index_name: "id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 7, table_id: 5, col_id: 1, index_name: "name_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 8, table_id: 5, col_id: 2, index_name: "age_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 0, table_id: 0, cols: NonEmpty::new(0), index_name: "table_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 1, table_id: 3, cols: NonEmpty::new(0), index_name: "index_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 2, table_id: 2, cols: NonEmpty::new(0), index_name: "sequences_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 3, table_id: 0, cols: NonEmpty::new(1), index_name: "table_name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 4, table_id: 4, cols: NonEmpty::new(0), index_name: "constraint_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 5, table_id: 1, cols: NonEmpty::new(0), index_name: "idx_ct_columns_table_id".to_string(), is_unique: false }, + StIndexRow { index_id: 6, table_id: 5, cols: NonEmpty::new(0), index_name: "id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 7, table_id: 5, cols: NonEmpty::new(1), index_name: "name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 8, table_id: 5, cols: NonEmpty::new(2), index_name: "age_idx".to_string(), is_unique: true }, ]); let row = ProductValue::from_iter(vec![ AlgebraicValue::U32(0), // 0 will be ignored. @@ -2838,7 +2859,7 @@ mod tests { Err(DBError::Index(IndexError::UniqueConstraintViolation { constraint_name: _, table_name: _, - col_name: _, + col_names: _, value: _, })) => (), _ => panic!("Expected an unique constraint violation error."), @@ -2874,7 +2895,7 @@ mod tests { let mut tx = datastore.begin_mut_tx(); let index_def = IndexDef { table_id: table_id.0, - col_id: 2, + cols: NonEmpty::new(2), name: "age_idx".to_string(), is_unique: true, }; @@ -2888,15 +2909,15 @@ mod tests { .collect::>(); #[rustfmt::skip] assert_eq!(index_rows, vec![ - StIndexRow { index_id: 0, table_id: 0, col_id: 0, index_name: "table_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 1, table_id: 3, col_id: 0, index_name: "index_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 2, table_id: 2, col_id: 0, index_name: "sequences_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 3, table_id: 0, col_id: 1, index_name: "table_name_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 4, table_id: 4, col_id: 0, index_name: "constraint_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 5, table_id: 1, col_id: 0, index_name: "idx_ct_columns_table_id".to_string(), is_unique: false }, - StIndexRow { index_id: 6, table_id: 5, col_id: 0, index_name: "id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 7, table_id: 5, col_id: 1, index_name: "name_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 8, table_id: 5, col_id: 2, index_name: "age_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 0, table_id: 0, cols: NonEmpty::new(0), index_name: "table_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 1, table_id: 3, cols: NonEmpty::new(0), index_name: "index_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 2, table_id: 2, cols: NonEmpty::new(0), index_name: "sequences_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 3, table_id: 0, cols: NonEmpty::new(1), index_name: "table_name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 4, table_id: 4, cols: NonEmpty::new(0), index_name: "constraint_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 5, table_id: 1, cols: NonEmpty::new(0), index_name: "idx_ct_columns_table_id".to_string(), is_unique: false }, + StIndexRow { index_id: 6, table_id: 5, cols: NonEmpty::new(0), index_name: "id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 7, table_id: 5, cols: NonEmpty::new(1), index_name: "name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 8, table_id: 5, cols: NonEmpty::new(2), index_name: "age_idx".to_string(), is_unique: true }, ]); let row = ProductValue::from_iter(vec![ AlgebraicValue::U32(0), // 0 will be ignored. @@ -2908,7 +2929,7 @@ mod tests { Err(DBError::Index(IndexError::UniqueConstraintViolation { constraint_name: _, table_name: _, - col_name: _, + col_names: _, value: _, })) => (), _ => panic!("Expected an unique constraint violation error."), @@ -2943,7 +2964,7 @@ mod tests { datastore.commit_mut_tx(tx)?; let mut tx = datastore.begin_mut_tx(); let index_def = IndexDef { - col_id: 2, + cols: NonEmpty::new(2), name: "age_idx".to_string(), is_unique: true, table_id: table_id.0, @@ -2958,14 +2979,14 @@ mod tests { .collect::>(); #[rustfmt::skip] assert_eq!(index_rows, vec![ - StIndexRow { index_id: 0, table_id: 0, col_id: 0, index_name: "table_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 1, table_id: 3, col_id: 0, index_name: "index_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 2, table_id: 2, col_id: 0, index_name: "sequences_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 3, table_id: 0, col_id: 1, index_name: "table_name_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 4, table_id: 4, col_id: 0, index_name: "constraint_id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 5, table_id: 1, col_id: 0, index_name: "idx_ct_columns_table_id".to_string(), is_unique: false }, - StIndexRow { index_id: 6, table_id: 5, col_id: 0, index_name: "id_idx".to_string(), is_unique: true }, - StIndexRow { index_id: 7, table_id: 5, col_id: 1, index_name: "name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 0, table_id: 0, cols: NonEmpty::new(0), index_name: "table_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 1, table_id: 3, cols: NonEmpty::new(0), index_name: "index_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 2, table_id: 2, cols: NonEmpty::new(0), index_name: "sequences_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 3, table_id: 0, cols: NonEmpty::new(1), index_name: "table_name_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 4, table_id: 4, cols: NonEmpty::new(0), index_name: "constraint_id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 5, table_id: 1, cols: NonEmpty::new(0), index_name: "idx_ct_columns_table_id".to_string(), is_unique: false }, + StIndexRow { index_id: 6, table_id: 5, cols: NonEmpty::new(0), index_name: "id_idx".to_string(), is_unique: true }, + StIndexRow { index_id: 7, table_id: 5, cols: NonEmpty::new(1), index_name: "name_idx".to_string(), is_unique: true }, ]); let row = ProductValue::from_iter(vec![ AlgebraicValue::U32(0), // 0 will be ignored. diff --git a/crates/core/src/db/datastore/locking_tx_datastore/table.rs b/crates/core/src/db/datastore/locking_tx_datastore/table.rs index f0d17b95786..8e9b68986ca 100644 --- a/crates/core/src/db/datastore/locking_tx_datastore/table.rs +++ b/crates/core/src/db/datastore/locking_tx_datastore/table.rs @@ -3,6 +3,7 @@ use super::{ RowId, }; use crate::db::datastore::traits::{ColId, TableSchema}; +use nonempty::NonEmpty; use spacetimedb_sats::{AlgebraicValue, ProductType, ProductValue}; use std::{ collections::{BTreeMap, HashMap}, @@ -12,14 +13,14 @@ use std::{ pub(crate) struct Table { pub(crate) row_type: ProductType, pub(crate) schema: TableSchema, - pub(crate) indexes: HashMap, + pub(crate) indexes: HashMap, BTreeIndex>, pub(crate) rows: BTreeMap, } impl Table { pub(crate) fn insert_index(&mut self, mut index: BTreeIndex) { index.build_from_rows(self.scan_rows()).unwrap(); - self.indexes.insert(ColId(index.col_id), index); + self.indexes.insert(index.cols.clone().map(ColId), index); } pub(crate) fn insert(&mut self, row_id: RowId, row: ProductValue) { @@ -31,9 +32,9 @@ impl Table { pub(crate) fn delete(&mut self, row_id: &RowId) -> Option { let row = self.rows.remove(row_id)?; - for (col_id, index) in self.indexes.iter_mut() { - let col_value = row.get_field(col_id.0 as usize, None).unwrap(); - index.delete(col_value, row_id) + for (cols, index) in self.indexes.iter_mut() { + let col_value = row.project_not_empty(&cols.clone().map(|x| x.0)).unwrap(); + index.delete(&col_value, row_id) } Some(row) } @@ -54,16 +55,16 @@ impl Table { self.rows.values() } - /// When there's an index for `col_id`, + /// When there's an index for `cols`, /// returns an iterator over the [`BTreeIndex`] that yields all the `RowId`s /// that match the specified `range` in the indexed column. /// /// Matching is defined by `Ord for AlgebraicValue`. pub(crate) fn index_seek( &self, - col_id: ColId, + cols: NonEmpty, range: &impl RangeBounds, ) -> Option> { - self.indexes.get(&col_id).map(|index| index.seek(range)) + self.indexes.get(&cols).map(|index| index.seek(range)) } } diff --git a/crates/core/src/db/datastore/system_tables.rs b/crates/core/src/db/datastore/system_tables.rs index 234644ac678..6135fa41cca 100644 --- a/crates/core/src/db/datastore/system_tables.rs +++ b/crates/core/src/db/datastore/system_tables.rs @@ -1,9 +1,11 @@ use super::traits::{ColumnSchema, IndexSchema, SequenceId, SequenceSchema, TableId, TableSchema}; use crate::db::datastore::traits::ConstraintSchema; use crate::error::{DBError, TableError}; +use nonempty::NonEmpty; use once_cell::sync::Lazy; use spacetimedb_lib::auth::{StAccess, StTableType}; use spacetimedb_lib::ColumnIndexAttribute; +use spacetimedb_sats::product_value::InvalidFieldError; use spacetimedb_sats::{product, AlgebraicType, AlgebraicValue, ArrayValue, ProductType, ProductValue}; /// The static ID of the table that defines tables @@ -123,7 +125,7 @@ impl StColumnFields { pub enum StIndexFields { IndexId = 0, TableId = 1, - ColId = 2, + Cols = 2, IndexName = 3, IsUnique = 4, } @@ -134,7 +136,7 @@ impl StIndexFields { match self { StIndexFields::IndexId => "index_id", StIndexFields::TableId => "table_id", - StIndexFields::ColId => "col_id", + StIndexFields::Cols => "cols", StIndexFields::IndexName => "index_name", StIndexFields::IsUnique => "is_unique", } @@ -208,14 +210,14 @@ pub fn st_table_schema() -> TableSchema { IndexSchema { index_id: ST_TABLE_ID_INDEX_ID, table_id: ST_TABLES_ID.0, - col_id: StTableFields::TableId as u32, + cols: NonEmpty::new(StTableFields::TableId as u32), index_name: "table_id_idx".into(), is_unique: true, }, IndexSchema { index_id: ST_TABLE_NAME_INDEX_ID, table_id: ST_TABLES_ID.0, - col_id: StTableFields::TableName as u32, + cols: NonEmpty::new(StTableFields::TableName as u32), index_name: "table_name_idx".into(), is_unique: true, }, @@ -325,9 +327,9 @@ pub static ST_COLUMNS_ROW_TYPE: Lazy = /// System Table [ST_INDEXES] /// -/// | index_id: u32 | table_id: u32 | col_id: u32 | index_name: String | is_unique: bool | -/// |---------------|---------------|-------------|--------------------|----------------------| -/// | 1 | 1 | 1 | "ix_sample" | 0 | +/// | index_id: u32 | table_id: u32 | cols: NonEmpty | index_name: String | is_unique: bool | +/// |---------------|---------------|---------------------|--------------------|----------------------| +/// | 1 | 1 | [1] | "ix_sample" | 0 | pub fn st_indexes_schema() -> TableSchema { TableSchema { table_id: ST_INDEXES_ID.0, @@ -336,7 +338,7 @@ pub fn st_indexes_schema() -> TableSchema { indexes: vec![IndexSchema { index_id: ST_INDEX_ID_INDEX_ID, table_id: ST_INDEXES_ID.0, - col_id: 0, + cols: NonEmpty::new(0), index_name: "index_id_idx".into(), is_unique: true, }], @@ -358,8 +360,8 @@ pub fn st_indexes_schema() -> TableSchema { ColumnSchema { table_id: ST_INDEXES_ID.0, col_id: 2, - col_name: "col_id".into(), - col_type: AlgebraicType::U32, + col_name: "cols".into(), + col_type: AlgebraicType::array(AlgebraicType::U32), is_autoinc: false, }, ColumnSchema { @@ -399,7 +401,7 @@ pub(crate) fn st_sequences_schema() -> TableSchema { indexes: vec![IndexSchema { index_id: ST_SEQUENCE_ID_INDEX_ID, table_id: ST_SEQUENCES_ID.0, - col_id: 0, + cols: NonEmpty::new(0), index_name: "sequences_id_idx".into(), is_unique: true, }], @@ -490,7 +492,7 @@ pub(crate) fn st_constraints_schema() -> TableSchema { indexes: vec![IndexSchema { index_id: ST_CONSTRAINT_ID_INDEX_ID, table_id: ST_CONSTRAINTS_ID.0, - col_id: 0, + cols: NonEmpty::new(0), index_name: "constraint_id_idx".into(), is_unique: true, }], @@ -671,7 +673,7 @@ impl> From<&StColumnRow> for ProductValue { pub struct StIndexRow> { pub(crate) index_id: u32, pub(crate) table_id: u32, - pub(crate) col_id: u32, + pub(crate) cols: NonEmpty, pub(crate) index_name: Name, pub(crate) is_unique: bool, } @@ -681,7 +683,7 @@ impl StIndexRow<&str> { StIndexRow { index_id: self.index_id, table_id: self.table_id, - col_id: self.col_id, + cols: self.cols.clone(), index_name: self.index_name.to_owned(), is_unique: self.is_unique, } @@ -693,13 +695,23 @@ impl<'a> TryFrom<&'a ProductValue> for StIndexRow<&'a str> { fn try_from(row: &'a ProductValue) -> Result, DBError> { let index_id = row.field_as_u32(StIndexFields::IndexId as usize, None)?; let table_id = row.field_as_u32(StIndexFields::TableId as usize, None)?; - let col_id = row.field_as_u32(StIndexFields::ColId as usize, None)?; + let cols = row.field_as_array(StIndexFields::Cols as usize, None)?; + let cols = if let ArrayValue::U32(x) = cols { + NonEmpty::from_slice(x).unwrap() + } else { + return Err(InvalidFieldError { + col_pos: StIndexFields::Cols as usize, + name: StIndexFields::Cols.name().into(), + } + .into()); + }; + let index_name = row.field_as_str(StIndexFields::IndexName as usize, None)?; let is_unique = row.field_as_bool(StIndexFields::IsUnique as usize, None)?; Ok(StIndexRow { index_id, table_id, - col_id, + cols, index_name, is_unique, }) @@ -711,7 +723,7 @@ impl> From<&StIndexRow> for ProductValue { product![ AlgebraicValue::U32(x.index_id), AlgebraicValue::U32(x.table_id), - AlgebraicValue::U32(x.col_id), + AlgebraicValue::ArrayOf(x.cols.clone()), AlgebraicValue::String(x.index_name.as_ref().to_string()), AlgebraicValue::Bool(x.is_unique) ] diff --git a/crates/core/src/db/datastore/traits.rs b/crates/core/src/db/datastore/traits.rs index 4409bcb9595..d35854a30c1 100644 --- a/crates/core/src/db/datastore/traits.rs +++ b/crates/core/src/db/datastore/traits.rs @@ -1,8 +1,10 @@ use crate::db::relational_db::ST_TABLES_ID; use core::fmt; +use nonempty::NonEmpty; use spacetimedb_lib::auth::{StAccess, StTableType}; use spacetimedb_lib::relation::{DbTable, FieldName, FieldOnly, Header, TableField}; use spacetimedb_lib::{ColumnIndexAttribute, DataKey}; +use spacetimedb_sats::product_value::InvalidFieldError; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductType, ProductTypeElement, ProductValue}; use spacetimedb_vm::expr::SourceExpr; use std::{ops::RangeBounds, sync::Arc}; @@ -79,16 +81,16 @@ pub struct SequenceDef { pub struct IndexSchema { pub(crate) index_id: u32, pub(crate) table_id: u32, - pub(crate) col_id: u32, pub(crate) index_name: String, pub(crate) is_unique: bool, + pub(crate) cols: NonEmpty, } /// This type is just the [IndexSchema] without the autoinc fields #[derive(Debug, Clone, PartialEq, Eq)] pub struct IndexDef { pub(crate) table_id: u32, - pub(crate) col_id: u32, + pub(crate) cols: NonEmpty, pub(crate) name: String, pub(crate) is_unique: bool, } @@ -96,7 +98,7 @@ pub struct IndexDef { impl IndexDef { pub fn new(name: String, table_id: u32, col_id: u32, is_unique: bool) -> Self { Self { - col_id, + cols: NonEmpty::new(col_id), name, is_unique, table_id, @@ -108,7 +110,7 @@ impl From for IndexDef { fn from(value: IndexSchema) -> Self { Self { table_id: value.table_id, - col_id: value.col_id, + cols: value.cols, name: value.index_name, is_unique: value.is_unique, } @@ -231,6 +233,27 @@ impl TableSchema { pub fn normalize_field(&self, or_use: &TableField) -> FieldName { FieldName::named(or_use.table.unwrap_or(&self.table_name), or_use.field) } + + /// Project the fields from the supplied `columns`. + pub fn project(&self, columns: impl Iterator) -> Result> { + columns + .map(|pos| { + self.get_column(pos).ok_or( + InvalidFieldError { + col_pos: pos, + name: None, + } + .into(), + ) + }) + .collect() + } + + /// Utility for project the fields from the supplied `columns` that is a [NonEmpty], + /// used for when the list of field columns have at least one value. + pub fn project_not_empty(&self, columns: &NonEmpty) -> Result> { + self.project(columns.iter().map(|&x| x as usize)) + } } impl From<&TableSchema> for ProductType { diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index c0ad920543b..5a5edd58039 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -12,10 +12,11 @@ use crate::db::db_metrics::{RDB_DELETE_BY_REL_TIME, RDB_DROP_TABLE_TIME, RDB_INS use crate::db::messages::commit::Commit; use crate::db::ostorage::hashmap_object_db::HashMapObjectDB; use crate::db::ostorage::ObjectDB; -use crate::error::{DBError, DatabaseError, TableError}; +use crate::error::{DBError, DatabaseError, IndexError, TableError}; use crate::hash::Hash; use crate::util::prometheus_handle::HistogramVecHandle; use fs2::FileExt; +use nonempty::NonEmpty; use prometheus::HistogramVec; use spacetimedb_lib::ColumnIndexAttribute; use spacetimedb_lib::{data_key::ToDataKey, PrimaryKey}; @@ -397,15 +398,23 @@ impl RelationalDB { &self, tx: &mut MutTxId, table_id: u32, - col_id: u32, - ) -> Result, DBError> { + cols: &NonEmpty, + ) -> Result { let table = self.inner.schema_for_table_mut_tx(tx, TableId(table_id))?; - let Some(column) = table.columns.get(col_id as usize) else { - return Ok(None); + let columns = table.project_not_empty(cols)?; + // Verify we don't have more than 1 auto_inc in the list of columns + let autoinc = columns.iter().filter(|x| x.is_autoinc).count(); + let is_autoinc = if autoinc < 2 { + autoinc == 1 + } else { + return Err(DBError::Index(IndexError::OneAutoInc( + TableId(table_id), + columns.iter().map(|x| x.col_name.clone()).collect(), + ))); }; - let unique_index = table.indexes.iter().find(|x| x.col_id == col_id).map(|x| x.is_unique); + let unique_index = table.indexes.iter().find(|x| &x.cols == cols).map(|x| x.is_unique); let mut attr = ColumnIndexAttribute::UNSET; - if column.is_autoinc { + if is_autoinc { attr |= ColumnIndexAttribute::AUTO_INC; } if let Some(is_unique) = unique_index { @@ -415,7 +424,7 @@ impl RelationalDB { ColumnIndexAttribute::INDEXED }; } - Ok(Some(attr)) + Ok(attr) } #[tracing::instrument(skip_all)] @@ -458,7 +467,7 @@ impl RelationalDB { /// Returns an iterator, /// yielding every row in the table identified by `table_id`, - /// where the column data identified by `col_id` matches `value`. + /// where the column data identified by `cols` matches `value`. /// /// Matching is defined by `Ord for AlgebraicValue`. #[tracing::instrument(skip(self, tx))] @@ -475,7 +484,7 @@ impl RelationalDB { /// Returns an iterator, /// yielding every row in the table identified by `table_id`, - /// where the column data identified by `col_id` matches what is within `range`. + /// where the column data identified by `cols` matches what is within `range`. /// /// Matching is defined by `Ord for AlgebraicValue`. pub fn iter_by_col_range<'a, R: RangeBounds>( @@ -590,6 +599,7 @@ pub(crate) mod tests_utils { mod tests { #![allow(clippy::disallowed_macros)] + use nonempty::NonEmpty; use std::sync::{Arc, Mutex}; use crate::address::Address; @@ -984,7 +994,7 @@ mod tests { }], indexes: vec![IndexDef { table_id: 0, - col_id: 0, + cols: NonEmpty::new(0), name: "MyTable_my_col_idx".to_string(), is_unique: false, }], @@ -1026,7 +1036,7 @@ mod tests { }], indexes: vec![IndexDef { table_id: 0, - col_id: 0, + cols: NonEmpty::new(0), name: "MyTable_my_col_idx".to_string(), is_unique: true, }], @@ -1073,7 +1083,7 @@ mod tests { }], indexes: vec![IndexDef { table_id: 0, - col_id: 0, + cols: NonEmpty::new(0), name: "MyTable_my_col_idx".to_string(), is_unique: true, }], @@ -1136,19 +1146,19 @@ mod tests { indexes: vec![ IndexDef { table_id: 0, - col_id: 0, + cols: NonEmpty::new(0), name: "MyTable_col1_idx".to_string(), is_unique: true, }, IndexDef { table_id: 0, - col_id: 2, + cols: NonEmpty::new(2), name: "MyTable_col3_idx".to_string(), is_unique: false, }, IndexDef { table_id: 0, - col_id: 3, + cols: NonEmpty::new(3), name: "MyTable_col4_idx".to_string(), is_unique: true, }, @@ -1206,7 +1216,7 @@ mod tests { }], indexes: vec![IndexDef { table_id: 0, - col_id: 0, + cols: NonEmpty::new(0), name: "MyTable_my_col_idx".to_string(), is_unique: true, }], diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index ca78831715e..790a959f335 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -1,5 +1,5 @@ use crate::client::ClientActorId; -use crate::db::datastore::traits::{IndexDef, IndexId}; +use crate::db::datastore::traits::{IndexDef, IndexId, TableId}; use hex::FromHexError; use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::error::{LibError, RelationError}; @@ -60,13 +60,15 @@ pub enum IndexError { IndexAlreadyExists(IndexDef, String), #[error("Column not found: {0:?}")] ColumnNotFound(IndexDef), - #[error("Unique constraint violation '{}' in table '{}': column: '{}' value: {}", constraint_name, table_name, col_name, value.to_satn())] + #[error("Unique constraint violation '{}' in table '{}': column(s): '{:?}' value: {}", constraint_name, table_name, col_names, value.to_satn())] UniqueConstraintViolation { constraint_name: String, table_name: String, - col_name: String, + col_names: Vec, value: AlgebraicValue, }, + #[error("Attempt to define a index with more than 1 auto_inc column: Table: {0:?}, Columns: {1:?}")] + OneAutoInc(TableId, Vec), } #[derive(Error, Debug, PartialEq, Eq)] diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 2e6ec940f7e..29c73e28cca 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,3 +1,4 @@ +use nonempty::NonEmpty; use parking_lot::{Mutex, MutexGuard}; use spacetimedb_lib::{bsatn, ProductValue}; use std::ops::DerefMut; @@ -77,7 +78,7 @@ impl InstanceEnv { crate::error::DBError::Index(IndexError::UniqueConstraintViolation { constraint_name: _, table_name: _, - col_name: _, + col_names: _, value: _, }) => {} _ => { @@ -139,7 +140,7 @@ impl InstanceEnv { */ /// Deletes all rows in the table identified by `table_id` - /// where the column identified by `col_id` equates to `value`. + /// where the column identified by `cols` equates to `value`. /// /// Returns an error if no columns were deleted or if the column wasn't found. #[tracing::instrument(skip_all)] @@ -168,7 +169,7 @@ impl InstanceEnv { pub fn delete_range( &self, table_id: u32, - col_id: u32, + cols: u32, start_buffer: &[u8], end_buffer: &[u8], ) -> Result { @@ -177,13 +178,13 @@ impl InstanceEnv { let stdb = &*self.dbic.relational_db; let tx = &mut *self.get_tx()?; - let col_type = stdb.schema_for_column(tx, table_id, col_id)?; + let col_type = stdb.schema_for_column(tx, table_id, cols)?; let decode = |b: &[u8]| AlgebraicValue::decode(&col_type, &mut &b[..]).map_err(NodesError::DecodeValue); let start = decode(start_buffer)?; let end = decode(end_buffer)?; - let range = stdb.range_scan(tx, table_id, col_id, start..end)?; + let range = stdb.range_scan(tx, table_id, cols, start..end)?; let range = range.map(|x| stdb.data_to_owned(x).into()).collect::>(); let count = stdb.delete_in(tx, table_id, range)?.ok_or(NodesError::RangeNotFound)?; @@ -193,7 +194,7 @@ impl InstanceEnv { measure.start_instant.unwrap(), measure.elapsed(), table_id, - col_id, + cols, start_buffer.into(), end_buffer.into(), count, @@ -249,12 +250,11 @@ impl InstanceEnv { /// on a product of the given columns in `col_ids`, /// in the table identified by `table_id`. /// - /// Currently only single-column-indices are supported - /// and they may only be of the btree index type. + /// Currently only btree index type are supported. /// /// The `table_name` is used together with the column ids to construct the name of the index. /// As only single-column-indices are supported right now, - /// the name will be in the format `{table_name}_{col_id}`. + /// the name will be in the format `{table_name}_{cols}`. #[tracing::instrument(skip_all)] pub fn create_index( &self, @@ -276,20 +276,15 @@ impl InstanceEnv { _ => return Err(NodesError::BadIndexType(index_type)), }; - // TODO(george) The index API right now only allows single column indexes. - let col_id = match &*col_ids { - [id] => *id as u32, - _ => todo!("Multi-column indexes are not yet supported"), - }; + let cols = NonEmpty::from_slice(&col_ids) + .expect("Attempt to create an index with zero columns") + .map(|x| x as u32); - let is_unique = stdb - .column_attrs(tx, table_id, col_id)? - .expect("invalid col_id") - .is_unique(); + let is_unique = stdb.column_attrs(tx, table_id, &cols)?.is_unique(); let index = IndexDef { table_id, - col_id, + cols, name: index_name, is_unique, }; @@ -300,7 +295,7 @@ impl InstanceEnv { } /// Finds all rows in the table identified by `table_id` - /// where the column identified by `col_id` matches to `value`. + /// where the column identified by `cols` matches to `value`. /// /// These rows are returned concatenated with each row bsatn encoded. /// diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 1e7a9f853bc..f6c71327fa7 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -329,7 +329,7 @@ pub fn err_to_errno(err: &NodesError) -> Option { DBError::Index(IndexError::UniqueConstraintViolation { constraint_name: _, table_name: _, - col_name: _, + col_names: _, value: _, }) => Some(errnos::UNIQUE_ALREADY_EXISTS), _ => None, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index f7141708fdd..92a9e73d623 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -8,6 +8,7 @@ use crate::host::scheduler::Scheduler; use crate::sql; use anyhow::Context; use bytes::Bytes; +use nonempty::NonEmpty; use spacetimedb_lib::buffer::DecodeError; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::{bsatn, IndexType, ModuleDef}; @@ -720,7 +721,8 @@ impl WasmModuleInstance { let mut index_for_column = None; for index in table.indexes.iter() { let [index_col_id] = *index.col_ids else { - anyhow::bail!("multi-column indexes not yet supported") + //Ignore multi-column indexes + continue; }; if index_col_id as usize != col_id { continue; @@ -740,7 +742,7 @@ impl WasmModuleInstance { } let index = IndexDef { table_id: 0, // Will be ignored - col_id: col_id as u32, + cols: NonEmpty::new(col_id as u32), name: index.name.clone(), is_unique: col_attr.is_unique(), }; @@ -750,7 +752,7 @@ impl WasmModuleInstance { // anyway. let index = IndexDef { table_id: 0, // Will be ignored - col_id: col_id as u32, + cols: NonEmpty::new(col_id as u32), name: format!("{}_{}_unique", table.name, col.col_name), is_unique: true, }; diff --git a/crates/core/src/host/wasmer/wasm_instance_env.rs b/crates/core/src/host/wasmer/wasm_instance_env.rs index 93ec96894f7..04d5ac03044 100644 --- a/crates/core/src/host/wasmer/wasm_instance_env.rs +++ b/crates/core/src/host/wasmer/wasm_instance_env.rs @@ -231,7 +231,7 @@ impl WasmInstanceEnv { } /// Deletes all rows in the table identified by `table_id` - /// where the column identified by `col_id` matches the byte string, + /// where the column identified by `cols` matches the byte string, /// in WASM memory, pointed to at by `value`. /// /// Matching is defined by decoding of `value` to an `AlgebraicValue` @@ -289,7 +289,7 @@ impl WasmInstanceEnv { pub fn delete_range( caller: FunctionEnvMut<'_, Self>, table_id: u32, - col_id: u32, + cols: u32, range_start: WasmPtr, range_start_len: u32, range_end: WasmPtr, @@ -302,7 +302,7 @@ impl WasmInstanceEnv { let n_deleted = caller .data() .instance_env - .delete_range(table_id, col_id, &start, &end)?; + .delete_range(table_id, cols, &start, &end)?; Ok(n_deleted) }) } @@ -384,7 +384,6 @@ impl WasmInstanceEnv { // Read the column ids on which to create an index from WASM memory. // This may be one column or an index on several columns. - // TODO(george) The index API right now only allows single column indexes. let cols = mem.read_bytes(&caller, col_ids, col_len)?; caller @@ -396,7 +395,7 @@ impl WasmInstanceEnv { } /// Finds all rows in the table identified by `table_id`, - /// where the row has a column, identified by `col_id`, + /// where the row has a column, identified by `cols`, /// with data matching the byte string, in WASM memory, pointed to at by `val`. /// /// Matching is defined by decoding of `value` to an `AlgebraicValue` diff --git a/crates/core/src/messages/instance_db_trace_log.rs b/crates/core/src/messages/instance_db_trace_log.rs index 9a083e2bb4b..ef0a5c757a1 100644 --- a/crates/core/src/messages/instance_db_trace_log.rs +++ b/crates/core/src/messages/instance_db_trace_log.rs @@ -33,7 +33,7 @@ pub struct DeleteByColEq { #[derive(Clone, Serialize, Deserialize)] pub struct DeleteRange { pub table_id: u32, - pub col_id: u32, + pub cols: u32, pub start_buffer: Vec, pub end_buffer: Vec, pub result_deleted_count: u32, diff --git a/crates/core/src/sql/compiler.rs b/crates/core/src/sql/compiler.rs index 40cdcaa415b..515ba45e6f0 100644 --- a/crates/core/src/sql/compiler.rs +++ b/crates/core/src/sql/compiler.rs @@ -1,3 +1,4 @@ +use nonempty::NonEmpty; use std::collections::HashMap; use crate::db::datastore::locking_tx_datastore::MutTxId; @@ -154,34 +155,39 @@ fn is_sargable(table: &TableSchema, op: &ColumnOp) -> Option { // lhs field must exist let column = table.get_column_by_field(name)?; // lhs field must have an index - let index = table.indexes.iter().find(|index| index.col_id == column.col_id)?; + let index = table + .indexes + .iter() + .find(|index| index.cols == NonEmpty::new(column.col_id))?; + + assert_eq!(index.cols.len(), 1, "No yet supported multi-column indexes"); match op { OpCmp::Eq => Some(IndexArgument::Eq { - col_id: index.col_id, + col_id: index.cols.head, value: value.clone(), }), // a < 5 => exclusive upper bound OpCmp::Lt => Some(IndexArgument::UpperBound { - col_id: index.col_id, + col_id: index.cols.head, value: value.clone(), inclusive: false, }), // a > 5 => exclusive lower bound OpCmp::Gt => Some(IndexArgument::LowerBound { - col_id: index.col_id, + col_id: index.cols.head, value: value.clone(), inclusive: false, }), // a <= 5 => inclusive upper bound OpCmp::LtEq => Some(IndexArgument::UpperBound { - col_id: index.col_id, + col_id: index.cols.head, value: value.clone(), inclusive: true, }), // a >= 5 => inclusive lower bound OpCmp::GtEq => Some(IndexArgument::LowerBound { - col_id: index.col_id, + col_id: index.cols.head, value: value.clone(), inclusive: true, }), @@ -430,7 +436,7 @@ mod tests { .iter() .map(|(col_id, index_name)| IndexDef { table_id: 0, - col_id: *col_id, + cols: NonEmpty::new(*col_id), name: index_name.to_string(), is_unique: false, }) diff --git a/crates/core/src/vm.rs b/crates/core/src/vm.rs index 079e0854bb0..5c5607061d3 100644 --- a/crates/core/src/vm.rs +++ b/crates/core/src/vm.rs @@ -4,6 +4,7 @@ use crate::db::datastore::locking_tx_datastore::MutTxId; use crate::db::datastore::traits::{ColumnDef, IndexDef, IndexId, SequenceId, TableDef}; use crate::db::relational_db::RelationalDB; use itertools::Itertools; +use nonempty::NonEmpty; use spacetimedb_lib::auth::{StAccess, StTableType}; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::relation::{DbTable, FieldExpr, Relation}; @@ -226,7 +227,7 @@ impl<'db, 'tx> DbProgram<'db, 'tx> { if meta.is_unique() { indexes.push(IndexDef { table_id: 0, // Ignored - col_id: i as u32, + cols: NonEmpty::new(i as u32), name: format!("{}_{}_idx", table_name, i), is_unique: true, }); @@ -611,7 +612,7 @@ pub(crate) mod tests { index_id: index_id.0, index_name: "idx_1", table_id, - col_id: 0, + cols: NonEmpty::new(0), is_unique: true, }) .into(), diff --git a/crates/lib/src/table.rs b/crates/lib/src/table.rs index 49765f49c6a..6441b978381 100644 --- a/crates/lib/src/table.rs +++ b/crates/lib/src/table.rs @@ -117,12 +117,3 @@ impl From for ProductType { value.columns } } - -impl<'a> FromIterator<&'a (&'a str, AlgebraicType, ColumnIndexAttribute)> for ProductTypeMeta { - fn from_iter>(iter: T) -> Self { - Self::with_attributes( - iter.into_iter() - .map(|(name, ty, attr)| (ProductTypeElement::new(ty.clone(), Some(name.to_string())), *attr)), - ) - } -} diff --git a/crates/sats/Cargo.toml b/crates/sats/Cargo.toml index 31c7728a95b..990f3c9932a 100644 --- a/crates/sats/Cargo.toml +++ b/crates/sats/Cargo.toml @@ -19,6 +19,7 @@ derive_more.workspace = true enum-as-inner.workspace = true hex = { workspace = true, optional = true } itertools.workspace = true +nonempty.workspace = true serde = { workspace = true, optional = true } thiserror.workspace = true tracing.workspace = true diff --git a/crates/sats/src/builtin_value.rs b/crates/sats/src/builtin_value.rs index e3468746066..ff71d27ffc3 100644 --- a/crates/sats/src/builtin_value.rs +++ b/crates/sats/src/builtin_value.rs @@ -2,6 +2,8 @@ use crate::algebraic_value::AlgebraicValue; use crate::builtin_type::BuiltinType; use crate::{AlgebraicType, ArrayType}; use enum_as_inner::EnumAsInner; +use itertools::Itertools; +use nonempty::NonEmpty; use std::collections::BTreeMap; use std::fmt; @@ -349,6 +351,16 @@ impl_from_array!(String, String); impl_from_array!(ArrayValue, Array); impl_from_array!(MapValue, Map); +impl From> for ArrayValue +where + ArrayValue: From>, +{ + fn from(v: NonEmpty) -> Self { + let arr = v.iter().cloned().collect_vec(); + arr.into() + } +} + impl ArrayValue { /// Returns `self` as `&dyn Debug`. fn as_dyn_debug(&self) -> &dyn fmt::Debug { diff --git a/crates/sats/src/product_value.rs b/crates/sats/src/product_value.rs index a59c0f2b0a0..648a8490960 100644 --- a/crates/sats/src/product_value.rs +++ b/crates/sats/src/product_value.rs @@ -1,6 +1,7 @@ use crate::algebraic_value::AlgebraicValue; use crate::product_type::ProductType; use crate::ArrayValue; +use nonempty::NonEmpty; /// A product value is made of a a list of /// "elements" / "fields" / "factors" of other `AlgebraicValue`s. @@ -46,10 +47,10 @@ impl crate::Value for ProductValue { /// An error that occurs when a field, of a product value, is accessed that doesn't exist. #[derive(thiserror::Error, Debug, Copy, Clone)] -#[error("Field {index}({name:?}) not found or has an invalid type")] +#[error("Field {col_pos}({name:?}) not found or has an invalid type")] pub struct InvalidFieldError { - /// The claimed index of the field within the product value. - pub index: usize, + /// The claimed col_pos of the field within the product value. + pub col_pos: usize, /// The name of the field, if any. pub name: Option<&'static str>, } @@ -59,7 +60,51 @@ impl ProductValue { /// /// The `name` is non-functional and is only used for error-messages. pub fn get_field(&self, index: usize, name: Option<&'static str>) -> Result<&AlgebraicValue, InvalidFieldError> { - self.elements.get(index).ok_or(InvalidFieldError { index, name }) + self.elements + .get(index) + .ok_or(InvalidFieldError { col_pos: index, name }) + } + + /// This function is used to project fields based on the provided `indexes`. + /// + /// It will raise an [InvalidFieldError] if any of the supplied `indexes` cannot be found. + /// + /// The optional parameter `name: Option<&'static str>` serves a non-functional role and is + /// solely utilized for generating error messages. + /// + /// **Important:** + /// + /// The resulting [AlgebraicValue] will wrap into a [ProductValue] when projecting multiple + /// fields, otherwise it will consist of a single [AlgebraicValue]. + /// + pub fn project(&self, indexes: &[(usize, Option<&'static str>)]) -> Result { + let fields = match indexes { + [(index, name)] => self.get_field(*index, *name)?.clone(), + indexes => { + let fields: Result, _> = indexes + .iter() + .map(|(index, name)| self.get_field(*index, *name).cloned()) + .collect(); + AlgebraicValue::Product(ProductValue::new(&fields?)) + } + }; + + Ok(fields) + } + + /// This utility function is designed to project fields based on the supplied `indexes`. + /// + /// **Important:** + /// + /// The resulting [AlgebraicValue] will wrap into a [ProductValue] when projecting multiple + /// fields, otherwise it will consist of a single [AlgebraicValue]. + /// + /// **Parameters:** + /// - `indexes`: A [NonEmpty] containing the indexes of fields to be projected. + /// + pub fn project_not_empty(&self, indexes: &NonEmpty) -> Result { + let indexes: Vec<_> = indexes.iter().map(|x| (*x as usize, None)).collect(); + self.project(&indexes) } /// Extracts the `value` at field of `self` identified by `index` @@ -70,7 +115,7 @@ impl ProductValue { name: Option<&'static str>, f: impl 'a + Fn(&'a AlgebraicValue) -> Option, ) -> Result { - f(self.get_field(index, name)?).ok_or(InvalidFieldError { index, name }) + f(self.get_field(index, name)?).ok_or(InvalidFieldError { col_pos: index, name }) } /// Interprets the value at field of `self` identified by `index` as a `bool`.