From 98c9399247854cbae9f8e7e0933067483b1cd33c Mon Sep 17 00:00:00 2001 From: James Gilles <jameshgilles@gmail.com> Date: Mon, 16 Dec 2024 15:38:34 -0500 Subject: [PATCH 1/2] Add pretty printing code for automigrations to schema --- crates/core/src/db/update.rs | 14 +- crates/schema/Cargo.toml | 2 + crates/schema/src/auto_migrate.rs | 13 +- .../schema/src/auto_migrate/pretty_print.rs | 266 ++++++++++++++++++ 4 files changed, 271 insertions(+), 24 deletions(-) create mode 100644 crates/schema/src/auto_migrate/pretty_print.rs diff --git a/crates/core/src/db/update.rs b/crates/core/src/db/update.rs index 8b040dffca..53d37ca6b9 100644 --- a/crates/core/src/db/update.rs +++ b/crates/core/src/db/update.rs @@ -7,7 +7,7 @@ use spacetimedb_lib::db::auth::StTableType; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::AlgebraicValue; use spacetimedb_primitives::ColSet; -use spacetimedb_schema::auto_migrate::{AutoMigratePlan, ManualMigratePlan, MigratePlan}; +use spacetimedb_schema::auto_migrate::{AutoMigratePlan, MigratePlan}; use spacetimedb_schema::def::TableDef; use spacetimedb_schema::schema::{IndexSchema, Schema, SequenceSchema, TableSchema}; use std::sync::Arc; @@ -45,22 +45,10 @@ pub fn update_database( } match plan { - MigratePlan::Manual(plan) => manual_migrate_database(stdb, tx, plan, system_logger, existing_tables), MigratePlan::Auto(plan) => auto_migrate_database(stdb, tx, auth_ctx, plan, system_logger, existing_tables), } } -/// Manually migrate a database. -fn manual_migrate_database( - _stdb: &RelationalDB, - _tx: &mut MutTxId, - _plan: ManualMigratePlan, - _system_logger: &SystemLogger, - _existing_tables: Vec<Arc<TableSchema>>, -) -> anyhow::Result<()> { - unimplemented!("Manual database migrations are not yet implemented") -} - /// Automatically migrate a database. fn auto_migrate_database( stdb: &RelationalDB, diff --git a/crates/schema/Cargo.toml b/crates/schema/Cargo.toml index 952d1a3b28..4442d64eff 100644 --- a/crates/schema/Cargo.toml +++ b/crates/schema/Cargo.toml @@ -16,7 +16,9 @@ spacetimedb-sats.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-sql-parser.workspace = true +regex.workspace = true anyhow.workspace = true +colored.workspace = true indexmap.workspace = true itertools.workspace = true lazy_static.workspace = true diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 977c2e7e77..9e46651a04 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -6,12 +6,13 @@ use spacetimedb_data_structures::{ use spacetimedb_lib::db::raw_def::v9::{RawRowLevelSecurityDefV9, TableType}; use spacetimedb_sats::WithTypespace; +pub mod pretty_print; + pub type Result<T> = std::result::Result<T, ErrorStream<AutoMigrateError>>; /// A plan for a migration. #[derive(Debug)] pub enum MigratePlan<'def> { - Manual(ManualMigratePlan<'def>), Auto(AutoMigratePlan<'def>), } @@ -19,7 +20,6 @@ impl<'def> MigratePlan<'def> { /// Get the old `ModuleDef` for this migration plan. pub fn old_def(&self) -> &'def ModuleDef { match self { - MigratePlan::Manual(plan) => plan.old, MigratePlan::Auto(plan) => plan.old, } } @@ -27,20 +27,11 @@ impl<'def> MigratePlan<'def> { /// Get the new `ModuleDef` for this migration plan. pub fn new_def(&self) -> &'def ModuleDef { match self { - MigratePlan::Manual(plan) => plan.new, MigratePlan::Auto(plan) => plan.new, } } } -/// A plan for a manual migration. -/// `new` must have a reducer marked with `Lifecycle::Update`. -#[derive(Debug)] -pub struct ManualMigratePlan<'def> { - pub old: &'def ModuleDef, - pub new: &'def ModuleDef, -} - /// A plan for an automatic migration. #[derive(Debug)] pub struct AutoMigratePlan<'def> { diff --git a/crates/schema/src/auto_migrate/pretty_print.rs b/crates/schema/src/auto_migrate/pretty_print.rs new file mode 100644 index 0000000000..b77c1aaf08 --- /dev/null +++ b/crates/schema/src/auto_migrate/pretty_print.rs @@ -0,0 +1,266 @@ +//! This module provides a function [`pretty_print`](pretty_print) that renders an automatic migration plan to a string. + +use super::{AutoMigratePlan, IndexAlgorithm, TableDef}; +use crate::{auto_migrate::AutoMigrateStep, def::ConstraintData}; +use colored::{self, ColoredString, Colorize}; +use lazy_static::lazy_static; +use regex::Regex; +use spacetimedb_lib::db::raw_def::v9::{TableAccess, TableType}; +use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_sats::{algebraic_type::fmt::fmt_algebraic_type, WithTypespace}; +use std::fmt::{self, Write}; + +lazy_static! { + // https://superuser.com/questions/380772/removing-ansi-color-codes-from-text-stream + static ref ANSI_ESCAPE_SEQUENCE: Regex = Regex::new( + r#"(?x) # verbose mode + (?: + \x1b \[ [\x30-\x3f]* [\x20-\x2f]* [\x40-\x7e] # CSI sequences (start with "ESC [") + | \x1b [PX^_] .*? \x1b \\ # String Terminator sequences (end with "ESC \") + | \x1b \] [^\x07]* (?: \x07 | \x1b \\ ) # Sequences ending in BEL ("\x07") + | \x1b [\x40-\x5f] + )"# + ) + .unwrap(); +} + +/// Strip ANSI escape sequences from a string. +/// This is needed when printing in a terminal without support for these sequences, +/// such as `CMD.exe`. +pub fn strip_ansi_escape_codes(s: &str) -> String { + ANSI_ESCAPE_SEQUENCE.replace_all(s, "").into_owned() +} + +/// Pretty print a migration plan, resulting in a string (containing ANSI escape codes). +/// If you are printing +pub fn pretty_print(plan: &AutoMigratePlan) -> Result<String, fmt::Error> { + let mut out = String::new(); + let outr = &mut out; + + writeln!(outr, "{}", "Migration plan".blue())?; + writeln!(outr, "{}", "--------------")?; + writeln!(outr, "")?; + + let added = "+".green().bold(); + let removed = "-".red().bold(); + + for step in &plan.steps { + match step { + AutoMigrateStep::AddTable(t) => { + let table = plan.new.table(*t).ok_or(fmt::Error)?; + + write!(outr, "{} table: {}", added, table_name(&*table.name))?; + if table.table_type == TableType::System { + write!(outr, " (system)")?; + } + match table.table_access { + TableAccess::Private => write!(outr, "(private)")?, + TableAccess::Public => write!(outr, "(public)")?, + } + writeln!(outr)?; + for column in &table.columns { + let resolved = WithTypespace::new(plan.new.typespace(), &column.ty) + .resolve_refs() + .map_err(|_| fmt::Error)?; + + writeln!( + outr, + " {} column: {}: {}", + added, + column.name, + fmt_algebraic_type(&resolved) + )?; + } + for constraint in table.constraints.values() { + match &constraint.data { + ConstraintData::Unique(unique) => { + write!(outr, " {} unique constraint on ", added)?; + write_col_list(outr, &unique.columns.clone().into(), table)?; + writeln!(outr)?; + } + } + } + for index in table.indexes.values() { + match &index.algorithm { + IndexAlgorithm::BTree(btree) => { + write!(outr, " {} btree index on ", added)?; + write_col_list(outr, &btree.columns, table)?; + writeln!(outr)?; + } + } + } + for sequence in table.sequences.values() { + let column = column_name(table, sequence.column); + writeln!(outr, " {} sequence on {}", added, column)?; + } + if let Some(schedule) = &table.schedule { + let reducer = reducer_name(&*schedule.reducer_name); + writeln!(outr, " {} schedule calling {}", added, reducer)?; + } + } + AutoMigrateStep::AddIndex(index) => { + let table_def = plan.new.stored_in_table_def(*index).ok_or(fmt::Error)?; + let index_def = table_def.indexes.get(*index).ok_or(fmt::Error)?; + + write!( + outr, + "{} index on table {} columns ", + added, + table_name(&*table_def.name) + )?; + write_col_list(outr, index_def.algorithm.columns(), table_def)?; + writeln!(outr)?; + } + AutoMigrateStep::RemoveIndex(index) => { + let table_def = plan.old.stored_in_table_def(*index).ok_or(fmt::Error)?; + let index_def = table_def.indexes.get(*index).ok_or(fmt::Error)?; + + write!( + outr, + "{} index on table {} columns ", + removed, + table_name(&*table_def.name) + )?; + write_col_list(outr, index_def.algorithm.columns(), table_def)?; + writeln!(outr)?; + } + AutoMigrateStep::RemoveConstraint(constraint) => { + let table_def = plan.old.stored_in_table_def(constraint).ok_or(fmt::Error)?; + let constraint_def = table_def.constraints.get(*constraint).ok_or(fmt::Error)?; + match &constraint_def.data { + ConstraintData::Unique(unique_constraint_data) => { + write!( + outr, + "{} unique constraint on table {} columns ", + removed, + table_name(&*table_def.name) + )?; + write_col_list(outr, &unique_constraint_data.columns.clone().into(), table_def)?; + writeln!(outr)?; + } + } + } + AutoMigrateStep::AddSequence(sequence) => { + let table_def = plan.new.stored_in_table_def(*sequence).ok_or(fmt::Error)?; + let sequence_def = table_def.sequences.get(*sequence).ok_or(fmt::Error)?; + + write!( + outr, + "{} sequence on table {} column {}", + added, + table_name(&*table_def.name), + column_name(table_def, sequence_def.column) + )?; + writeln!(outr)?; + } + AutoMigrateStep::RemoveSequence(sequence) => { + let table_def = plan.old.stored_in_table_def(*sequence).ok_or(fmt::Error)?; + let sequence_def = table_def.sequences.get(*sequence).ok_or(fmt::Error)?; + + write!( + outr, + "{} sequence on table {} column {}", + removed, + table_name(&*table_def.name), + column_name(table_def, sequence_def.column) + )?; + writeln!(outr)?; + } + AutoMigrateStep::ChangeAccess(table) => { + let table_def = plan.new.table(*table).ok_or(fmt::Error)?; + + write!( + outr, + "{} table access for table {}", + added, + table_name(&*table_def.name) + )?; + match table_def.table_access { + TableAccess::Private => write!(outr, " (public -> private)")?, + TableAccess::Public => write!(outr, " (private -> public)")?, + } + writeln!(outr)?; + } + AutoMigrateStep::AddSchedule(schedule) => { + let table_def = plan.new.table(*schedule).ok_or(fmt::Error)?; + let schedule_def = table_def.schedule.as_ref().ok_or(fmt::Error)?; + + let reducer = reducer_name(&*schedule_def.reducer_name); + write!( + outr, + "{} schedule for table {} calling {}", + added, + table_name(&*table_def.name), + reducer + )?; + writeln!(outr)?; + } + AutoMigrateStep::RemoveSchedule(schedule) => { + let table_def = plan.old.table(*schedule).ok_or(fmt::Error)?; + let schedule_def = table_def.schedule.as_ref().ok_or(fmt::Error)?; + + let reducer = reducer_name(&*schedule_def.reducer_name); + write!( + outr, + "{} schedule for table {} calling {}", + removed, + table_name(&*table_def.name), + reducer + )?; + writeln!(outr)?; + } + AutoMigrateStep::AddRowLevelSecurity(rls) => { + writeln!(outr, "{} row level security policy:", added)?; + writeln!(outr, "=============================")?; + writeln!(outr, "{}", rls)?; + writeln!(outr, "=============================")?; + } + AutoMigrateStep::RemoveRowLevelSecurity(rls) => { + writeln!(outr, "{} row level security policy:", removed)?; + writeln!(outr, "=============================")?; + writeln!(outr, "{}", rls)?; + writeln!(outr, "=============================")?; + } + } + } + + Ok(out) +} + +fn column_name(table_def: &TableDef, col_id: ColId) -> ColoredString { + table_def + .columns + .get(col_id.idx()) + .map(|def| &*def.name) + .unwrap_or("unknown_column") + .magenta() +} + +fn reducer_name(name: &str) -> ColoredString { + name.blue() +} + +fn table_name(name: &str) -> ColoredString { + name.green() +} + +fn write_col_list(out: &mut String, col_list: &ColList, table_def: &TableDef) -> Result<(), fmt::Error> { + write!(out, "[")?; + for (i, col) in col_list.iter().enumerate() { + let join = if i == 0 { "" } else { ", " }; + write!(out, "{}{}", join, column_name(table_def, col))?; + } + write!(out, "]")?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn test_strip_ansi_escape_sequences() { + let input = "\x1b[31mHello, \x1b[32mworld!\x1b[0m"; + let expected = "Hello, world!"; + assert_eq!(super::strip_ansi_escape_codes(input), expected); + } +} From d66684bafbb922a88c4a71d7970bd81930b39165 Mon Sep 17 00:00:00 2001 From: James Gilles <jameshgilles@gmail.com> Date: Mon, 16 Dec 2024 15:58:18 -0500 Subject: [PATCH 2/2] Add plumbing to print auto migration plans --- Cargo.lock | 2 ++ crates/cli/src/subcommands/publish.rs | 4 ++++ crates/client-api-messages/src/name.rs | 5 +++++ crates/client-api/src/routes/database.rs | 10 +++++++--- crates/core/src/host/host_controller.rs | 2 +- crates/core/src/host/module_host.rs | 6 ++++-- crates/core/src/host/wasm_common/module_host_actor.rs | 7 ++++++- crates/schema/src/auto_migrate/pretty_print.rs | 7 +++++-- 8 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78d877eb29..d033ee3729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5151,6 +5151,7 @@ name = "spacetimedb-schema" version = "1.0.0-rc2" dependencies = [ "anyhow", + "colored", "enum-as-inner", "hashbrown 0.15.1", "indexmap 2.6.0", @@ -5159,6 +5160,7 @@ dependencies = [ "petgraph", "pretty_assertions", "proptest", + "regex", "serde_json", "smallvec", "spacetimedb-cli", diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 397c8ab762..8f326b78ba 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -170,6 +170,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error domain, database_identity, op, + update_summary, } => { let op = match op { PublishOp::Created => "Created new", @@ -180,6 +181,9 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error } else { println!("{} database with identity: {}", op, database_identity); } + if let Some(update_summary) = update_summary { + println!("{}", update_summary); + } } PublishResult::TldNotRegistered { domain } => { return Err(anyhow::anyhow!( diff --git a/crates/client-api-messages/src/name.rs b/crates/client-api-messages/src/name.rs index 0678992303..302366e052 100644 --- a/crates/client-api-messages/src/name.rs +++ b/crates/client-api-messages/src/name.rs @@ -57,6 +57,11 @@ pub enum PublishResult { /// or not. database_identity: Identity, op: PublishOp, + + /// If the database was updated, may contain a string describing the update. + /// Contains ANSI escape codes for color. + /// Suitable for printing to the console. + update_summary: Option<String>, }, // TODO: below variants are obsolete with control db module diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 55412d7dad..46f364c5a3 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -658,7 +658,7 @@ pub async fn publish<S: NodeDelegate + ControlStateDelegate>( .await .map_err(log_and_500)?; - if let Some(updated) = maybe_updated { + let update_summary = if let Some(updated) = maybe_updated { match updated { UpdateDatabaseResult::AutoMigrateError(errs) => { return Err((StatusCode::BAD_REQUEST, format!("Database update rejected: {errs}")).into()); @@ -670,14 +670,18 @@ pub async fn publish<S: NodeDelegate + ControlStateDelegate>( ) .into()); } - UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed => {} + UpdateDatabaseResult::NoUpdateNeeded => None, + UpdateDatabaseResult::UpdatePerformed(summary) => Some(summary), } - } + } else { + None + }; Ok(axum::Json(PublishResult::Success { domain: db_name.as_ref().map(ToString::to_string), database_identity, op, + update_summary, })) } diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 7e6308779b..270250bbb3 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -426,7 +426,7 @@ impl HostController { ) .await?; match update_result { - UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed => { + UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed(_) => { *guard = Some(host); } UpdateDatabaseResult::AutoMigrateError(e) => { diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 1a7258d159..feb2d1b9b1 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -448,7 +448,9 @@ pub struct WeakModuleHost { #[derive(Debug)] pub enum UpdateDatabaseResult { NoUpdateNeeded, - UpdatePerformed, + /// The string is a printable summary of the update that happened. + /// Contains ANSI escape sequences for color. + UpdatePerformed(String), AutoMigrateError(ErrorStream<AutoMigrateError>), ErrorExecutingMigration(anyhow::Error), } @@ -457,7 +459,7 @@ impl UpdateDatabaseResult { pub fn was_successful(&self) -> bool { matches!( self, - UpdateDatabaseResult::UpdatePerformed | UpdateDatabaseResult::NoUpdateNeeded + UpdateDatabaseResult::UpdatePerformed(_) | UpdateDatabaseResult::NoUpdateNeeded ) } } diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 5ee0d84cb5..7582f316b8 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -3,6 +3,7 @@ use bytes::Bytes; use spacetimedb_client_api_messages::timestamp::Timestamp; use spacetimedb_primitives::TableId; use spacetimedb_schema::auto_migrate::ponder_migrate; +use spacetimedb_schema::auto_migrate::pretty_print::pretty_print; use spacetimedb_schema::def::ModuleDef; use spacetimedb_schema::schema::{Schema, TableSchema}; use std::sync::Arc; @@ -340,6 +341,10 @@ impl<T: WasmInstance> ModuleInstance for WasmModuleInstance<T> { return Ok(UpdateDatabaseResult::AutoMigrateError(errs)); } }; + let summary = pretty_print(&plan).unwrap_or_else(|_| { + log::warn!("Failed to pretty-print migration plan: {plan:#?}"); + "(plan not rendered, but succeeded)".to_string() + }); let stdb = &*self.replica_context().relational_db; let program_hash = program.hash; @@ -361,7 +366,7 @@ impl<T: WasmInstance> ModuleInstance for WasmModuleInstance<T> { stdb.commit_tx(tx)?; self.system_logger().info("Database updated"); log::info!("Database updated, {}", stdb.database_identity()); - Ok(UpdateDatabaseResult::UpdatePerformed) + Ok(UpdateDatabaseResult::UpdatePerformed(summary)) } } } diff --git a/crates/schema/src/auto_migrate/pretty_print.rs b/crates/schema/src/auto_migrate/pretty_print.rs index b77c1aaf08..58de24e64a 100644 --- a/crates/schema/src/auto_migrate/pretty_print.rs +++ b/crates/schema/src/auto_migrate/pretty_print.rs @@ -1,6 +1,6 @@ //! This module provides a function [`pretty_print`](pretty_print) that renders an automatic migration plan to a string. -use super::{AutoMigratePlan, IndexAlgorithm, TableDef}; +use super::{IndexAlgorithm, MigratePlan, TableDef}; use crate::{auto_migrate::AutoMigrateStep, def::ConstraintData}; use colored::{self, ColoredString, Colorize}; use lazy_static::lazy_static; @@ -33,7 +33,10 @@ pub fn strip_ansi_escape_codes(s: &str) -> String { /// Pretty print a migration plan, resulting in a string (containing ANSI escape codes). /// If you are printing -pub fn pretty_print(plan: &AutoMigratePlan) -> Result<String, fmt::Error> { +pub fn pretty_print(plan: &MigratePlan) -> Result<String, fmt::Error> { + let plan = match plan { + MigratePlan::Auto(plan) => plan, + }; let mut out = String::new(); let outr = &mut out;