diff --git a/butane_cli/src/lib.rs b/butane_cli/src/lib.rs index 66e785fa..10e1db0b 100644 --- a/butane_cli/src/lib.rs +++ b/butane_cli/src/lib.rs @@ -378,7 +378,13 @@ pub fn regenerate_migrations(base_dir: &Path) -> Result<()> { from_migration = migrations.get_migration(&from_migration_name); } - migrations.create_migration_to(&backends, &m.name(), from_migration.as_ref(), to_db)?; + m.delete_db()?; + migrations.create_migration_to( + &backends, + &m.name(), + from_migration.as_ref(), + to_db.clone(), + )?; from_migration_name = Some(m.name().to_string()); } @@ -393,24 +399,23 @@ pub fn regenerate_migrations(base_dir: &Path) -> Result<()> { pub fn load_latest_migration_backends(base_dir: &Path) -> Result>> { if let Ok(ms) = get_migrations(base_dir) { if let Some(latest_migration) = ms.latest() { - if let Ok(backend_names) = latest_migration.sql_backends() { - assert!(!backend_names.is_empty()); - log::info!( - "Latest migration contains backends: {}", - backend_names.join(", ") + let backend_names = latest_migration.sql_backends()?; + assert!(!backend_names.is_empty()); + log::info!( + "Latest migration contains backends: {}", + backend_names.join(", ") + ); + + let mut backends: Vec> = vec![]; + + for backend_name in backend_names { + backends.push( + db::get_backend(&backend_name) + .ok_or(anyhow::anyhow!("Backend {backend_name} not found"))?, ); - - let mut backends: Vec> = vec![]; - - for backend_name in backend_names { - backends.push( - db::get_backend(&backend_name) - .ok_or(anyhow::anyhow!("Backend {backend_name} not found"))?, - ); - } - - return Ok(NonEmpty::>::from_vec(backends).unwrap()); } + + return Ok(NonEmpty::>::from_vec(backends).unwrap()); } } Err(anyhow::anyhow!("There are no exiting migrations.")) diff --git a/butane_core/src/codegen/migration.rs b/butane_core/src/codegen/migration.rs index a759434a..22317e4a 100644 --- a/butane_core/src/codegen/migration.rs +++ b/butane_core/src/codegen/migration.rs @@ -18,7 +18,7 @@ where { let current_migration = ms.current(); for table in create_atables(ast_struct, config) { - current_migration.write_table(&table)?; + current_migration.add_modified_table(&table)?; } if let Some(name) = &config.table_name { // Custom table name, need to also be able to map with the type name diff --git a/butane_core/src/migrations/fsmigrations.rs b/butane_core/src/migrations/fsmigrations.rs index 97da928c..7bb3006b 100644 --- a/butane_core/src/migrations/fsmigrations.rs +++ b/butane_core/src/migrations/fsmigrations.rs @@ -16,22 +16,32 @@ use crate::{Error, Result}; type SqlTypeMap = BTreeMap; const TYPES_FILENAME: &str = "types.json"; -#[derive(Debug, Deserialize, Serialize)] +/// Metadata stored in each migration in the filesystem. +#[derive(Debug, Default, Deserialize, Serialize)] struct MigrationInfo { /// The migration this one is based on, or None if this is the /// first migration in the chain + #[serde(default, skip_serializing_if = "Option::is_none")] from_name: Option, + /// A mapping of table name to the prior migration where it was + /// last modified, and therefore where the last .table file for + /// it exists. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + table_bases: BTreeMap, + /// List of backends supported by this migration. backends: Vec, } impl MigrationInfo { fn new() -> Self { MigrationInfo { from_name: None, + table_bases: BTreeMap::new(), backends: Vec::new(), } } } +/// Metadata about the migration series. #[derive(Debug, Deserialize, Serialize)] struct MigrationsState { latest: Option, @@ -42,7 +52,7 @@ impl MigrationsState { } } -/// A migration stored in the filesystem +/// A migration stored in the filesystem. #[derive(Clone, Debug)] pub struct FsMigration { fs: Rc, @@ -135,16 +145,70 @@ impl FsMigration { fn lock_shared(&self) -> Result { MigrationLock::new_shared(&self.root.join("lock")) } + + /// Delete all of the files except info.json which is recreated + /// with only `from_name` set to allow migration series traversal. + pub fn delete_db(&self) -> Result<()> { + let entries = self.fs.list_dir(&self.root)?; + for entry in entries { + match entry.file_name() { + None => continue, + Some(name) => { + let name = name.to_string_lossy(); + if name == "info.json" { + // Re-create info.json using the minimum required to allow + // `all_migrations` to traverse the list. + let info = self.info()?; + let info = MigrationInfo { + from_name: info.from_name, + ..Default::default() + }; + self.write_info(&info)?; + } else { + self.fs.delete(&entry)?; + } + } + } + } + Ok(()) + } } impl MigrationMut for FsMigration { - fn write_table(&mut self, table: &ATable) -> Result<()> { + fn add_modified_table(&mut self, table: &ATable) -> Result<()> { self.write_contents( &format!("{}.table", table.name), serde_json::to_string_pretty(table)?.as_bytes(), ) } + fn add_unmodified_table(&mut self, table: &ATable, from_migration_name: &str) -> Result<()> { + // Fetches a FsMigration for the from_migration_name, and retrieves its MigrationInfo. + let migrations_dir = self + .root + .parent() + .ok_or(Error::MigrationError("Migrations fs not found".into()))?; + let migrations = crate::migrations::from_root(migrations_dir); + let migration_list = migrations.all_migrations()?; + let from_info = migration_list + .iter() + .find(|&m| m.name() == *from_migration_name) + .ok_or(Error::MigrationError("Migration not found".into()))? + .info()?; + let from_table_bases = from_info.table_bases; + // In the "from" migration, either the 'source' migration is in the `table_bases`, + // or the previous migration is the 'source'. + let migration_name = if let Some(migration_name) = from_table_bases.get(&table.name) { + migration_name + } else { + from_migration_name + }; + let mut info = self.info()?; + info.table_bases + .insert(table.name.clone(), migration_name.to_owned()); + self.write_info(&info) + } + fn delete_table(&mut self, table: &str) -> Result<()> { let fname = format!("{table}.table"); self.ensure_dir()?; @@ -211,6 +275,18 @@ impl Migration for FsMigration { self.ensure_dir()?; let _lock = self.lock_shared()?; let mut db = ADB::new(); + let table_bases = self.info()?.table_bases; + for (table_name, migration_name) in table_bases { + let mut filename = PathBuf::from( + self.root + .parent() + .ok_or(Error::MigrationError("Migrations fs not found".into()))?, + ); + filename.push(migration_name); + filename.push(format!("{table_name}.table")); + let table: ATable = serde_json::from_reader(self.fs.read(&filename)?)?; + db.replace_table(table) + } let entries = self.fs.list_dir(&self.root)?; for entry in entries { match entry.file_name() { @@ -331,7 +407,9 @@ impl FsMigrations { let mut detached_directory_names: Vec = vec![]; for entry in std::fs::read_dir(self.root.clone())? { let path = entry?.path(); - let name = path.file_name().unwrap(); + let name = path + .file_name() + .ok_or(Error::MigrationError("Migration name is missing".into()))?; if !path.is_dir() || name == "current" { continue; } diff --git a/butane_core/src/migrations/memmigrations.rs b/butane_core/src/migrations/memmigrations.rs index b6ee9cd9..4583f68c 100644 --- a/butane_core/src/migrations/memmigrations.rs +++ b/butane_core/src/migrations/memmigrations.rs @@ -66,11 +66,15 @@ impl PartialEq for MemMigration { impl Eq for MemMigration {} impl MigrationMut for MemMigration { - fn write_table(&mut self, table: &ATable) -> Result<()> { + fn add_modified_table(&mut self, table: &ATable) -> Result<()> { self.db.replace_table(table.clone()); self.db.resolve_types()?; Ok(()) } + #[allow(unused_variables)] + fn add_unmodified_table(&mut self, table: &ATable, from_migration_name: &str) -> Result<()> { + self.add_modified_table(table) + } fn delete_table(&mut self, table: &str) -> Result<()> { self.db.remove_table(table); Ok(()) diff --git a/butane_core/src/migrations/migration.rs b/butane_core/src/migrations/migration.rs index 86daf8b7..db2dc903 100644 --- a/butane_core/src/migrations/migration.rs +++ b/butane_core/src/migrations/migration.rs @@ -86,8 +86,13 @@ pub trait MigrationMut: Migration { /// Adds an abstract table to the migration. The table state should /// represent the expected state after the migration has been /// applied. It is expected that all tables will be added to the - /// migration in this fashion. - fn write_table(&mut self, table: &ATable) -> Result<()>; + /// migration in this fashion, if they were modified in this migration. + fn add_modified_table(&mut self, table: &ATable) -> Result<()>; + + /// Marks a table as not modified in this migration. + /// Use instead of `add_modified_table`. + #[allow(unused_variables)] + fn add_unmodified_table(&mut self, table: &ATable, from_migration_name: &str) -> Result<()>; /// Delete the table with the given name. Note that simply /// deleting a table in code does not work -- it will remain with diff --git a/butane_core/src/migrations/mod.rs b/butane_core/src/migrations/mod.rs index dfe90f05..0d0ad257 100644 --- a/butane_core/src/migrations/mod.rs +++ b/butane_core/src/migrations/mod.rs @@ -185,6 +185,22 @@ where return Ok(false); } + let mut modified_tables: Vec = Vec::new(); + + for op in &ops { + match op { + Operation::AddTable(table) + | Operation::AddTableConstraints(table) + | Operation::AddTableIfNotExists(table) => modified_tables.push(table.name.clone()), + Operation::AddColumn(table_name, _) => modified_tables.push(table_name.clone()), + Operation::RemoveColumn(table_name, _) => modified_tables.push(table_name.clone()), + Operation::ChangeColumn(table_name, _, _) => { + modified_tables.push(table_name.clone()) + } + Operation::RemoveTable(_) => {} + } + } + if from_none { // This may be the first migration. Create the butane_migration table ops.push(Operation::AddTableIfNotExists(migrations_table())); @@ -193,7 +209,13 @@ where let mut m = self.new_migration(name); // Save the DB for use by other migrations from this one for table in to_db.tables() { - m.write_table(table)?; + if modified_tables.contains(&table.name) { + m.add_modified_table(table)?; + } else { + let from = + from.ok_or(Error::MigrationError("unmodified requires a from".into()))?; + m.add_unmodified_table(table, &from.name())?; + } } for backend in backends { @@ -238,7 +260,7 @@ pub fn copy_migration(from: &impl Migration, to: &mut impl MigrationMut) -> Resu to.set_migration_from(from.migration_from()?.map(|s| s.to_string()))?; let db = from.db()?; for table in db.tables() { - to.write_table(table)?; + to.add_modified_table(table)?; } for (k, v) in db.types() { to.add_type(k.clone(), v.clone())?; diff --git a/examples/getting_started/.butane/migrations/20201229_144636751_init/info.json b/examples/getting_started/.butane/migrations/20201229_144636751_init/info.json index 88f6a1a2..beabd46e 100644 --- a/examples/getting_started/.butane/migrations/20201229_144636751_init/info.json +++ b/examples/getting_started/.butane/migrations/20201229_144636751_init/info.json @@ -1,5 +1,4 @@ { - "from_name": null, "backends": [ "sqlite", "pg" diff --git a/examples/getting_started/.butane/migrations/20201229_171630604_likes/Blog.table b/examples/getting_started/.butane/migrations/20201229_171630604_likes/Blog.table deleted file mode 100644 index 8fc11406..00000000 --- a/examples/getting_started/.butane/migrations/20201229_171630604_likes/Blog.table +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "Blog", - "columns": [ - { - "name": "id", - "sqltype": { - "Known": "BigInt" - }, - "nullable": false, - "pk": true, - "auto": true, - "unique": false, - "default": null - }, - { - "name": "name", - "sqltype": { - "Known": "Text" - }, - "nullable": false, - "pk": false, - "auto": false, - "unique": false, - "default": null - } - ] -} diff --git a/examples/getting_started/.butane/migrations/20201229_171630604_likes/Post_tags_Many.table b/examples/getting_started/.butane/migrations/20201229_171630604_likes/Post_tags_Many.table deleted file mode 100644 index de7e3b2a..00000000 --- a/examples/getting_started/.butane/migrations/20201229_171630604_likes/Post_tags_Many.table +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "Post_tags_Many", - "columns": [ - { - "name": "owner", - "sqltype": { - "Known": "Int" - }, - "nullable": false, - "pk": false, - "auto": false, - "unique": false, - "default": null - }, - { - "name": "has", - "sqltype": { - "Known": "Text" - }, - "nullable": false, - "pk": false, - "auto": false, - "unique": false, - "default": null - } - ] -} diff --git a/examples/getting_started/.butane/migrations/20201229_171630604_likes/Tag.table b/examples/getting_started/.butane/migrations/20201229_171630604_likes/Tag.table deleted file mode 100644 index 1d3b1e5b..00000000 --- a/examples/getting_started/.butane/migrations/20201229_171630604_likes/Tag.table +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "Tag", - "columns": [ - { - "name": "tag", - "sqltype": { - "Known": "Text" - }, - "nullable": false, - "pk": true, - "auto": false, - "unique": false, - "default": null - } - ] -} diff --git a/examples/getting_started/.butane/migrations/20201229_171630604_likes/info.json b/examples/getting_started/.butane/migrations/20201229_171630604_likes/info.json index 6adcfb79..473354cf 100644 --- a/examples/getting_started/.butane/migrations/20201229_171630604_likes/info.json +++ b/examples/getting_started/.butane/migrations/20201229_171630604_likes/info.json @@ -1,5 +1,10 @@ { "from_name": "20201229_144636751_init", + "table_bases": { + "Blog": "20201229_144636751_init", + "Post_tags_Many": "20201229_144636751_init", + "Tag": "20201229_144636751_init" + }, "backends": [ "sqlite", "pg"