Avoid duplicate .table files

butane_cli/src/lib.rs

@@ -378,7 +378,13 @@ pub fn regenerate_migrations(base_dir: &Path) -> Result<()> {
             from_migration = migrations.get_migration(&from_migration_name);
         }
 
-        migrations.create_migration_to(&backends, &m.name(), from_migration.as_ref(), to_db)?;
+        m.delete_db()?;
+        migrations.create_migration_to(
+            &backends,
+            &m.name(),
+            from_migration.as_ref(),
+            to_db.clone(),
+        )?;
 
         from_migration_name = Some(m.name().to_string());
     }
@@ -393,24 +399,23 @@ pub fn regenerate_migrations(base_dir: &Path) -> Result<()> {
 pub fn load_latest_migration_backends(base_dir: &Path) -> Result<NonEmpty<Box<dyn db::Backend>>> {
     if let Ok(ms) = get_migrations(base_dir) {
         if let Some(latest_migration) = ms.latest() {
-            if let Ok(backend_names) = latest_migration.sql_backends() {
-                assert!(!backend_names.is_empty());
-                log::info!(
-                    "Latest migration contains backends: {}",
-                    backend_names.join(", ")
+            let backend_names = latest_migration.sql_backends()?;
+            assert!(!backend_names.is_empty());
+            log::info!(
+                "Latest migration contains backends: {}",
+                backend_names.join(", ")
+            );
+
+            let mut backends: Vec<Box<dyn db::Backend>> = vec![];
+
+            for backend_name in backend_names {
+                backends.push(
+                    db::get_backend(&backend_name)
+                        .ok_or(anyhow::anyhow!("Backend {backend_name} not found"))?,
                 );
-
-                let mut backends: Vec<Box<dyn db::Backend>> = vec![];
-
-                for backend_name in backend_names {
-                    backends.push(
-                        db::get_backend(&backend_name)
-                            .ok_or(anyhow::anyhow!("Backend {backend_name} not found"))?,
-                    );
-                }
-
-                return Ok(NonEmpty::<Box<dyn db::Backend>>::from_vec(backends).unwrap());
             }
+
+            return Ok(NonEmpty::<Box<dyn db::Backend>>::from_vec(backends).unwrap());
         }
     }
     Err(anyhow::anyhow!("There are no exiting migrations."))

butane_core/src/codegen/migration.rs

@@ -18,7 +18,7 @@ where
 {
     let current_migration = ms.current();
     for table in create_atables(ast_struct, config) {
-        current_migration.write_table(&table)?;
+        current_migration.add_modified_table(&table)?;
     }
     if let Some(name) = &config.table_name {
         // Custom table name, need to also be able to map with the type name

butane_core/src/migrations/fsmigrations.rs

@@ -16,22 +16,32 @@ use crate::{Error, Result};
 type SqlTypeMap = BTreeMap<TypeKey, DeferredSqlType>;
 const TYPES_FILENAME: &str = "types.json";
 
-#[derive(Debug, Deserialize, Serialize)]
+/// Metadata stored in each migration in the filesystem.
+#[derive(Debug, Default, Deserialize, Serialize)]
 struct MigrationInfo {
     /// The migration this one is based on, or None if this is the
     /// first migration in the chain
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     from_name: Option<String>,
+    /// A mapping of table name to the prior migration where it was
+    /// last modified, and therefore where the last .table file for
+    /// it exists.
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    table_bases: BTreeMap<String, String>,
+    /// List of backends supported by this migration.
     backends: Vec<String>,
 }
 impl MigrationInfo {
     fn new() -> Self {
         MigrationInfo {
             from_name: None,
+            table_bases: BTreeMap::new(),
             backends: Vec::new(),
         }
     }
 }
 
+/// Metadata about the migration series.
 #[derive(Debug, Deserialize, Serialize)]
 struct MigrationsState {
     latest: Option<String>,
@@ -42,7 +52,7 @@ impl MigrationsState {
     }
 }
 
-/// A migration stored in the filesystem
+/// A migration stored in the filesystem.
 #[derive(Clone, Debug)]
 pub struct FsMigration {
     fs: Rc<dyn Filesystem>,
@@ -135,16 +145,70 @@ impl FsMigration {
     fn lock_shared(&self) -> Result<MigrationLock> {
         MigrationLock::new_shared(&self.root.join("lock"))
     }
+
+    /// Delete all of the files except info.json which is recreated
+    /// with only `from_name` set to allow migration series traversal.
+    pub fn delete_db(&self) -> Result<()> {
+        let entries = self.fs.list_dir(&self.root)?;
+        for entry in entries {
+            match entry.file_name() {
+                None => continue,
+                Some(name) => {
+                    let name = name.to_string_lossy();
+                    if name == "info.json" {
+                        // Re-create info.json using the minimum required to allow
+                        // `all_migrations` to traverse the list.
+                        let info = self.info()?;
+                        let info = MigrationInfo {
+                            from_name: info.from_name,
+                            ..Default::default()
+                        };
+                        self.write_info(&info)?;
+                    } else {
+                        self.fs.delete(&entry)?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
 }
 
 impl MigrationMut for FsMigration {
-    fn write_table(&mut self, table: &ATable) -> Result<()> {
+    fn add_modified_table(&mut self, table: &ATable) -> Result<()> {
         self.write_contents(
             &format!("{}.table", table.name),
             serde_json::to_string_pretty(table)?.as_bytes(),
         )
     }
 
+    fn add_unmodified_table(&mut self, table: &ATable, from_migration_name: &str) -> Result<()> {
+        // Fetches a FsMigration for the from_migration_name, and retrieves its MigrationInfo.
+        let migrations_dir = self
+            .root
+            .parent()
+            .ok_or(Error::MigrationError("Migrations fs not found".into()))?;
+        let migrations = crate::migrations::from_root(migrations_dir);
+        let migration_list = migrations.all_migrations()?;
+        let from_info = migration_list
+            .iter()
+            .find(|&m| m.name() == *from_migration_name)
+            .ok_or(Error::MigrationError("Migration not found".into()))?
+            .info()?;
+        let from_table_bases = from_info.table_bases;
+        // In the "from" migration, either the 'source' migration is in the `table_bases`,
+        // or the previous migration is the 'source'.
+        let migration_name = if let Some(migration_name) = from_table_bases.get(&table.name) {
+            migration_name
+        } else {
+            from_migration_name
+        };
+        let mut info = self.info()?;
+        info.table_bases
+            .insert(table.name.clone(), migration_name.to_owned());
+        self.write_info(&info)
+    }
+
     fn delete_table(&mut self, table: &str) -> Result<()> {
         let fname = format!("{table}.table");
         self.ensure_dir()?;
@@ -211,6 +275,18 @@ impl Migration for FsMigration {
         self.ensure_dir()?;
         let _lock = self.lock_shared()?;
         let mut db = ADB::new();
+        let table_bases = self.info()?.table_bases;
+        for (table_name, migration_name) in table_bases {
+            let mut filename = PathBuf::from(
+                self.root
+                    .parent()
+                    .ok_or(Error::MigrationError("Migrations fs not found".into()))?,
+            );
+            filename.push(migration_name);
+            filename.push(format!("{table_name}.table"));
+            let table: ATable = serde_json::from_reader(self.fs.read(&filename)?)?;
+            db.replace_table(table)
+        }
         let entries = self.fs.list_dir(&self.root)?;
         for entry in entries {
             match entry.file_name() {
@@ -331,7 +407,9 @@ impl FsMigrations {
         let mut detached_directory_names: Vec<String> = vec![];
         for entry in std::fs::read_dir(self.root.clone())? {
             let path = entry?.path();
-            let name = path.file_name().unwrap();
+            let name = path
+                .file_name()
+                .ok_or(Error::MigrationError("Migration name is missing".into()))?;
             if !path.is_dir() || name == "current" {
                 continue;
             }

butane_core/src/migrations/memmigrations.rs

@@ -66,11 +66,15 @@ impl PartialEq for MemMigration {
 impl Eq for MemMigration {}
 
 impl MigrationMut for MemMigration {
-    fn write_table(&mut self, table: &ATable) -> Result<()> {
+    fn add_modified_table(&mut self, table: &ATable) -> Result<()> {
         self.db.replace_table(table.clone());
         self.db.resolve_types()?;
         Ok(())
     }
+    #[allow(unused_variables)]
+    fn add_unmodified_table(&mut self, table: &ATable, from_migration_name: &str) -> Result<()> {
+        self.add_modified_table(table)
+    }
     fn delete_table(&mut self, table: &str) -> Result<()> {
         self.db.remove_table(table);
         Ok(())

butane_core/src/migrations/migration.rs

@@ -86,8 +86,13 @@ pub trait MigrationMut: Migration {
     /// Adds an abstract table to the migration. The table state should
     /// represent the expected state after the migration has been
     /// applied. It is expected that all tables will be added to the
-    /// migration in this fashion.
-    fn write_table(&mut self, table: &ATable) -> Result<()>;
+    /// migration in this fashion, if they were modified in this migration.
+    fn add_modified_table(&mut self, table: &ATable) -> Result<()>;
+
+    /// Marks a table as not modified in this migration.
+    /// Use instead of `add_modified_table`.
+    #[allow(unused_variables)]
+    fn add_unmodified_table(&mut self, table: &ATable, from_migration_name: &str) -> Result<()>;
 
     /// Delete the table with the given name. Note that simply
     /// deleting a table in code does not work -- it will remain with

butane_core/src/migrations/mod.rs

@@ -185,6 +185,22 @@ where
             return Ok(false);
         }
 
+        let mut modified_tables: Vec<String> = Vec::new();
+
+        for op in &ops {
+            match op {
+                Operation::AddTable(table)
+                | Operation::AddTableConstraints(table)
+                | Operation::AddTableIfNotExists(table) => modified_tables.push(table.name.clone()),
+                Operation::AddColumn(table_name, _) => modified_tables.push(table_name.clone()),
+                Operation::RemoveColumn(table_name, _) => modified_tables.push(table_name.clone()),
+                Operation::ChangeColumn(table_name, _, _) => {
+                    modified_tables.push(table_name.clone())
+                }
+                Operation::RemoveTable(_) => {}
+            }
+        }
+
         if from_none {
             // This may be the first migration. Create the butane_migration table
             ops.push(Operation::AddTableIfNotExists(migrations_table()));
@@ -193,7 +209,13 @@ where
         let mut m = self.new_migration(name);
         // Save the DB for use by other migrations from this one
         for table in to_db.tables() {
-            m.write_table(table)?;
+            if modified_tables.contains(&table.name) {
+                m.add_modified_table(table)?;
+            } else {
+                let from =
+                    from.ok_or(Error::MigrationError("unmodified requires a from".into()))?;
+                m.add_unmodified_table(table, &from.name())?;
+            }
         }
 
         for backend in backends {
@@ -238,7 +260,7 @@ pub fn copy_migration(from: &impl Migration, to: &mut impl MigrationMut) -> Resu
     to.set_migration_from(from.migration_from()?.map(|s| s.to_string()))?;
     let db = from.db()?;
     for table in db.tables() {
-        to.write_table(table)?;
+        to.add_modified_table(table)?;
     }
     for (k, v) in db.types() {
         to.add_type(k.clone(), v.clone())?;

examples/getting_started/.butane/migrations/20201229_144636751_init/info.json

@@ -1,5 +1,4 @@
 {
-  "from_name": null,
   "backends": [
     "sqlite",
     "pg"

examples/getting_started/.butane/migrations/20201229_171630604_likes/info.json

@@ -1,5 +1,10 @@
 {
   "from_name": "20201229_144636751_init",
+  "table_bases": {
+    "Blog": "20201229_144636751_init",
+    "Post_tags_Many": "20201229_144636751_init",
+    "Tag": "20201229_144636751_init"
+  },
   "backends": [
     "sqlite",
     "pg"