From 5d1b5214d1f613c7061f29d7f80219e8b59449ee Mon Sep 17 00:00:00 2001 From: Dossy Shiobara Date: Fri, 21 Apr 2023 12:14:04 -0400 Subject: [PATCH] Support multiple --migrations-dir directories (#428) For reference, see: https://github.com/amacneil/dbmate/discussions/424 --- main.go | 6 ++-- pkg/dbmate/db.go | 64 ++++++++++++++++++++++--------------------- pkg/dbmate/db_test.go | 39 ++++++++++++++++++++++++-- 3 files changed, 72 insertions(+), 37 deletions(-) diff --git a/main.go b/main.go index 6576384f..83b85bf8 100644 --- a/main.go +++ b/main.go @@ -49,11 +49,11 @@ func NewApp() *cli.App { Value: "DATABASE_URL", Usage: "specify an environment variable containing the database URL", }, - &cli.StringFlag{ + &cli.StringSliceFlag{ Name: "migrations-dir", Aliases: []string{"d"}, EnvVars: []string{"DBMATE_MIGRATIONS_DIR"}, - Value: defaultDB.MigrationsDir, + Value: cli.NewStringSlice(defaultDB.MigrationsDir[0]), Usage: "specify the directory containing migration files", }, &cli.StringFlag{ @@ -231,7 +231,7 @@ func action(f func(*dbmate.DB, *cli.Context) error) cli.ActionFunc { } db := dbmate.New(u) db.AutoDumpSchema = !c.Bool("no-dump-schema") - db.MigrationsDir = c.String("migrations-dir") + db.MigrationsDir = c.StringSlice("migrations-dir") db.MigrationsTableName = c.String("migrations-table") db.SchemaFile = c.String("schema-file") db.WaitBefore = c.Bool("wait") diff --git a/pkg/dbmate/db.go b/pkg/dbmate/db.go index fc404d6b..ae598ffd 100644 --- a/pkg/dbmate/db.go +++ b/pkg/dbmate/db.go @@ -43,8 +43,8 @@ type DB struct { FS fs.FS // Log is the interface to write stdout Log io.Writer - // MigrationsDir specifies the directory to find migration files - MigrationsDir string + // MigrationsDir specifies the directory or directories to find migration files + MigrationsDir []string // MigrationsTableName specifies the database table to record migrations in MigrationsTableName string // SchemaFile specifies the location for schema.sql file @@ -72,7 +72,7 @@ func New(databaseURL *url.URL) *DB { DatabaseURL: databaseURL, FS: nil, Log: os.Stdout, - MigrationsDir: "./db/migrations", + MigrationsDir: []string{"./db/migrations"}, MigrationsTableName: "schema_migrations", SchemaFile: "./db/schema.sql", Verbose: false, @@ -239,12 +239,12 @@ func (db *DB) NewMigration(name string) error { name = fmt.Sprintf("%s_%s.sql", timestamp, name) // create migrations dir if missing - if err := ensureDir(db.MigrationsDir); err != nil { + if err := ensureDir(db.MigrationsDir[0]); err != nil { return err } // check file does not already exist - path := filepath.Join(db.MigrationsDir, name) + path := filepath.Join(db.MigrationsDir[0], name) fmt.Fprintf(db.Log, "Creating migration: %s\n", path) if _, err := os.Stat(path); !os.IsNotExist(err) { @@ -372,8 +372,8 @@ func (db *DB) printVerbose(result sql.Result) { } } -func (db *DB) readMigrationsDir() ([]fs.DirEntry, error) { - path := filepath.Clean(db.MigrationsDir) +func (db *DB) readMigrationsDir(dir string) ([]fs.DirEntry, error) { + path := filepath.Clean(dir) // We use nil instead of os.DirFS() because DirFS cannot support both relative and absolute // directory paths - it must be anchored at either "." or "/", which we do not know in advance. @@ -412,35 +412,37 @@ func (db *DB) FindMigrations() ([]Migration, error) { } } - // find filesystem migrations - files, err := db.readMigrationsDir() - if err != nil { - return nil, fmt.Errorf("%w `%s`", ErrMigrationDirNotFound, db.MigrationsDir) - } - migrations := []Migration{} - for _, file := range files { - if file.IsDir() { - continue + for _, dir := range db.MigrationsDir { + // find filesystem migrations + files, err := db.readMigrationsDir(dir) + if err != nil { + return nil, fmt.Errorf("%w `%s`", ErrMigrationDirNotFound, dir) } - matches := migrationFileRegexp.FindStringSubmatch(file.Name()) - if len(matches) < 2 { - continue - } + for _, file := range files { + if file.IsDir() { + continue + } - migration := Migration{ - Applied: false, - FileName: matches[0], - FilePath: filepath.Join(db.MigrationsDir, matches[0]), - FS: db.FS, - Version: matches[1], - } - if ok := appliedMigrations[migration.Version]; ok { - migration.Applied = true - } + matches := migrationFileRegexp.FindStringSubmatch(file.Name()) + if len(matches) < 2 { + continue + } - migrations = append(migrations, migration) + migration := Migration{ + Applied: false, + FileName: matches[0], + FilePath: filepath.Join(dir, matches[0]), + FS: db.FS, + Version: matches[1], + } + if ok := appliedMigrations[migration.Version]; ok { + migration.Applied = true + } + + migrations = append(migrations, migration) + } } sort.Slice(migrations, func(i, j int) bool { diff --git a/pkg/dbmate/db_test.go b/pkg/dbmate/db_test.go index 1953ad72..6ba8d5bb 100644 --- a/pkg/dbmate/db_test.go +++ b/pkg/dbmate/db_test.go @@ -42,7 +42,7 @@ func TestNew(t *testing.T) { db := dbmate.New(dbutil.MustParseURL("foo:test")) require.True(t, db.AutoDumpSchema) require.Equal(t, "foo:test", db.DatabaseURL.String()) - require.Equal(t, "./db/migrations", db.MigrationsDir) + require.Equal(t, []string{"./db/migrations"}, db.MigrationsDir) require.Equal(t, "schema_migrations", db.MigrationsTableName) require.Equal(t, "./db/schema.sql", db.SchemaFile) require.False(t, db.WaitBefore) @@ -462,7 +462,7 @@ func TestFindMigrationsAbsolute(t *testing.T) { t.Run("relative path", func(t *testing.T) { u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) db := newTestDB(t, u) - db.MigrationsDir = "db/migrations" + db.MigrationsDir = []string{"db/migrations"} migrations, err := db.FindMigrations() require.NoError(t, err) @@ -482,7 +482,7 @@ func TestFindMigrationsAbsolute(t *testing.T) { u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) db := newTestDB(t, u) - db.MigrationsDir = dir + db.MigrationsDir = []string{dir} require.Nil(t, db.FS) migrations, err := db.FindMigrations() @@ -550,3 +550,36 @@ drop table users; require.Equal(t, "-- migrate:down\ndrop table users;\n", parsed.Down) require.True(t, parsed.DownOptions.Transaction()) } + +func TestFindMigrationsFSMultipleDirs(t *testing.T) { + mapFS := fstest.MapFS{ + "db/migrations_a/001_test_migration_a.sql": {}, + "db/migrations_a/005_test_migration_a.sql": {}, + "db/migrations_b/003_test_migration_b.sql": {}, + "db/migrations_b/004_test_migration_b.sql": {}, + "db/migrations_c/002_test_migration_c.sql": {}, + "db/migrations_c/006_test_migration_c.sql": {}, + } + + u := dbutil.MustParseURL(os.Getenv("POSTGRES_TEST_URL")) + db := newTestDB(t, u) + db.FS = mapFS + db.MigrationsDir = []string{"./db/migrations_a", "./db/migrations_b", "./db/migrations_c"} + + // drop and recreate database + err := db.Drop() + require.NoError(t, err) + err = db.Create() + require.NoError(t, err) + + actual, err := db.FindMigrations() + require.NoError(t, err) + + // test migrations are correct and in order + require.Equal(t, "db/migrations_a/001_test_migration_a.sql", actual[0].FilePath) + require.Equal(t, "db/migrations_c/002_test_migration_c.sql", actual[1].FilePath) + require.Equal(t, "db/migrations_b/003_test_migration_b.sql", actual[2].FilePath) + require.Equal(t, "db/migrations_b/004_test_migration_b.sql", actual[3].FilePath) + require.Equal(t, "db/migrations_a/005_test_migration_a.sql", actual[4].FilePath) + require.Equal(t, "db/migrations_c/006_test_migration_c.sql", actual[5].FilePath) +}