chore: glob seed paths when loading config (#2726)

internal/db/push/push.go

@@ -41,11 +41,7 @@ func Run(ctx context.Context, dryRun, ignoreVersionMismatch bool, includeRoles,
 		fmt.Fprintln(os.Stderr, "Would push these migrations:")
 		fmt.Fprint(os.Stderr, utils.Bold(confirmPushAll(pending)))
 		if includeSeed {
-			seedPaths, err := utils.GetSeedFiles(fsys)
-			if err != nil {
-				return err
-			}
-			fmt.Fprintf(os.Stderr, "Would seed data %v...\n", seedPaths)
+			fmt.Fprintf(os.Stderr, "Would seed data %v...\n", utils.Config.Db.Seed.SqlPaths)
 		}
 	} else {
 		msg := fmt.Sprintf("Do you want to push these migrations to the remote database?\n%s\n", confirmPushAll(pending))

internal/db/push/push_test.go

@@ -161,8 +161,9 @@ func TestPushAll(t *testing.T) {
 	})
 
 	t.Run("throws error on seed failure", func(t *testing.T) {
-		// Setup in-memory fs
 		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
+		utils.Config.Db.Seed.SqlPaths = []string{seedPath}
+		// Setup in-memory fs
 		fsys := &fstest.OpenErrorFs{DenyPath: seedPath}
 		_, _ = fsys.Create(seedPath)
 		path := filepath.Join(utils.MigrationsDir, "0_test.sql")

internal/db/start/start_test.go

@@ -52,6 +52,8 @@ func TestInitBranch(t *testing.T) {
 
 func TestStartDatabase(t *testing.T) {
 	t.Run("initialize main branch", func(t *testing.T) {
+		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
+		utils.Config.Db.Seed.SqlPaths = []string{seedPath}
 		utils.Config.Db.MajorVersion = 15
 		utils.DbId = "supabase_db_test"
 		utils.ConfigId = "supabase_config_test"
@@ -61,7 +63,7 @@ func TestStartDatabase(t *testing.T) {
 		roles := "create role test"
 		require.NoError(t, afero.WriteFile(fsys, utils.CustomRolesPath, []byte(roles), 0644))
 		seed := "INSERT INTO employees(name) VALUES ('Alice')"
-		require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(seed), 0644))
+		require.NoError(t, afero.WriteFile(fsys, seedPath, []byte(seed), 0644))
 		// Setup mock docker
 		require.NoError(t, apitest.MockDocker(utils.Docker))
 		defer gock.OffAll()

internal/migration/apply/apply.go

@@ -27,11 +27,7 @@ func MigrateAndSeed(ctx context.Context, version string, conn *pgx.Conn, fsys af
 }
 
 func SeedDatabase(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {
-	seedPaths, err := utils.GetSeedFiles(fsys)
-	if err != nil {
-		return err
-	}
-	return migration.SeedData(ctx, seedPaths, conn, afero.NewIOFS(fsys))
+	return migration.SeedData(ctx, utils.Config.Db.Seed.SqlPaths, conn, afero.NewIOFS(fsys))
 }
 
 func CreateCustomRoles(ctx context.Context, conn *pgx.Conn, fsys afero.Fs) error {

internal/migration/apply/apply_test.go

@@ -77,12 +77,15 @@ func TestMigrateDatabase(t *testing.T) {
 }
 
 func TestSeedDatabase(t *testing.T) {
+	seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
+	utils.Config.Db.Seed.SqlPaths = []string{seedPath}
+
 	t.Run("seeds from file", func(t *testing.T) {
 		// Setup in-memory fs
 		fsys := afero.NewMemMapFs()
 		// Setup seed file
 		sql := "INSERT INTO employees(name) VALUES ('Alice')"
-		require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(sql), 0644))
+		require.NoError(t, afero.WriteFile(fsys, seedPath, []byte(sql), 0644))
 		// Setup mock postgres
 		conn := pgtest.NewConn()
 		defer conn.Close(t)
@@ -95,12 +98,19 @@ func TestSeedDatabase(t *testing.T) {
 	})
 
 	t.Run("ignores missing seed", func(t *testing.T) {
-		assert.NoError(t, SeedDatabase(context.Background(), nil, afero.NewMemMapFs()))
+		sqlPaths := utils.Config.Db.Seed.SqlPaths
+		utils.Config.Db.Seed.SqlPaths = []string{}
+		t.Cleanup(func() { utils.Config.Db.Seed.SqlPaths = sqlPaths })
+		// Setup in-memory fs
+		fsys := afero.NewMemMapFs()
+		// Run test
+		err := SeedDatabase(context.Background(), nil, fsys)
+		// Check error
+		assert.NoError(t, err)
 	})
 
 	t.Run("throws error on read failure", func(t *testing.T) {
 		// Setup in-memory fs
-		seedPath := filepath.Join(utils.SupabaseDirPath, "seed.sql")
 		fsys := &fstest.OpenErrorFs{DenyPath: seedPath}
 		_, _ = fsys.Create(seedPath)
 		// Run test
@@ -114,7 +124,7 @@ func TestSeedDatabase(t *testing.T) {
 		fsys := afero.NewMemMapFs()
 		// Setup seed file
 		sql := "INSERT INTO employees(name) VALUES ('Alice')"
-		require.NoError(t, afero.WriteFile(fsys, filepath.Join(utils.SupabaseDirPath, "seed.sql"), []byte(sql), 0644))
+		require.NoError(t, afero.WriteFile(fsys, seedPath, []byte(sql), 0644))
 		// Setup mock postgres
 		conn := pgtest.NewConn()
 		defer conn.Close(t)

internal/utils/misc.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-	"sort"
 	"time"
 
 	"github.com/docker/docker/client"
@@ -157,26 +156,6 @@ var (
 	ErrNotRunning  = errors.Errorf("%s is not running.", Aqua("supabase start"))
 )
 
-// Match the glob patterns from the config to get a deduplicated
-// array of all migrations files to apply in the declared order.
-func GetSeedFiles(fsys afero.Fs) ([]string, error) {
-	seedPaths := Config.Db.Seed.SqlPaths
-	var files []string
-	for _, pattern := range seedPaths {
-		fullPattern := filepath.Join(SupabaseDirPath, pattern)
-		matches, err := afero.Glob(fsys, fullPattern)
-		if err != nil {
-			return nil, errors.Errorf("failed to apply glob pattern for %w", err)
-		}
-		if len(matches) == 0 {
-			fmt.Fprintf(os.Stderr, "%s Your pattern %s matched 0 seed files.\n", Yellow("WARNING:"), pattern)
-		}
-		sort.Strings(matches)
-		files = append(files, matches...)
-	}
-	return RemoveDuplicates(files), nil
-}
-
 func GetCurrentTimestamp() string {
 	// Magic number: https://stackoverflow.com/q/45160822.
 	return time.Now().UTC().Format("20060102150405")

internal/utils/misc_test.go

@@ -75,76 +75,3 @@ func TestProjectRoot(t *testing.T) {
 		assert.Equal(t, cwd, path)
 	})
 }
-
-func TestGetSeedFiles(t *testing.T) {
-	t.Run("returns seed files matching patterns", func(t *testing.T) {
-		// Setup in-memory fs
-		fsys := afero.NewMemMapFs()
-		// Create seed files
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed1.sql", []byte("INSERT INTO table1 VALUES (1);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed2.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed3.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/another.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/ignore.sql", []byte("INSERT INTO table3 VALUES (3);"), 0644))
-		// Mock config patterns
-		Config.Db.Seed.SqlPaths = []string{"seeds/seed[12].sql", "seeds/ano*.sql"}
-
-		// Run test
-		files, err := GetSeedFiles(fsys)
-
-		// Check error
-		assert.NoError(t, err)
-		// Validate files
-		assert.ElementsMatch(t, []string{"supabase/seeds/seed1.sql", "supabase/seeds/seed2.sql", "supabase/seeds/another.sql"}, files)
-	})
-	t.Run("returns seed files matching patterns skip duplicates", func(t *testing.T) {
-		// Setup in-memory fs
-		fsys := afero.NewMemMapFs()
-		// Create seed files
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed1.sql", []byte("INSERT INTO table1 VALUES (1);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed2.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/seed3.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/another.sql", []byte("INSERT INTO table2 VALUES (2);"), 0644))
-		require.NoError(t, afero.WriteFile(fsys, "supabase/seeds/ignore.sql", []byte("INSERT INTO table3 VALUES (3);"), 0644))
-		// Mock config patterns
-		Config.Db.Seed.SqlPaths = []string{"seeds/seed[12].sql", "seeds/ano*.sql", "seeds/seed*.sql"}
-
-		// Run test
-		files, err := GetSeedFiles(fsys)
-
-		// Check error
-		assert.NoError(t, err)
-		// Validate files
-		assert.ElementsMatch(t, []string{"supabase/seeds/seed1.sql", "supabase/seeds/seed2.sql", "supabase/seeds/another.sql", "supabase/seeds/seed3.sql"}, files)
-	})
-
-	t.Run("returns error on invalid pattern", func(t *testing.T) {
-		// Setup in-memory fs
-		fsys := afero.NewMemMapFs()
-		// Mock config patterns
-		Config.Db.Seed.SqlPaths = []string{"[*!#@D#"}
-
-		// Run test
-		files, err := GetSeedFiles(fsys)
-
-		// Check error
-		assert.Nil(t, err)
-		// The resuling seed list should be empty
-		assert.ElementsMatch(t, []string{}, files)
-	})
-
-	t.Run("returns empty list if no files match", func(t *testing.T) {
-		// Setup in-memory fs
-		fsys := afero.NewMemMapFs()
-		// Mock config patterns
-		Config.Db.Seed.SqlPaths = []string{"seeds/*.sql"}
-
-		// Run test
-		files, err := GetSeedFiles(fsys)
-
-		// Check error
-		assert.NoError(t, err)
-		// Validate files
-		assert.Empty(t, files)
-	})
-}

pkg/config/config.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"text/template"
@@ -172,8 +173,9 @@ type (
 	}
 
 	seed struct {
-		Enabled  bool     `toml:"enabled"`
-		SqlPaths []string `toml:"sql_paths"`
+		Enabled      bool     `toml:"enabled"`
+		GlobPatterns []string `toml:"sql_paths"`
+		SqlPaths     []string `toml:"-"`
 	}
 
 	pooler struct {
@@ -483,8 +485,8 @@ func NewConfig(editors ...ConfigEditor) config {
 				SecretKeyBase: "EAx3IQ/wRG1v47ZD4NE4/9RzBI8Jmil3x0yhcW4V2NHBP6c2iPIzwjofi2Ep4HIG",
 			},
 			Seed: seed{
-				Enabled:  true,
-				SqlPaths: []string{"./seed.sql"},
+				Enabled:      true,
+				GlobPatterns: []string{"./seed.sql"},
 			},
 		},
 		Realtime: realtime{
@@ -708,6 +710,9 @@ func (c *config) Load(path string, fsys fs.FS) error {
 		}
 		c.Functions[slug] = function
 	}
+	if err := c.Db.Seed.loadSeedPaths(builder.SupabaseDirPath, fsys); err != nil {
+		return err
+	}
 	if err := c.baseConfig.Validate(); err != nil {
 		return err
 	}
@@ -1041,6 +1046,40 @@ func loadEnvIfExists(path string) error {
 	return nil
 }
 
+// Match the glob patterns from the config to get a deduplicated
+// array of all migrations files to apply in the declared order.
+func (c *seed) loadSeedPaths(basePath string, fsys fs.FS) error {
+	if !c.Enabled {
+		return nil
+	}
+	if c.SqlPaths != nil {
+		// Reuse already allocated array
+		c.SqlPaths = c.SqlPaths[:0]
+	}
+	set := make(map[string]struct{})
+	for _, pattern := range c.GlobPatterns {
+		if !filepath.IsAbs(pattern) {
+			pattern = filepath.Join(basePath, pattern)
+		}
+		matches, err := fs.Glob(fsys, pattern)
+		if err != nil {
+			return errors.Errorf("failed to apply glob pattern: %w", err)
+		}
+		if len(matches) == 0 {
+			fmt.Fprintln(os.Stderr, "No seed files matched pattern:", pattern)
+		}
+		sort.Strings(matches)
+		// Remove duplicates
+		for _, item := range matches {
+			if _, exists := set[item]; !exists {
+				set[item] = struct{}{}
+				c.SqlPaths = append(c.SqlPaths, item)
+			}
+		}
+	}
+	return nil
+}
+
 func (h *hookConfig) HandleHook(hookType string) error {
 	// If not enabled do nothing
 	if !h.Enabled {

pkg/config/config_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"bytes"
 	_ "embed"
+	"path"
 	"strings"
 	"testing"
 	fs "testing/fstest"
@@ -247,3 +248,90 @@ func TestValidateHookURI(t *testing.T) {
 		})
 	}
 }
+
+func TestLoadSeedPaths(t *testing.T) {
+	t.Run("returns seed files matching patterns", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := fs.MapFS{
+			"supabase/seeds/seed1.sql":   &fs.MapFile{Data: []byte("INSERT INTO table1 VALUES (1);")},
+			"supabase/seeds/seed2.sql":   &fs.MapFile{Data: []byte("INSERT INTO table2 VALUES (2);")},
+			"supabase/seeds/seed3.sql":   &fs.MapFile{Data: []byte("INSERT INTO table2 VALUES (2);")},
+			"supabase/seeds/another.sql": &fs.MapFile{Data: []byte("INSERT INTO table2 VALUES (2);")},
+			"supabase/seeds/ignore.sql":  &fs.MapFile{Data: []byte("INSERT INTO table3 VALUES (3);")},
+		}
+		// Mock config patterns
+		config := seed{
+			Enabled: true,
+			GlobPatterns: []string{
+				"seeds/seed[12].sql",
+				"seeds/ano*.sql",
+			},
+		}
+		// Run test
+		err := config.loadSeedPaths("supabase", fsys)
+		// Check error
+		assert.NoError(t, err)
+		// Validate files
+		assert.ElementsMatch(t, []string{
+			"supabase/seeds/seed1.sql",
+			"supabase/seeds/seed2.sql",
+			"supabase/seeds/another.sql",
+		}, config.SqlPaths)
+	})
+	t.Run("returns seed files matching patterns skip duplicates", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := fs.MapFS{
+			"supabase/seeds/seed1.sql":   &fs.MapFile{Data: []byte("INSERT INTO table1 VALUES (1);")},
+			"supabase/seeds/seed2.sql":   &fs.MapFile{Data: []byte("INSERT INTO table2 VALUES (2);")},
+			"supabase/seeds/seed3.sql":   &fs.MapFile{Data: []byte("INSERT INTO table2 VALUES (2);")},
+			"supabase/seeds/another.sql": &fs.MapFile{Data: []byte("INSERT INTO table2 VALUES (2);")},
+			"supabase/seeds/ignore.sql":  &fs.MapFile{Data: []byte("INSERT INTO table3 VALUES (3);")},
+		}
+		// Mock config patterns
+		config := seed{
+			Enabled: true,
+			GlobPatterns: []string{
+				"seeds/seed[12].sql",
+				"seeds/ano*.sql",
+				"seeds/seed*.sql",
+			},
+		}
+		// Run test
+		err := config.loadSeedPaths("supabase", fsys)
+		// Check error
+		assert.NoError(t, err)
+		// Validate files
+		assert.ElementsMatch(t, []string{
+			"supabase/seeds/seed1.sql",
+			"supabase/seeds/seed2.sql",
+			"supabase/seeds/another.sql",
+			"supabase/seeds/seed3.sql",
+		}, config.SqlPaths)
+	})
+
+	t.Run("returns error on invalid pattern", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := fs.MapFS{}
+		// Mock config patterns
+		config := seed{Enabled: true, GlobPatterns: []string{"[*!#@D#"}}
+		// Run test
+		err := config.loadSeedPaths("", fsys)
+		// Check error
+		assert.ErrorIs(t, err, path.ErrBadPattern)
+		// The resuling seed list should be empty
+		assert.Empty(t, config.SqlPaths)
+	})
+
+	t.Run("returns empty list if no files match", func(t *testing.T) {
+		// Setup in-memory fs
+		fsys := fs.MapFS{}
+		// Mock config patterns
+		config := seed{Enabled: true, GlobPatterns: []string{"seeds/*.sql"}}
+		// Run test
+		err := config.loadSeedPaths("", fsys)
+		// Check error
+		assert.NoError(t, err)
+		// Validate files
+		assert.Empty(t, config.SqlPaths)
+	})
+}