Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support restoring local db from backup #3015

Merged
merged 1 commit into from
Jan 7, 2025
+325 −247
Merged

feat: support restoring local db from backup #3015

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion cmd/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,13 @@ var (
},
}

fromBackup string

dbStartCmd = &cobra.Command{
Use: "start",
Short: "Starts local Postgres database",
RunE: func(cmd *cobra.Command, args []string) error {
return start.Run(cmd.Context(), afero.NewOsFs())
return start.Run(cmd.Context(), fromBackup, afero.NewOsFs())
},
}

Expand Down Expand Up @@ -329,6 +331,8 @@ func init() {
lintFlags.Var(&lintFailOn, "fail-on", "Error level to exit with non-zero status.")
dbCmd.AddCommand(dbLintCmd)
// Build start command
startFlags := dbStartCmd.Flags()
startFlags.StringVar(&fromBackup, "from-backup", "", "Path to a logical backup file.")
dbCmd.AddCommand(dbStartCmd)
// Build test command
dbCmd.AddCommand(dbTestCmd)
Expand Down
42 changes: 38 additions & 4 deletions internal/db/start/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -29,11 +30,15 @@ var (
HealthTimeout = 120 * time.Second
//go:embed templates/schema.sql
initialSchema string
//go:embed templates/webhook.sql
webhookSchema string
//go:embed templates/_supabase.sql
_supabaseSchema string
//go:embed templates/restore.sh
restoreScript string
)

func Run(ctx context.Context, fsys afero.Fs) error {
func Run(ctx context.Context, fromBackup string, fsys afero.Fs) error {
if err := utils.LoadConfigFS(fsys); err != nil {
return err
}
Expand All @@ -43,7 +48,7 @@ func Run(ctx context.Context, fsys afero.Fs) error {
} else if !errors.Is(err, utils.ErrNotRunning) {
return err
}
err := StartDatabase(ctx, fsys, os.Stderr)
err := StartDatabase(ctx, fromBackup, fsys, os.Stderr)
if err != nil {
if err := utils.DockerRemoveAll(context.Background(), os.Stderr, utils.Config.ProjectId); err != nil {
fmt.Fprintln(os.Stderr, err)
Expand Down Expand Up @@ -86,6 +91,7 @@ cat <<'EOF' > /etc/postgresql-custom/pgsodium_root.key && \
cat <<'EOF' >> /etc/postgresql/postgresql.conf && \
docker-entrypoint.sh postgres -D /etc/postgresql
` + initialSchema + `
` + webhookSchema + `
` + _supabaseSchema + `
EOF
` + utils.Config.Db.RootKey + `
Expand Down Expand Up @@ -116,7 +122,7 @@ func NewHostConfig() container.HostConfig {
return hostConfig
}

func StartDatabase(ctx context.Context, fsys afero.Fs, w io.Writer, options ...func(*pgx.ConnConfig)) error {
func StartDatabase(ctx context.Context, fromBackup string, fsys afero.Fs, w io.Writer, options ...func(*pgx.ConnConfig)) error {
config := NewContainerConfig()
hostConfig := NewHostConfig()
networkingConfig := network.NetworkingConfig{
Expand All @@ -137,11 +143,35 @@ EOF
EOF`}
hostConfig.Tmpfs = map[string]string{"/docker-entrypoint-initdb.d": ""}
}
if len(fromBackup) > 0 {
config.Entrypoint = []string{"sh", "-c", `
cat <<'EOF' > /etc/postgresql.schema.sql && \
cat <<'EOF' > /docker-entrypoint-initdb.d/migrate.sh && \
cat <<'EOF' > /etc/postgresql-custom/pgsodium_root.key && \
cat <<'EOF' >> /etc/postgresql/postgresql.conf && \
docker-entrypoint.sh postgres -D /etc/postgresql
` + initialSchema + `
` + _supabaseSchema + `
EOF
` + restoreScript + `
EOF
` + utils.Config.Db.RootKey + `
EOF
` + utils.Config.Db.Settings.ToPostgresConfig() + `
EOF`}
if !filepath.IsAbs(fromBackup) {
fromBackup = filepath.Join(utils.CurrentDirAbs, fromBackup)
}
hostConfig.Binds = append(hostConfig.Binds, utils.ToDockerPath(fromBackup)+":/etc/backup.sql:ro")
}
// Creating volume will not override existing volume, so we must inspect explicitly
_, err := utils.Docker.VolumeInspect(ctx, utils.DbId)
utils.NoBackupVolume = client.IsErrNotFound(err)
if utils.NoBackupVolume {
fmt.Fprintln(w, "Starting database...")
} else if len(fromBackup) > 0 {
utils.CmdSuggestion = fmt.Sprintf("Run %s to remove existing docker volumes.", utils.Aqua("supabase stop --no-backup"))
return errors.Errorf("backup volume already exists")
} else {
fmt.Fprintln(w, "Starting database from backup...")
}
Expand All @@ -152,7 +182,11 @@ EOF`}
return err
}
// Initialize if we are on PG14 and there's no existing db volume
if utils.NoBackupVolume {
if len(fromBackup) > 0 {
if err := initSchema15(ctx, utils.DbId); err != nil {
return err
}
} else if utils.NoBackupVolume {
if err := SetupLocalDatabase(ctx, "", fsys, w, options...); err != nil {
return err
}
Expand Down
16 changes: 8 additions & 8 deletions internal/db/start/start_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func TestStartDatabase(t *testing.T) {
conn.Query(roles).
Reply("CREATE ROLE")
// Run test
err := StartDatabase(context.Background(), fsys, io.Discard, conn.Intercept)
err := StartDatabase(context.Background(), "", fsys, io.Discard, conn.Intercept)
// Check error
assert.NoError(t, err)
assert.Empty(t, apitest.ListUnmatchedRequests())
Expand Down Expand Up @@ -124,7 +124,7 @@ func TestStartDatabase(t *testing.T) {
},
}})
// Run test
err := StartDatabase(context.Background(), fsys, io.Discard)
err := StartDatabase(context.Background(), "", fsys, io.Discard)
// Check error
assert.NoError(t, err)
assert.Empty(t, apitest.ListUnmatchedRequests())
Expand All @@ -149,7 +149,7 @@ func TestStartDatabase(t *testing.T) {
Get("/v" + utils.Docker.ClientVersion() + "/images/" + utils.GetRegistryImageUrl(utils.Config.Db.Image) + "/json").
Reply(http.StatusInternalServerError)
// Run test
err := StartDatabase(context.Background(), fsys, io.Discard)
err := StartDatabase(context.Background(), "", fsys, io.Discard)
// Check error
assert.ErrorContains(t, err, "request returned Internal Server Error for API route and version")
assert.Empty(t, apitest.ListUnmatchedRequests())
Expand All @@ -161,7 +161,7 @@ func TestStartCommand(t *testing.T) {
// Setup in-memory fs
fsys := afero.NewMemMapFs()
// Run test
err := Run(context.Background(), fsys)
err := Run(context.Background(), "", fsys)
// Check error
assert.ErrorIs(t, err, os.ErrNotExist)
})
Expand All @@ -177,7 +177,7 @@ func TestStartCommand(t *testing.T) {
Get("/v" + utils.Docker.ClientVersion() + "/containers").
ReplyError(errors.New("network error"))
// Run test
err := Run(context.Background(), fsys)
err := Run(context.Background(), "", fsys)
// Check error
assert.ErrorContains(t, err, "network error")
assert.Empty(t, apitest.ListUnmatchedRequests())
Expand All @@ -195,7 +195,7 @@ func TestStartCommand(t *testing.T) {
Reply(http.StatusOK).
JSON(types.ContainerJSON{})
// Run test
err := Run(context.Background(), fsys)
err := Run(context.Background(), "", fsys)
// Check error
assert.NoError(t, err)
assert.Empty(t, apitest.ListUnmatchedRequests())
Expand All @@ -221,7 +221,7 @@ func TestStartCommand(t *testing.T) {
// Cleanup resources
apitest.MockDockerStop(utils.Docker)
// Run test
err := Run(context.Background(), fsys)
err := Run(context.Background(), "", fsys)
// Check error
assert.ErrorContains(t, err, "network error")
assert.Empty(t, apitest.ListUnmatchedRequests())
Expand Down Expand Up @@ -350,7 +350,7 @@ func TestStartDatabaseWithCustomSettings(t *testing.T) {
defer conn.Close(t)

// Run test
err := StartDatabase(context.Background(), fsys, io.Discard, conn.Intercept)
err := StartDatabase(context.Background(), "", fsys, io.Discard, conn.Intercept)

// Check error
assert.NoError(t, err)
Expand Down
41 changes: 41 additions & 0 deletions internal/db/start/templates/restore.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#!/bin/sh
set -eu

#######################################
# Used by both ami and docker builds to initialise database schema.
# Env vars:
# POSTGRES_DB defaults to postgres
# POSTGRES_HOST defaults to localhost
# POSTGRES_PORT defaults to 5432
# POSTGRES_PASSWORD defaults to ""
# USE_DBMATE defaults to ""
# Exit code:
# 0 if migration succeeds, non-zero on error.
#######################################

export PGDATABASE="${POSTGRES_DB:-postgres}"
export PGHOST="${POSTGRES_HOST:-localhost}"
export PGPORT="${POSTGRES_PORT:-5432}"
export PGPASSWORD="${POSTGRES_PASSWORD:-}"

echo "$0: restoring roles"
cat "/etc/backup.sql" \
| grep 'CREATE ROLE' \
| grep -v 'supabase_admin' \
| psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin

echo "$0: restoring schema"
cat "/etc/backup.sql" \
| sed -E 's/^CREATE VIEW /CREATE OR REPLACE VIEW /' \
| sed -E 's/^CREATE FUNCTION /CREATE OR REPLACE FUNCTION /' \
| sed -E 's/^CREATE TRIGGER /CREATE OR REPLACE TRIGGER /' \
| sed -E 's/^GRANT ALL ON FUNCTION graphql_public\./-- &/' \
| sed -E 's/^CREATE ROLE /-- &/' \
| psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin

# run any post migration script to update role passwords
postinit="/etc/postgresql.schema.sql"
if [ -e "$postinit" ]; then
echo "$0: running $postinit"
psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U supabase_admin -f "$postinit"
fi
Loading
Loading