From 228adc462289644ddc439862bd78a5827084324d Mon Sep 17 00:00:00 2001 From: Fabian Holler Date: Mon, 30 Sep 2024 10:53:41 +0200 Subject: [PATCH 1/3] change version to 5.1.0 --- internal/version/ver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version/ver b/internal/version/ver index 0062ac97..831446cb 100644 --- a/internal/version/ver +++ b/internal/version/ver @@ -1 +1 @@ -5.0.0 +5.1.0 From 2684cbeb071a1474c7091ce858cdbfafee64fc12 Mon Sep 17 00:00:00 2001 From: Fabian Holler Date: Wed, 2 Oct 2024 15:03:27 +0200 Subject: [PATCH 2/3] postgres: add index for task_run_file_input(input_file_id) Deleting untracked file_inputs from the database takes a very long time. The table has a multicolumn b-tree index for both it's columns. The input_file_id file is the second part of the index which still requires the full index to be scanned[^1]: Constraints on columns to the right of these columns are checked in the index, so they save visits to the table proper, but they do not reduce the portion of the index that has to be scanned. Create an index for the input_file_id column. Comparison of the query: EXPLAIN SELECT id FROM input_file WHERE NOT EXISTS (SELECT 1 FROM task_run_file_input AS trfi WHERE input_file.id = trfi.input_file_id); Without index: Gather (cost=2102870.71..2331530.97 rows=51672 width=4) Workers Planned: 2 -> Parallel Hash Anti Join (cost=2101870.71..2325363.77 rows=21530 width=4) Hash Cond: (input_file.id = trfi.input_file_id) -> Parallel Index Only Scan using input_file_pkey on input_file (cost=0.42..1127.35 rows=28633 width=4) -> Parallel Hash (cost=1170539.13..1170539.13 rows=56766813 width=4) -> Parallel Seq Scan on task_run_file_input trfi (cost=0.00..1170539.13 rows=56766813 width=4) With index: Gather (cost=1000.99..24457.81 rows=51672 width=4) (actual time=194.390..206.568 rows=0 loops=1) Workers Planned: 2 Workers Launched: 2 -> Nested Loop Anti Join (cost=0.99..18290.61 rows=21530 width=4) (actual time=153.282..153.283 rows=0 loops=3) -> Parallel Index Only Scan using input_file_pkey on input_file (cost=0.42..1127.35 rows=28633 width=4) (actual time=0.035..13.810 rows=22907 loops=3) Heap Fetches: 19485 -> Index Only Scan using task_run_file_input_input_file_id_idx on task_run_file_input trfi (cost=0.57..157.21 rows=7992 width=4) (actual time=0.006..0.006 rows=1 loops=68720) Index Cond: (input_file_id = input_file.id) Heap Fetches: 1352 [^1]: https://www.postgresql.org/docs/current/indexes-multicolumn.html --- pkg/storage/postgres/migrations/5.sql | 1 + 1 file changed, 1 insertion(+) create mode 100644 pkg/storage/postgres/migrations/5.sql diff --git a/pkg/storage/postgres/migrations/5.sql b/pkg/storage/postgres/migrations/5.sql new file mode 100644 index 00000000..8bf2f14b --- /dev/null +++ b/pkg/storage/postgres/migrations/5.sql @@ -0,0 +1 @@ +CREATE INDEX idx_task_run_file_input_input_file_id on task_run_file_input(input_file_id); From 8ec4ac1642b74d8dadd0e5150a481aeea12fdae3 Mon Sep 17 00:00:00 2001 From: Fabian Holler Date: Mon, 11 Nov 2024 17:04:41 +0100 Subject: [PATCH 3/3] upgrade db: support defining min. and max. database schema version Instead of requiring an up2date database schema allow to define a minimum and maximum compatible database schema version. The max. schema version is always the newest existing one at the time of a release. The minimum schema version can be an older one if it is still compatible. This can make upgrading baur easier, when db schema changes are downwards compatible, multiple baur version can be used with the same database schema. Even better would be to use a semantic version as schema version. This would require more work though :-) --- internal/command/upgrade_db.go | 6 +++--- internal/command/upgrade_db_test.go | 2 +- pkg/storage/postgres/schema.go | 20 ++++++++++++++------ pkg/storage/postgres/schema_test.go | 24 ++++++++++++++++++++++++ pkg/storage/storage.go | 7 ++++--- 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/internal/command/upgrade_db.go b/internal/command/upgrade_db.go index ba2b1ef6..a91ef5e6 100644 --- a/internal/command/upgrade_db.go +++ b/internal/command/upgrade_db.go @@ -84,17 +84,17 @@ func (*upgradeDbCmd) run(_ *cobra.Command, args []string) { } exitOnErr(err, "querying database schema version failed") - if curVer == clt.RequiredSchemaVersion() { + if curVer == clt.MaxSchemaVersion() { stdout.Println("database schema is already up to date, nothing to do") return } - if curVer > clt.RequiredSchemaVersion() { + if curVer > clt.MaxSchemaVersion() { fatal("database schema is from a newer baur version, please update baur") } err = clt.Upgrade(ctx) exitOnErr(err, "upgrading database schema failed") - stdout.Printf("database schema successfully upgraded from version %d to %d\n", curVer, clt.RequiredSchemaVersion()) + stdout.Printf("database schema successfully upgraded from version %d to %d\n", curVer, clt.MaxSchemaVersion()) } diff --git a/internal/command/upgrade_db_test.go b/internal/command/upgrade_db_test.go index 755b53b6..99c0bdb9 100644 --- a/internal/command/upgrade_db_test.go +++ b/internal/command/upgrade_db_test.go @@ -64,5 +64,5 @@ func TestUpgradeDb(t *testing.T) { upgradeDbCmd := newUpgradeDatabaseCmd() upgradeDbCmd.Command.Run(&upgradeDbCmd.Command, nil) - assert.Contains(t, stdoutBuf.String(), "database schema successfully upgraded from version 1 to 4") + assert.Contains(t, stdoutBuf.String(), "database schema successfully upgraded from version 1 to 5") } diff --git a/pkg/storage/postgres/schema.go b/pkg/storage/postgres/schema.go index 753ae6fe..175f708d 100644 --- a/pkg/storage/postgres/schema.go +++ b/pkg/storage/postgres/schema.go @@ -16,8 +16,12 @@ import ( "github.com/simplesurance/baur/v5/pkg/storage" ) -// schemaVer is the database schema version required by this package. -const schemaVer int32 = 4 +const ( + // minSchemaVer is the minimum required database schema version + minSchemaVer int32 = 4 + // maxSchemaVer is the highest database schema version that is compatible + maxSchemaVer int32 = 5 +) // migration represents a database schema migration. type migration struct { @@ -220,8 +224,12 @@ func (c *Client) ensureSchemaIsCompatible(ctx context.Context) error { return nil } - if ver != schemaVer { - return fmt.Errorf("database schema version is not compatible with baur version, schema version: %d, expected version: %d", ver, schemaVer) + if ver < minSchemaVer || ver > maxSchemaVer { + if minSchemaVer == maxSchemaVer { + return fmt.Errorf("database schema version is not compatible with baur version, schema version: %d, expecting version: %d", ver, minSchemaVer) + } + + return fmt.Errorf("database schema version is not compatible with baur version, schema version: %d, expecting schema version >=%d and <=%d", ver, minSchemaVer, maxSchemaVer) } return nil @@ -282,6 +290,6 @@ func (c *Client) SchemaVersion(ctx context.Context) (int32, error) { return c.schemaVersion(ctx) } -func (c *Client) RequiredSchemaVersion() int32 { - return schemaVer +func (c *Client) MaxSchemaVersion() int32 { + return maxSchemaVer } diff --git a/pkg/storage/postgres/schema_test.go b/pkg/storage/postgres/schema_test.go index da9e09a3..89fb9dfb 100644 --- a/pkg/storage/postgres/schema_test.go +++ b/pkg/storage/postgres/schema_test.go @@ -39,6 +39,30 @@ func TestIsCompatible_OldBaurSchemaExist(t *testing.T) { require.Contains(t, err.Error(), "incompatible database schema") } +func TestIsCompatible_MinVer(t *testing.T) { + client, cleanupFn := newTestClient(t) + t.Cleanup(cleanupFn) + + require.NoError(t, client.Init(ctx)) + + _, err := client.db.Exec(ctx, "UPDATE migrations set schema_version = $1", minSchemaVer) + require.NoError(t, err) + + require.NoError(t, client.IsCompatible(ctx)) +} + +func TestIsCompatible_MaxVer(t *testing.T) { + client, cleanupFn := newTestClient(t) + t.Cleanup(cleanupFn) + + require.NoError(t, client.Init(ctx)) + + _, err := client.db.Exec(ctx, "UPDATE migrations set schema_version = $1", maxSchemaVer) + require.NoError(t, err) + + require.NoError(t, client.IsCompatible(ctx)) +} + func TestIsCompatible_SchemaVersionDoesNotMatch(t *testing.T) { client, cleanupFn := newTestClient(t) defer cleanupFn() diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a6473797..e37a6ad5 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -135,9 +135,10 @@ type Storer interface { // SchemaVersion returns the version of the schema that the storage is // using. SchemaVersion(ctx context.Context) (int32, error) - // RequiredSchemaVersion returns the schema version that the Storer - // implementation requires. - RequiredSchemaVersion() int32 + // MaxSchemaVersion returns the max. supported database schema version. + // It is also the version to that [Upgrade] can migrate the current + // schema. + MaxSchemaVersion() int32 // IsCompatible verifies that the storage is compatible with the baur version IsCompatible(context.Context) error // Upgrade upgrades the schema to RequiredSchemaVersion().