feat: migrations post stateless version

cmd/buckets_upgrade.go

@@ -36,7 +36,7 @@ func NewBucketUpgrade() *cobra.Command {
 				return upgradeAll(cmd)
 			}
 
-			logger := logging.NewDefaultLogger(cmd.OutOrStdout(), service.IsDebug(cmd), false)
+			logger := logging.NewDefaultLogger(cmd.OutOrStdout(), service.IsDebug(cmd), false, false)
 
 			return driver.UpgradeBucket(logging.ContextWithLogger(cmd.Context(), logger), args[0])
 		},
@@ -49,7 +49,7 @@ func NewBucketUpgrade() *cobra.Command {
 }
 
 func upgradeAll(cmd *cobra.Command) error {
-	logger := logging.NewDefaultLogger(cmd.OutOrStdout(), service.IsDebug(cmd), false)
+	logger := logging.NewDefaultLogger(cmd.OutOrStdout(), service.IsDebug(cmd), false, false)
 	ctx := logging.ContextWithLogger(cmd.Context(), logger)
 
 	connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(cmd)

go.mod

@@ -14,26 +14,29 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10
 	github.com/bluele/gcache v0.0.2
 	github.com/dop251/goja v0.0.0-20241009100908-5f46f2705ca3
-	github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163904-e440de7907c7
+	github.com/formancehq/go-libs/v2 v2.0.1-0.20241025143036-1a15bfa1e18e
 	github.com/formancehq/ledger/pkg/client v0.0.0-00010101000000-000000000000
 	github.com/go-chi/chi/v5 v5.1.0
 	github.com/go-chi/cors v1.2.1
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
 	github.com/invopop/jsonschema v0.12.0
+	github.com/jackc/pgx/v5 v5.7.1
 	github.com/jamiealquiza/tachymeter v2.0.0+incompatible
 	github.com/logrusorgru/aurora v2.0.3+incompatible
 	github.com/nats-io/nats.go v1.37.0
 	github.com/onsi/ginkgo/v2 v2.20.2
 	github.com/onsi/gomega v1.34.2
 	github.com/ory/dockertest/v3 v3.11.0
 	github.com/pborman/uuid v1.2.1
+	github.com/pkg/errors v0.9.1
 	github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	github.com/uptrace/bun v1.2.3
 	github.com/uptrace/bun/dialect/pgdialect v1.2.3
+	github.com/uptrace/bun/extra/bundebug v1.2.3
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/xo/dburl v0.23.2
 	go.opentelemetry.io/otel v1.31.0
@@ -44,8 +47,11 @@ require (
 	go.uber.org/mock v0.4.0
 	golang.org/x/oauth2 v0.23.0
 	golang.org/x/sync v0.8.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
+require github.com/hashicorp/go-hclog v1.6.3 // indirect
+
 require (
 	dario.cat/mergo v1.0.1 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
@@ -89,6 +95,7 @@ require (
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/ebitengine/purego v0.8.0 // indirect
 	github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731 // indirect
+	github.com/fatih/color v1.17.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
@@ -116,7 +123,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/pgx/v5 v5.7.1
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
@@ -128,6 +134,8 @@ require (
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
 	github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/minio/highwayhash v1.0.3 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/term v0.5.0 // indirect
@@ -142,7 +150,6 @@ require (
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/opencontainers/runc v1.1.14 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect
-	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
@@ -199,5 +206,4 @@ require (
 	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -99,12 +99,15 @@ github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+Gv
 github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731 h1:R/ZjJpjQKsZ6L/+Gf9WHbt31GG8NMVcpRqUE+1mMIyo=
 github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731/go.mod h1:M9R1FoZ3y//hwwnJtO51ypFGwm8ZfpxPT/ZLtO1mcgQ=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
 github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163904-e440de7907c7 h1:x8vIRM5+y01pLs2YqnYcoUsvFJ/6cP5qDtsM248OmWM=
-github.com/formancehq/go-libs/v2 v2.0.1-0.20241023163904-e440de7907c7/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241025112506-882046e08347 h1:4mT6Q+UCRpc2bc1PaNcNRJJf+o298XWStUmBhL1Ooww=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241025112506-882046e08347/go.mod h1:DgV5ZYXS1tI6oryEiuwtGcJ9FWIDgrbN83m9F98WW7U=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241025143036-1a15bfa1e18e h1:3mvyKpdcGrrIbjujbQPl3yVJPL/AwJhVSJY7nN/xurE=
+github.com/formancehq/go-libs/v2 v2.0.1-0.20241025143036-1a15bfa1e18e/go.mod h1:LtNas5a+aL/oK2Qfsy39QngT6dsa2fUbSTYbCZY1eAY=
 github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 h1:LOd5hxnXDIBcehFrpW1OnXk+VSs0yJXeu1iAOO+Hji4=
 github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417/go.mod h1:btuSv05cYwi9BvLRxVs5zrunU+O1vTgigG1T6UsawcY=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
@@ -311,6 +314,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
@@ -453,16 +457,23 @@ golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

internal/storage/bucket/bucket.go

@@ -4,7 +4,6 @@
 	"bytes"
 	"context"
 	_ "embed"
-	"errors"
 	"fmt"
 	"github.com/formancehq/go-libs/v2/migrations"
 	ledger "github.com/formancehq/ledger/internal"
@@ -13,21 +12,26 @@
 	"text/template"
 )
 
+// stateless version (+1 regarding directory name, as migrations start from 1 in the lib)
+const MinimalSchemaVersion = 12
+
 type Bucket struct {
 	name string
 	db   bun.IDB
 }
 
 func (b *Bucket) Migrate(ctx context.Context, tracer trace.Tracer) error {
-	return Migrate(ctx, tracer, b.db, b.name)
+	return migrate(ctx, tracer, b.db, b.name)
 }
 
 func (b *Bucket) IsUpToDate(ctx context.Context) (bool, error) {
-	ret, err := GetMigrator(b.name).IsUpToDate(ctx, b.db)
-	if err != nil && errors.Is(err, migrations.ErrMissingVersionTable) {
-		return false, nil
+	migrator := GetMigrator(b.name)
+	lastVersion, err := migrator.GetLastVersion(ctx, b.db)
+	if err != nil {
+		return false, err
 	}
-	return ret, err
+
+	return lastVersion >= MinimalSchemaVersion, nil
 }
 
 func (b *Bucket) GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) {

internal/storage/bucket/bucket_test.go

@@ -3,6 +3,7 @@
 package bucket_test
 
 import (
+	"github.com/formancehq/go-libs/v2/bun/bundebug"
 	"github.com/formancehq/ledger/internal/storage/bucket"
 	"github.com/formancehq/ledger/internal/storage/driver"
 	"go.opentelemetry.io/otel/trace/noop"
@@ -23,6 +24,10 @@ func TestBuckets(t *testing.T) {
 	db, err := bunconnect.OpenSQLDB(ctx, pgDatabase.ConnectionOptions())
 	require.NoError(t, err)
 
+	if testing.Verbose() {
+		db.AddQueryHook(bundebug.NewQueryHook())
+	}
+
 	require.NoError(t, driver.Migrate(ctx, db))
 
 	b := bucket.New(db, name)

internal/storage/bucket/migrations.go

@@ -9,18 +9,22 @@
 )
 
 //go:embed migrations
-var migrationsDir embed.FS
+var MigrationsFS embed.FS
 
 func GetMigrator(name string) *migrations.Migrator {
 	migrator := migrations.NewMigrator(migrations.WithSchema(name, true))
-	migrator.RegisterMigrationsFromFileSystem(migrationsDir, "migrations")
+	_migrations, err := migrations.CollectMigrations(MigrationsFS, name)
+	if err != nil {
+		panic(err)
+	}
+	migrator.RegisterMigrations(_migrations...)
 
 	return migrator
 }
 
-func Migrate(ctx context.Context, tracer trace.Tracer, db bun.IDB, name string) error {
+func migrate(ctx context.Context, tracer trace.Tracer, db bun.IDB, name string) error {
 	ctx, span := tracer.Start(ctx, "Migrate bucket")
 	defer span.End()
 
 	return GetMigrator(name).Up(ctx, db)
-}
+}

internal/storage/bucket/migrations/0-init-schema/notes.yaml

@@ -0,0 +1 @@
+name: Initialize schema

internal/storage/bucket/migrations/0-init-schema.sql → internal/storage/bucket/migrations/0-init-schema/up.sql

@@ -1,3 +1,5 @@
+set search_path = '{{.Schema}}';
+
 create aggregate aggregate_objects(jsonb) (
     sfunc = jsonb_concat,
     stype = jsonb,

internal/storage/bucket/migrations/0-init-schema/up_tests_after.sql

@@ -0,0 +1,43 @@
+set search_path = '{{.Schema}}';
+
+create extension if not exists "uuid-ossp";
+
+insert into logs(seq, ledger, id, type, date, data, hash)
+select
+	seq,
+	'ledger' || seq % 5,
+	(seq/5) + (seq % 5),
+	'NEW_TRANSACTION',
+	now(),
+	('{'
+		'"transaction": {'
+			'"id": ' || (seq/5) + (seq % 5) || ','
+			'"timestamp": "' || now() || '",'
+			'"postings": ['
+	            '{'
+					'"source": "world",'
+					'"destination": "orders:' || seq || '",'
+					'"asset": "USD",'
+					'"amount": 100'
+				'},'
+				'{'
+					'"destination": "fees",'
+					'"source": "orders:' || seq || '",'
+					'"asset": "USD",'
+					'"amount": 1'
+				'},'
+				'{'
+					'"destination": "sellers:' || (seq % 5) || '",'
+					'"source": "orders:' || seq || '",'
+					'"asset": "USD",'
+					'"amount": 99'
+				'}'
+	        '],'
+	        '"metadata": { "tax": "1%" }'
+		'},'
+		'"accountMetadata": {'
+			'"orders:' || seq || '": { "tax": "1%" }'
+		'}'
+	'}')::jsonb,
+	'invalid-hash'
+from generate_series(0, 100) as seq;

internal/storage/bucket/migrations/1-fix-trigger/notes.yaml

@@ -0,0 +1 @@
+name: Fix triggers

internal/storage/bucket/migrations/1-fix-trigger.sql → internal/storage/bucket/migrations/1-fix-trigger/up.sql

@@ -1,3 +1,5 @@
+set search_path = '{{.Schema}}';
+
 create or replace function insert_posting(_transaction_seq bigint, _ledger varchar, _insertion_date timestamp without time zone,
                                _effective_date timestamp without time zone, posting jsonb, _account_metadata jsonb)
     returns void

internal/storage/bucket/migrations/10-fillfactor-on-moves/notes.yaml

@@ -0,0 +1 @@
+name: Define fill factor of moves table

internal/storage/bucket/migrations/10-fillfactor-on-moves.sql → internal/storage/bucket/migrations/10-fillfactor-on-moves/up.sql

@@ -1 +1,3 @@
+set search_path = '{{.Schema}}';
+
 alter table moves set (fillfactor = 80);

internal/storage/bucket/migrations/11-make-stateless/notes.yaml

@@ -0,0 +1 @@
+name: Make stateless

internal/storage/bucket/migrations/11-make-stateless.sql → internal/storage/bucket/migrations/11-make-stateless/up.sql

@@ -1,3 +1,5 @@
+set search_path = '{{.Schema}}';
+
 drop trigger insert_account  on accounts;
 drop trigger update_account  on accounts;
 drop trigger insert_transaction  on transactions;
@@ -511,4 +513,4 @@ $do$
 			execute vsql;
 		end loop;
 	END
-$do$;
+$do$;

internal/storage/bucket/migrations/12-moves-fill-transaction-id/notes.yaml

@@ -0,0 +1 @@
+name: Fill transaction ids of table moves

internal/storage/bucket/migrations/12-moves-fill-transaction-id/up.sql

@@ -0,0 +1,33 @@
+do $$
+	declare
+		_batch_size integer := 30;
+	begin
+		set search_path = '{{ .Schema }}';
+
+		loop
+			with _outdated_moves as (
+				select *
+				from moves
+				where transactions_id is null
+				limit _batch_size
+			)
+			update moves
+			set transactions_id = (
+				select id
+				from transactions
+				where seq = moves.transactions_seq
+			)
+			from _outdated_moves
+			where moves.seq in (_outdated_moves.seq);
+
+			exit when not found;
+
+			raise info 'commit batch';
+			commit ;
+		end loop;
+
+		alter table moves
+		alter column transactions_id set not null;
+	end
+$$
+language plpgsql;

internal/storage/bucket/migrations/12-moves-fill-transaction-id/up_tests_after.sql

@@ -0,0 +1,10 @@
+set search_path = '{{.Schema}}';
+
+do $$
+begin
+	assert (
+		select count(*)
+		from moves
+		where transactions_id is null
+    ) = 0, 'Still some rows with null transactions_id';
+end$$;

internal/storage/bucket/migrations/12-moves-fill-transaction-id/up_tests_before.sql

@@ -0,0 +1,10 @@
+set search_path = '{{.Schema}}';
+
+do $$
+begin
+	assert (
+		select count(*)
+		from moves
+		where transactions_id is null
+	) > 0, 'Should have some transactions with null transactions_id';
+end$$;

internal/storage/bucket/migrations/13-transactions-fill-inserted-at/notes.yaml

@@ -0,0 +1 @@
+name: Fill inserted_at column of transactions table