diff --git a/site/docs/migrations.mdx b/site/docs/migrations.mdx index 3ad94f516..263b96add 100644 --- a/site/docs/migrations.mdx +++ b/site/docs/migrations.mdx @@ -26,7 +26,21 @@ Migrations can use the `Kysely.schema` module to modify the schema. Migrations c ## Execution order -Execution order of the migrations is the alpabetical order of their names. An excellent way to name your migrations is to prefix them with an ISO 8601 date string. A date prefix works well in large teams where multiple team members may add migrations at the same time in parallel commits without knowing about the other migrations. +Migrations will be run in the alpha-numeric order of your migration names. An excellent way to name your migrations is to prefix them with an ISO 8601 date string. + +By default, Kysely will also ensure this order matches the execution order of any previously executed migrations in your database. If the orders do not match (for example, a new migration was added alphabetically before a previously executed one), an error will be returned. This adds safety by always executing your migrations in the correct, alphanumeric order. + +There is also an `allowUnorderedMigrations` option. This option will allow new migrations to be run even if they are added alphabetically before ones that have already executed. Allowing unordered migrations works well in large teams where multiple team members may add migrations at the same time in parallel commits without knowing about the other migrations. Pending (unexecuted) migrations will be run in alpha-numeric order when migrating up. When migrating down, migrations will be undone in the opposite order in which they were executed (reverse sorted by execution timestamp). + +To allow unordered migrations, pass the `allowUnorderedMigrations` option to Migrator: + +```ts +const migrator = new Migrator({ + db, + provider: new FileMigrationProvider(...), + allowUnorderedMigrations: true +}) +``` ## Single file vs multiple file migrations diff --git a/src/migration/migrator.ts b/src/migration/migrator.ts index 1fa1a3fcd..c2f1b58dc 100644 --- a/src/migration/migrator.ts +++ b/src/migration/migrator.ts @@ -9,6 +9,7 @@ import { freeze, getLast } from '../util/object-utils.js' export const DEFAULT_MIGRATION_TABLE = 'kysely_migration' export const DEFAULT_MIGRATION_LOCK_TABLE = 'kysely_migration_lock' +export const DEFAULT_ALLOW_UNORDERED_MIGRATIONS = false export const MIGRATION_LOCK_ID = 'migration_lock' export const NO_MIGRATIONS: NoMigrations = freeze({ __noMigrations__: true }) @@ -96,7 +97,7 @@ export class Migrator { * This method goes through all possible migrations provided by the provider and runs the * ones whose names come alphabetically after the last migration that has been run. If the * list of executed migrations doesn't match the beginning of the list of possible migrations - * an error is thrown. + * an error is returned. * * ### Examples * @@ -133,7 +134,7 @@ export class Migrator { * ``` */ async migrateToLatest(): Promise { - return this.#migrate(({ migrations }) => migrations.length - 1) + return this.#migrate(() => ({ direction: 'Up', step: Infinity })) } /** @@ -162,21 +163,43 @@ export class Migrator { async migrateTo( targetMigrationName: string | NoMigrations ): Promise { - return this.#migrate(({ migrations }) => { - if (targetMigrationName === NO_MIGRATIONS) { - return -1 - } + return this.#migrate( + ({ + migrations, + executedMigrations, + pendingMigrations, + }: MigrationState) => { + if (targetMigrationName === NO_MIGRATIONS) { + return { direction: 'Down', step: Infinity } + } - const index = migrations.findIndex( - (it) => it.name === targetMigrationName - ) + if ( + !migrations.find((m) => m.name === (targetMigrationName as string)) + ) { + throw new Error(`migration "${targetMigrationName}" doesn't exist`) + } - if (index === -1) { - throw new Error(`migration "${targetMigrationName}" doesn't exist`) - } + const executedIndex = executedMigrations.indexOf( + targetMigrationName as string + ) + const pendingIndex = pendingMigrations.findIndex( + (m) => m.name === (targetMigrationName as string) + ) - return index - }) + if (executedIndex !== -1) { + return { + direction: 'Down', + step: executedMigrations.length - executedIndex - 1, + } + } else if (pendingIndex !== -1) { + return { direction: 'Up', step: pendingIndex + 1 } + } else { + throw new Error( + `migration "${targetMigrationName}" isn't executed or pending` + ) + } + } + ) } /** @@ -194,9 +217,7 @@ export class Migrator { * ``` */ async migrateUp(): Promise { - return this.#migrate(({ currentIndex, migrations }) => - Math.min(currentIndex + 1, migrations.length - 1) - ) + return this.#migrate(() => ({ direction: 'Up', step: 1 })) } /** @@ -214,15 +235,18 @@ export class Migrator { * ``` */ async migrateDown(): Promise { - return this.#migrate(({ currentIndex }) => Math.max(currentIndex - 1, -1)) + return this.#migrate(() => ({ direction: 'Down', step: 1 })) } async #migrate( - getTargetMigrationIndex: (state: MigrationState) => number | undefined + getMigrationDirectionAndStep: (state: MigrationState) => { + direction: MigrationDirection + step: number + } ): Promise { try { await this.#ensureMigrationTablesExists() - return await this.#runMigrations(getTargetMigrationIndex) + return await this.#runMigrations(getMigrationDirectionAndStep) } catch (error) { if (error instanceof MigrationResultSetError) { return error.resultSet @@ -244,6 +268,12 @@ export class Migrator { return this.#props.migrationLockTableName ?? DEFAULT_MIGRATION_LOCK_TABLE } + get #allowUnorderedMigrations(): boolean { + return ( + this.#props.allowUnorderedMigrations ?? DEFAULT_ALLOW_UNORDERED_MIGRATIONS + ) + } + get #schemaPlugin(): KyselyPlugin { if (this.#migrationTableSchema) { return new WithSchemaPlugin(this.#migrationTableSchema) @@ -383,7 +413,10 @@ export class Migrator { } async #runMigrations( - getTargetMigrationIndex: (state: MigrationState) => number | undefined + getMigrationDirectionAndStep: (state: MigrationState) => { + direction: MigrationDirection + step: number + } ): Promise { const adapter = this.#props.db.getExecutor().adapter @@ -397,23 +430,22 @@ export class Migrator { const run = async (db: Kysely): Promise => { try { await adapter.acquireMigrationLock(db, lockOptions) - const state = await this.#getState(db) if (state.migrations.length === 0) { return { results: [] } } - const targetIndex = getTargetMigrationIndex(state) + const { direction, step } = getMigrationDirectionAndStep(state) - if (targetIndex === undefined) { + if (step <= 0) { return { results: [] } } - if (targetIndex < state.currentIndex) { - return await this.#migrateDown(db, state, targetIndex) - } else if (targetIndex > state.currentIndex) { - return await this.#migrateUp(db, state, targetIndex) + if (direction === 'Down') { + return await this.#migrateDown(db, state, step) + } else if (direction === 'Up') { + return await this.#migrateUp(db, state, step) } return { results: [] } @@ -433,13 +465,30 @@ export class Migrator { const migrations = await this.#resolveMigrations() const executedMigrations = await this.#getExecutedMigrations(db) - this.#ensureMigrationsNotCorrupted(migrations, executedMigrations) + this.#ensureNoMissingMigrations(migrations, executedMigrations) + if (!this.#allowUnorderedMigrations) { + this.#ensureMigrationsInOrder(migrations, executedMigrations) + } + + const pendingMigrations = this.#getPendingMigrations( + migrations, + executedMigrations + ) return freeze({ migrations, - currentIndex: migrations.findIndex( - (it) => it.name === getLast(executedMigrations) - ), + executedMigrations, + lastMigration: getLast(executedMigrations), + pendingMigrations, + }) + } + + #getPendingMigrations( + migrations: ReadonlyArray, + executedMigrations: ReadonlyArray + ): ReadonlyArray { + return migrations.filter((migration) => { + return !executedMigrations.includes(migration.name) }) } @@ -461,16 +510,17 @@ export class Migrator { .withPlugin(this.#schemaPlugin) .selectFrom(this.#migrationTable) .select('name') - .orderBy('name') + .orderBy(['timestamp', 'name']) .execute() return executedMigrations.map((it) => it.name) } - #ensureMigrationsNotCorrupted( + #ensureNoMissingMigrations( migrations: ReadonlyArray, executedMigrations: ReadonlyArray ) { + // Ensure all executed migrations exist in the `migrations` list. for (const executed of executedMigrations) { if (!migrations.some((it) => it.name === executed)) { throw new Error( @@ -478,10 +528,13 @@ export class Migrator { ) } } + } - // Now we know all executed migrations exist in the `migrations` list. - // Next we need to make sure that the executed migratiosns are the first - // ones in the migration list. + #ensureMigrationsInOrder( + migrations: ReadonlyArray, + executedMigrations: ReadonlyArray + ) { + // Ensure the executed migrations are the first ones in the migration list. for (let i = 0; i < executedMigrations.length; ++i) { if (migrations[i].name !== executedMigrations[i]) { throw new Error( @@ -494,22 +547,27 @@ export class Migrator { async #migrateDown( db: Kysely, state: MigrationState, - targetIndex: number + step: number ): Promise { - const results: MigrationResult[] = [] + const migrationsToRollback: ReadonlyArray = + state.executedMigrations + .slice() + .reverse() + .slice(0, step) + .map((name) => { + return state.migrations.find((it) => it.name === name)! + }) - for (let i = state.currentIndex; i > targetIndex; --i) { - results.push({ - migrationName: state.migrations[i].name, + const results: MigrationResult[] = migrationsToRollback.map((migration) => { + return { + migrationName: migration.name, direction: 'Down', status: 'NotExecuted', - }) - } + } + }) for (let i = 0; i < results.length; ++i) { - const migration = state.migrations.find( - (it) => it.name === results[i].migrationName - )! + const migration = migrationsToRollback[i] try { if (migration.down) { @@ -546,22 +604,21 @@ export class Migrator { async #migrateUp( db: Kysely, state: MigrationState, - targetIndex: number + step: number ): Promise { - const results: MigrationResult[] = [] + const migrationsToRun: ReadonlyArray = + state.pendingMigrations.slice(0, step) - for (let i = state.currentIndex + 1; i <= targetIndex; ++i) { - results.push({ - migrationName: state.migrations[i].name, + const results: MigrationResult[] = migrationsToRun.map((migration) => { + return { + migrationName: migration.name, direction: 'Up', status: 'NotExecuted', - }) - } + } + }) - for (let i = 0; i < results.length; ++i) { - const migration = state.migrations.find( - (it) => it.name === results[i].migrationName - )! + for (let i = 0; i < results.length; i++) { + const migration = state.pendingMigrations[i] try { await migration.up(db) @@ -657,6 +714,21 @@ export interface MigratorProps { * This only works on postgres. */ readonly migrationTableSchema?: string + + /** + * Enforces whether or not migrations must be run in alpha-numeric order. + * + * When false, migrations must be run in their exact alpha-numeric order. + * This is checked against the migrations already run in the database + * (`migrationTableName'). This ensures your migrations are always run in + * the same order and is the safest option. + * + * When true, migrations are still run in alpha-numeric order, but + * the order is not checked against already-run migrations in the database. + * Kysely will simply run all migrations that haven't run yet, in alpha-numeric + * order. + */ + readonly allowUnorderedMigrations?: boolean } /** @@ -694,13 +766,15 @@ export interface MigrationResultSet { readonly results?: MigrationResult[] } +type MigrationDirection = 'Up' | 'Down' + export interface MigrationResult { readonly migrationName: string /** * The direction in which this migration was executed. */ - readonly direction: 'Up' | 'Down' + readonly direction: MigrationDirection /** * The execution status. @@ -772,8 +846,14 @@ interface MigrationState { // All migrations sorted by name. readonly migrations: ReadonlyArray - // Index of the last executed migration. - readonly currentIndex: number + // Names of executed migrations sorted by execution timestamp + readonly executedMigrations: ReadonlyArray + + // Name of the last executed migration. + readonly lastMigration?: string + + // Migrations that have not yet ran + readonly pendingMigrations: ReadonlyArray } class MigrationResultSetError extends Error { diff --git a/test/node/src/migration.test.ts b/test/node/src/migration.test.ts index 853e14e7a..7556edd8b 100644 --- a/test/node/src/migration.test.ts +++ b/test/node/src/migration.test.ts @@ -130,6 +130,35 @@ for (const dialect of DIALECTS) { expect(executedUpMethods2).to.eql([]) }) + it('should run a new migration added before the last executed one with allowUnorderedMigrations enabled', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration3'], + { allowUnorderedMigrations: true } + ) + + const { results: results1 } = await migrator1.migrateToLatest() + + const [migrator2, executedUpMethods2] = createMigrations( + ['migration1', 'migration2', 'migration3', 'migration4'], + { allowUnorderedMigrations: true } + ) + + const { results: results2 } = await migrator2.migrateToLatest() + + expect(results1).to.eql([ + { migrationName: 'migration1', direction: 'Up', status: 'Success' }, + { migrationName: 'migration3', direction: 'Up', status: 'Success' }, + ]) + + expect(results2).to.eql([ + { migrationName: 'migration2', direction: 'Up', status: 'Success' }, + { migrationName: 'migration4', direction: 'Up', status: 'Success' }, + ]) + + expect(executedUpMethods1).to.eql(['migration1', 'migration3']) + expect(executedUpMethods2).to.eql(['migration2', 'migration4']) + }) + it('should return an error if a previously executed migration is missing', async () => { const [migrator1, executedUpMethods1] = createMigrations([ 'migration1', @@ -189,6 +218,63 @@ for (const dialect of DIALECTS) { expect(executedUpMethods2).to.eql([]) }) + describe('with allowUnorderedMigrations', () => { + it('should return an error if a previously executed migration is missing', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration2', 'migration3'], + { allowUnorderedMigrations: true } + ) + + await migrator1.migrateToLatest() + + const [migrator2, executedUpMethods2] = createMigrations( + ['migration2', 'migration3', 'migration4'], + { allowUnorderedMigrations: true } + ) + + const { error } = await migrator2.migrateToLatest() + + expect(error).to.be.an.instanceOf(Error) + expect(getMessage(error)).to.eql( + 'corrupted migrations: previously executed migration migration1 is missing' + ) + + expect(executedUpMethods1).to.eql([ + 'migration1', + 'migration2', + 'migration3', + ]) + expect(executedUpMethods2).to.eql([]) + }) + + it('should return an error if a the last executed migration is not found', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration2', 'migration3'], + { allowUnorderedMigrations: true } + ) + + const [migrator2, executedUpMethods2] = createMigrations( + ['migration1', 'migration2', 'migration4'], + { allowUnorderedMigrations: true } + ) + + await migrator1.migrateToLatest() + const { error } = await migrator2.migrateToLatest() + + expect(error).to.be.an.instanceOf(Error) + expect(getMessage(error)).to.eql( + 'corrupted migrations: previously executed migration migration3 is missing' + ) + + expect(executedUpMethods1).to.eql([ + 'migration1', + 'migration2', + 'migration3', + ]) + expect(executedUpMethods2).to.eql([]) + }) + }) + it('should return an error if one of the migrations fails', async () => { const [migrator, executedUpMethods] = createMigrations([ 'migration1', @@ -369,6 +455,149 @@ for (const dialect of DIALECTS) { expect(executedUpMethods2).to.eql([]) expect(executedDownMethods2).to.eql(['migration4', 'migration3']) }) + + describe('with allowUnorderedMigrations enabled', () => { + it('should migrate up to a specific migration', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration3', 'migration4', 'migration5'], + { allowUnorderedMigrations: true } + ) + + const { results: results1 } = await migrator1.migrateTo('migration3') + + const [migrator2, executedUpMethods2] = createMigrations( + [ + 'migration1', + 'migration2', + 'migration3', + 'migration4', + 'migration5', + ], + { allowUnorderedMigrations: true } + ) + + const { results: results2 } = await migrator2.migrateTo('migration4') + + expect(results1).to.eql([ + { migrationName: 'migration1', direction: 'Up', status: 'Success' }, + { migrationName: 'migration3', direction: 'Up', status: 'Success' }, + ]) + + expect(results2).to.eql([ + { migrationName: 'migration2', direction: 'Up', status: 'Success' }, + { migrationName: 'migration4', direction: 'Up', status: 'Success' }, + ]) + + expect(executedUpMethods1).to.eql(['migration1', 'migration3']) + expect(executedUpMethods2).to.eql(['migration2', 'migration4']) + }) + + it('should migrate all the way down', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration2', 'migration4'], + { allowUnorderedMigrations: true } + ) + + const { results: results1 } = await migrator1.migrateToLatest() + + const [migrator2, executedUpMethods2, executedDownMethods2] = + createMigrations( + ['migration1', 'migration2', 'migration3', 'migration4'], + { allowUnorderedMigrations: true } + ) + + const { results: results2 } = await migrator2.migrateTo(NO_MIGRATIONS) + + expect(results1).to.eql([ + { migrationName: 'migration1', direction: 'Up', status: 'Success' }, + { migrationName: 'migration2', direction: 'Up', status: 'Success' }, + { migrationName: 'migration4', direction: 'Up', status: 'Success' }, + ]) + + expect(results2).to.eql([ + { + migrationName: 'migration4', + direction: 'Down', + status: 'Success', + }, + { + migrationName: 'migration2', + direction: 'Down', + status: 'Success', + }, + { + migrationName: 'migration1', + direction: 'Down', + status: 'Success', + }, + ]) + + expect(executedUpMethods1).to.eql([ + 'migration1', + 'migration2', + 'migration4', + ]) + expect(executedUpMethods2).to.eql([]) + expect(executedDownMethods2).to.eql([ + 'migration4', + 'migration2', + 'migration1', + ]) + }) + + it('should migrate down to a specific migration', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration2', 'migration3', 'migration5'], + { allowUnorderedMigrations: true } + ) + + const { results: results1 } = await migrator1.migrateTo('migration5') + + const [migrator2, executedUpMethods2, executedDownMethods2] = + createMigrations( + [ + 'migration1', + 'migration2', + 'migration3', + 'migration4', + 'migration5', + ], + { allowUnorderedMigrations: true } + ) + + const { results: results2 } = await migrator2.migrateTo('migration2') + + expect(results1).to.eql([ + { migrationName: 'migration1', direction: 'Up', status: 'Success' }, + { migrationName: 'migration2', direction: 'Up', status: 'Success' }, + { migrationName: 'migration3', direction: 'Up', status: 'Success' }, + { migrationName: 'migration5', direction: 'Up', status: 'Success' }, + ]) + + expect(results2).to.eql([ + { + migrationName: 'migration5', + direction: 'Down', + status: 'Success', + }, + { + migrationName: 'migration3', + direction: 'Down', + status: 'Success', + }, + ]) + + expect(executedUpMethods1).to.eql([ + 'migration1', + 'migration2', + 'migration3', + 'migration5', + ]) + + expect(executedUpMethods2).to.eql([]) + expect(executedDownMethods2).to.eql(['migration5', 'migration3']) + }) + }) }) describe('migrateUp', () => { @@ -399,6 +628,65 @@ for (const dialect of DIALECTS) { expect(results3).to.eql([]) expect(executedUpMethods).to.eql(['migration1', 'migration2']) }) + + it('should return an error when migrating up if a new migration is added before the last executed one', async () => { + const [migrator1, executedUpMethods1] = createMigrations([ + 'migration1', + 'migration3', + ]) + + await migrator1.migrateToLatest() + + const [migrator2, executedUpMethods2] = createMigrations([ + 'migration1', + 'migration2', + 'migration3', + 'migration4', + ]) + + const { error } = await migrator2.migrateUp() + + expect(error).to.be.an.instanceOf(Error) + expect(getMessage(error)).to.eql( + 'corrupted migrations: expected previously executed migration migration3 to be at index 1 but migration2 was found in its place. New migrations must always have a name that comes alphabetically after the last executed migration.' + ) + + expect(executedUpMethods1).to.eql(['migration1', 'migration3']) + expect(executedUpMethods2).to.eql([]) + }) + + it('should migrate up one step with allowUnorderedMigrations enabled', async () => { + const [migrator1, executedUpMethods1] = createMigrations( + ['migration1', 'migration3'], + { allowUnorderedMigrations: true } + ) + + const { results: results1 } = await migrator1.migrateToLatest() + + const [migrator2, executedUpMethods2] = createMigrations( + ['migration1', 'migration2', 'migration3', 'migration4'], + { allowUnorderedMigrations: true } + ) + + const { results: results2 } = await migrator2.migrateUp() + const { results: results3 } = await migrator2.migrateUp() + + expect(results1).to.eql([ + { migrationName: 'migration1', direction: 'Up', status: 'Success' }, + { migrationName: 'migration3', direction: 'Up', status: 'Success' }, + ]) + + expect(results2).to.eql([ + { migrationName: 'migration2', direction: 'Up', status: 'Success' }, + ]) + + expect(results3).to.eql([ + { migrationName: 'migration4', direction: 'Up', status: 'Success' }, + ]) + + expect(executedUpMethods1).to.eql(['migration1', 'migration3']) + expect(executedUpMethods2).to.eql(['migration2', 'migration4']) + }) }) describe('migrateDown', () => { @@ -431,6 +719,79 @@ for (const dialect of DIALECTS) { expect(executedUpMethods).to.eql(['migration1', 'migration2']) expect(executedDownMethods).to.eql(['migration2', 'migration1']) }) + + it('should return an error if a new migration is added before the last executed one', async () => { + const [migrator1, executedUpMethods1] = createMigrations([ + 'migration1', + 'migration3', + ]) + + await migrator1.migrateToLatest() + + const [migrator2, _executedUpMethods2, executedDownMethods2] = + createMigrations(['migration1', 'migration2', 'migration3']) + + const { error } = await migrator2.migrateDown() + + expect(error).to.be.an.instanceOf(Error) + expect(getMessage(error)).to.eql( + 'corrupted migrations: expected previously executed migration migration3 to be at index 1 but migration2 was found in its place. New migrations must always have a name that comes alphabetically after the last executed migration.' + ) + + expect(executedUpMethods1).to.eql(['migration1', 'migration3']) + expect(executedDownMethods2).to.eql([]) + }) + + it('should migrate down one step with allowUnorderedMigrations enabled', async () => { + const [migrator1, executedUpMethods1, _executedDownMethods1] = + createMigrations(['migration1', 'migration2', 'migration4'], { + allowUnorderedMigrations: true, + }) + + await migrator1.migrateToLatest() + + const [migrator2, _executedUpMethods2, executedDownMethods2] = + createMigrations( + [ + 'migration1', + 'migration2', + 'migration3', + 'migration4', + 'migration5', + ], + { allowUnorderedMigrations: true } + ) + + const { results: results1 } = await migrator2.migrateDown() + const { results: results2 } = await migrator2.migrateDown() + const { results: results3 } = await migrator2.migrateDown() + const { results: results4 } = await migrator2.migrateDown() + + expect(results1).to.eql([ + { migrationName: 'migration4', direction: 'Down', status: 'Success' }, + ]) + + expect(results2).to.eql([ + { migrationName: 'migration2', direction: 'Down', status: 'Success' }, + ]) + + expect(results3).to.eql([ + { migrationName: 'migration1', direction: 'Down', status: 'Success' }, + ]) + + expect(results4).to.eql([]) + + expect(executedUpMethods1).to.eql([ + 'migration1', + 'migration2', + 'migration4', + ]) + expect(executedDownMethods2).to.eql([ + 'migration4', + 'migration2', + 'migration1', + ]) + }) }) if (dialect === 'postgres') {