feat: friendly error when migration cycle failed

Author: rafaelcr

src/datastore/event-requests.ts

@@ -119,21 +119,28 @@ export async function* getRawEventRequests(
   }
 }
 
-export async function databaseHasData(): Promise<boolean> {
+/**
+ * Check the `pg_class` table for any data structures contained in the database. We will consider
+ * any and all results here as "data" contained in the DB, since anything that is not a completely
+ * empty DB could lead to strange errors when running the API. See:
+ * https://www.postgresql.org/docs/current/catalog-pg-class.html
+ * @returns `boolean` if the DB has data
+ */
+export async function databaseHasData(args?: {
+  ignoreMigrationTables?: boolean;
+}): Promise<boolean> {
   const sql = await connectPostgres({
     usageName: 'contains-data-check',
     pgServer: PgServer.primary,
   });
   try {
-    // Check the `pg_class` table for any data structures contained in the database. We will
-    // consider any and all results here as "data" contained in the DB, since anything that is not a
-    // completely empty DB could lead to strange errors when running the API. See:
-    // https://www.postgresql.org/docs/current/catalog-pg-class.html
+    const ignoreMigrationTables = args?.ignoreMigrationTables ?? false;
     const result = await sql<{ count: number }[]>`
       SELECT COUNT(*)
       FROM pg_class c
       JOIN pg_namespace s ON s.oid = c.relnamespace
       WHERE s.nspname = ${sql.options.connection.search_path}
+      ${ignoreMigrationTables ? sql`AND c.relname NOT LIKE 'pgmigrations%'` : sql``}
     `;
     return result.count > 0 && result[0].count > 0;
   } catch (error: any) {

src/datastore/migrations.ts

@@ -4,6 +4,7 @@ import { Client } from 'pg';
 import { APP_DIR, isDevEnv, isTestEnv, logError, logger } from '../helpers';
 import { getPgClientConfig, PgClientConfig } from './connection-legacy';
 import { connectPostgres, PgServer } from './connection';
+import { databaseHasData } from './event-requests';
 
 const MIGRATIONS_TABLE = 'pgmigrations';
 const MIGRATIONS_DIR = path.join(APP_DIR, 'migrations');
@@ -53,10 +54,14 @@ export async function runMigrations(
 export async function cycleMigrations(opts?: {
   // Bypass the NODE_ENV check when performing a "down" migration which irreversibly drops data.
   dangerousAllowDataLoss?: boolean;
+  checkForEmptyData?: boolean;
 }): Promise<void> {
   const clientConfig = getPgClientConfig({ usageName: 'cycle-migrations' });
 
   await runMigrations(clientConfig, 'down', opts);
+  if (opts?.checkForEmptyData && (await databaseHasData({ ignoreMigrationTables: true }))) {
+    throw new Error('Migration down process did not completely remove DB tables');
+  }
   await runMigrations(clientConfig, 'up', opts);
 }
 

src/event-replay/event-replay.ts

@@ -98,10 +98,14 @@ export async function importEventsFromTsv(
     await dangerousDropAllTables({ acknowledgePotentialCatastrophicConsequences: 'yes' });
   }
 
-  // This performs a "migration down" which drops the tables, then re-creates them.
-  // If there's a breaking change in the migration files, this will throw, and the pg database needs wiped manually,
-  // or the `--force` option can be used.
-  await cycleMigrations({ dangerousAllowDataLoss: true });
+  try {
+    await cycleMigrations({ dangerousAllowDataLoss: true, checkForEmptyData: true });
+  } catch (error) {
+    logger.error(error);
+    throw new Error(
+      `DB migration cycle failed, possibly due to an incompatible API version upgrade. Add --wipe-db --force or perform a manual DB wipe before importing.`
+    );
+  }
 
   // Look for the TSV's block height and determine the prunable block window.
   const tsvBlockHeight = await findTsvBlockHeight(resolvedFilePath);

src/tests-event-replay/import-export-tests.ts

@@ -57,15 +57,42 @@ describe('import/export tests', () => {
     }
   });
 
+  test('import with db wipe options', async () => {
+    // Migrate first so we have some data.
+    const clientConfig = getPgClientConfig({ usageName: 'cycle-migrations' });
+    await runMigrations(clientConfig, 'up', {});
+    await expect(
+      importEventsFromTsv('src/tests-event-replay/tsv/mocknet.tsv', 'archival', false, false)
+    ).rejects.toThrowError('contains existing data');
+
+    // Create strange table
+    await db.sql`CREATE TABLE IF NOT EXISTS test (a varchar(10))`;
+    await expect(
+      importEventsFromTsv('src/tests-event-replay/tsv/mocknet.tsv', 'archival', true, false)
+    ).rejects.toThrowError('migration cycle failed');
+
+    // Force and test
+    await expect(
+      importEventsFromTsv('src/tests-event-replay/tsv/mocknet.tsv', 'archival', true, true)
+    ).resolves.not.toThrow();
+  });
+
   test('db contains data', async () => {
     const clientConfig = getPgClientConfig({ usageName: 'cycle-migrations' });
     await runMigrations(clientConfig, 'up', {});
 
     // Having tables counts as having data as this may change across major versions.
     await expect(databaseHasData()).resolves.toBe(true);
 
+    // Dropping all tables removes everything.
     await dangerousDropAllTables({ acknowledgePotentialCatastrophicConsequences: 'yes' });
     await expect(databaseHasData()).resolves.toBe(false);
+
+    // Cycling migrations leaves the `pgmigrations` table.
+    await runMigrations(clientConfig, 'up', {});
+    await runMigrations(clientConfig, 'down', {});
+    await expect(databaseHasData()).resolves.toBe(true);
+    await expect(databaseHasData({ ignoreMigrationTables: true })).resolves.toBe(false);
   });
 
   afterEach(async () => {