misc: initial setup for migration of audit logs

.env.migration.example

@@ -1 +1,2 @@
 DB_CONNECTION_URI=
+AUDIT_LOGS_DB_CONNECTION_URI=

backend/package.json

@@ -45,13 +45,19 @@
     "test:e2e-coverage": "vitest run --coverage -c vitest.e2e.config.ts",
     "generate:component": "tsx ./scripts/create-backend-file.ts",
     "generate:schema": "tsx ./scripts/generate-schema-types.ts",
+    "auditlog-migration:latest": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:latest",
+    "auditlog-migration:up": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:up",
+    "auditlog-migration:down": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:down",
+    "auditlog-migration:list": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:list",
+    "auditlog-migration:status": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:status",
+    "auditlog-migration:rollback": "knex --knexfile ./src/db/auditlog-knexfile.ts migrate:rollback",
     "migration:new": "tsx ./scripts/create-migration.ts",
-    "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up",
-    "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down",
-    "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list",
-    "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest",
-    "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status",
-    "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback",
+    "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up && npm run auditlog-migration:up",
+    "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down && npm run auditlog-migration:down",
+    "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list && npm run auditlog-migration:list",
+    "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest && npm run auditlog-migration:latest",
+    "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status && npm run auditlog-migration:status",
+    "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback && npm run auditlog-migration:rollback",
     "seed:new": "tsx ./scripts/create-seed-file.ts",
     "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
     "db:reset": "npm run migration:rollback -- --all && npm run migration:latest"

backend/src/db/auditlog-knexfile.ts

@@ -0,0 +1,73 @@
+// eslint-disable-next-line
+import "ts-node/register";
+
+import dotenv from "dotenv";
+import type { Knex } from "knex";
+import path from "path";
+
+// Update with your config settings. .
+dotenv.config({
+  path: path.join(__dirname, "../../../.env.migration")
+});
+dotenv.config({
+  path: path.join(__dirname, "../../../.env")
+});
+
+if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST) {
+  console.info("Dedicated audit log database not found. No further migrations necessary");
+  process.exit(0);
+}
+
+export default {
+  development: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    seeds: {
+      directory: "./seeds"
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  },
+  production: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  }
+} as Knex.Config;

backend/src/db/index.ts

@@ -1,2 +1,2 @@
 export type { TDbClient } from "./instance";
-export { initDbConnection } from "./instance";
+export { initAuditLogDbConnection, initDbConnection } from "./instance";

backend/src/db/instance.ts

@@ -70,3 +70,45 @@ export const initDbConnection = ({
 
   return db;
 };
+
+export const initAuditLogDbConnection = ({
+  dbConnectionUri,
+  dbRootCert
+}: {
+  dbConnectionUri: string;
+  dbRootCert?: string;
+}) => {
+  // akhilmhdh: the default Knex is knex.Knex<any, any[]>. but when assigned with knex({<config>}) the value is knex.Knex<any, unknown[]>
+  // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[]
+  // eslint-disable-next-line
+  const db: Knex<any, unknown[]> = knex({
+    client: "pg",
+    connection: {
+      connectionString: dbConnectionUri,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      // @ts-expect-error I have no clue why only for the port there is a type error
+      // eslint-disable-next-line
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: dbRootCert
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(dbRootCert, "base64").toString("ascii")
+          }
+        : false
+    }
+  });
+
+  // we add these overrides so that auditLogDb and the primary DB are interchangeable
+  db.primaryNode = () => {
+    return db;
+  };
+
+  db.replicaNode = () => {
+    return db;
+  };
+
+  return db;
+};

backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts

@@ -0,0 +1,53 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesTableExist = await knex.schema.hasTable(TableName.AuditLog);
+
+  const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+  if (doesTableExist) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      // remove existing FKs
+      if (doesOrgIdExist) {
+        t.dropForeign("orgId");
+      }
+
+      if (doesProjectIdExist) {
+        t.dropForeign("projectId");
+      }
+
+      // add normalized fields necessary after FK removal
+      if (!doesProjectNameExist) {
+        t.string("projectName");
+      }
+    });
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesTableExist = await knex.schema.hasTable(TableName.AuditLog);
+  const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+  if (doesTableExist) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      // add back FKs
+      if (doesOrgIdExist) {
+        t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      }
+      if (doesProjectIdExist) {
+        t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      }
+
+      // remove normalized fields
+      if (doesProjectNameExist) {
+        t.dropColumn("projectName");
+      }
+    });
+  }
+}

backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts

@@ -0,0 +1,15 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+export async function up(knex: Knex): Promise<void> {
+  if (await knex.schema.hasColumn(TableName.AuditLog, "actorMetadata")) {
+    await knex.raw(
+      `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)`
+    );
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw(`DROP INDEX IF EXISTS "audit_logs_actorMetadata_idx"`);
+}

backend/src/db/schemas/audit-logs.ts

@@ -20,7 +20,8 @@ export const AuditLogsSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   orgId: z.string().uuid().nullable().optional(),
-  projectId: z.string().nullable().optional()
+  projectId: z.string().nullable().optional(),
+  projectName: z.string().nullable().optional()
 });
 
 export type TAuditLogs = z.infer<typeof AuditLogsSchema>;

backend/src/ee/services/audit-log/audit-log-dal.ts

@@ -1,7 +1,7 @@
 import { Knex } from "knex";
 
 import { TDbClient } from "@app/db";
-import { AuditLogsSchema, TableName } from "@app/db/schemas";
+import { TableName } from "@app/db/schemas";
 import { DatabaseError } from "@app/lib/errors";
 import { ormify, selectAllTableCols } from "@app/lib/knex";
 import { logger } from "@app/lib/logger";
@@ -55,11 +55,10 @@ export const auditLogDALFactory = (db: TDbClient) => {
     try {
       // Find statements
       const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog)
-        .leftJoin(TableName.Project, `${TableName.AuditLog}.projectId`, `${TableName.Project}.id`)
         // eslint-disable-next-line func-names
         .where(function () {
           if (orgId) {
-            void this.where(`${TableName.Project}.orgId`, orgId).orWhere(`${TableName.AuditLog}.orgId`, orgId);
+            void this.where(`${TableName.AuditLog}.orgId`, orgId);
           } else if (projectId) {
             void this.where(`${TableName.AuditLog}.projectId`, projectId);
           }
@@ -72,10 +71,6 @@ export const auditLogDALFactory = (db: TDbClient) => {
       // Select statements
       void sqlQuery
         .select(selectAllTableCols(TableName.AuditLog))
-        .select(
-          db.ref("name").withSchema(TableName.Project).as("projectName"),
-          db.ref("slug").withSchema(TableName.Project).as("projectSlug")
-        )
         .limit(limit)
         .offset(offset)
         .orderBy(`${TableName.AuditLog}.createdAt`, "desc");
@@ -111,21 +106,7 @@ export const auditLogDALFactory = (db: TDbClient) => {
       }
       const docs = await sqlQuery;
 
-      return docs.map((doc) => {
-        // Our type system refuses to acknowledge that the project name and slug are present in the doc, due to the disjointed query structure above.
-        // This is a quick and dirty way to get around the types.
-        const projectDoc = doc as unknown as { projectName: string; projectSlug: string };
-
-        return {
-          ...AuditLogsSchema.parse(doc),
-          ...(projectDoc?.projectSlug && {
-            project: {
-              name: projectDoc.projectName,
-              slug: projectDoc.projectSlug
-            }
-          })
-        };
-      });
+      return docs;
     } catch (error) {
       throw new DatabaseError({ error });
     }
@@ -148,6 +129,7 @@ export const auditLogDALFactory = (db: TDbClient) => {
           .where("expiresAt", "<", today)
           .select("id")
           .limit(AUDIT_LOG_PRUNE_BATCH_SIZE);
+
         // eslint-disable-next-line no-await-in-loop
         deletedAuditLogIds = await (tx || db)(TableName.AuditLog)
           .whereIn("id", findExpiredLogSubQuery)

backend/src/lib/config/env.ts

@@ -34,6 +34,12 @@ const envSchema = z
     DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default(
       `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`
     ),
+    AUDIT_LOGS_DB_CONNECTION_URI: zpStr(
+      z.string().describe("Postgres database connection string for Audit logs").optional()
+    ),
+    AUDIT_LOGS_DB_ROOT_CERT: zpStr(
+      z.string().describe("Postgres database base64-encoded CA cert for Audit logs").optional()
+    ),
     MAX_LEASE_LIMIT: z.coerce.number().default(10000),
     DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()),
     DB_HOST: zpStr(z.string().describe("Postgres database host").optional()),

backend/src/main.ts

@@ -1,7 +1,7 @@
 import dotenv from "dotenv";
 import path from "path";
 
-import { initDbConnection } from "./db";
+import { initAuditLogDbConnection, initDbConnection } from "./db";
 import { keyStoreFactory } from "./keystore/keystore";
 import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env";
 import { isMigrationMode } from "./lib/fn";
@@ -25,6 +25,13 @@ const run = async () => {
     }))
   });
 
+  const auditLogDb = appCfg.AUDIT_LOGS_DB_CONNECTION_URI
+    ? initAuditLogDbConnection({
+        dbConnectionUri: appCfg.AUDIT_LOGS_DB_CONNECTION_URI,
+        dbRootCert: appCfg.AUDIT_LOGS_DB_ROOT_CERT
+      })
+    : undefined;
+
   // Case: App is running in packaged mode (binary), and migration mode is enabled.
   // Run the migrations and exit the process after completion.
   if (IS_PACKAGED && isMigrationMode()) {
@@ -46,7 +53,7 @@ const run = async () => {
   const queue = queueServiceFactory(appCfg.REDIS_URL);
   const keyStore = keyStoreFactory(appCfg.REDIS_URL);
 
-  const server = await main({ db, smtp, logger, queue, keyStore });
+  const server = await main({ db, auditLogDb, smtp, logger, queue, keyStore });
   const bootstrap = await bootstrapCheck({ db });
 
   // eslint-disable-next-line

backend/src/server/app.ts

@@ -30,6 +30,7 @@ import { fastifySwagger } from "./plugins/swagger";
 import { registerRoutes } from "./routes";
 
 type TMain = {
+  auditLogDb?: Knex;
   db: Knex;
   smtp: TSmtpService;
   logger?: Logger;
@@ -38,7 +39,7 @@ type TMain = {
 };
 
 // Run the server!
-export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
+export const main = async ({ db, auditLogDb, smtp, logger, queue, keyStore }: TMain) => {
   const appCfg = getConfig();
   const server = fastify({
     logger: appCfg.NODE_ENV === "test" ? false : logger,
@@ -94,7 +95,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => {
 
     await server.register(maintenanceMode);
 
-    await server.register(registerRoutes, { smtp, queue, db, keyStore });
+    await server.register(registerRoutes, { smtp, queue, db, auditLogDb, keyStore });
 
     if (appCfg.isProductionMode) {
       await server.register(registerExternalNextjs, {

backend/src/server/routes/index.ts

@@ -213,11 +213,12 @@ import { registerV3Routes } from "./v3";
 export const registerRoutes = async (
   server: FastifyZodProvider,
   {
+    auditLogDb,
     db,
     smtp: smtpService,
     queue: queueService,
     keyStore
-  }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory }
+  }: { auditLogDb?: Knex; db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory }
 ) => {
   const appCfg = getConfig();
   if (!appCfg.DISABLE_SECRET_SCANNING) {
@@ -282,7 +283,7 @@ export const registerRoutes = async (
   const identityOidcAuthDAL = identityOidcAuthDALFactory(db);
   const identityAzureAuthDAL = identityAzureAuthDALFactory(db);
 
-  const auditLogDAL = auditLogDALFactory(db);
+  const auditLogDAL = auditLogDALFactory(auditLogDb ?? db);
   const auditLogStreamDAL = auditLogStreamDALFactory(db);
   const trustedIpDAL = trustedIpDALFactory(db);
   const telemetryDAL = telemetryDALFactory(db);