misc: audit log migration + special handing

.env.migration.example

@@ -1 +1,2 @@
 DB_CONNECTION_URI=
+AUDIT_LOGS_DB_CONNECTION_URI=

backend/package.json

@@ -45,13 +45,19 @@
     "test:e2e-coverage": "vitest run --coverage -c vitest.e2e.config.ts",
     "generate:component": "tsx ./scripts/create-backend-file.ts",
     "generate:schema": "tsx ./scripts/generate-schema-types.ts",
+    "auditlog-migration:latest": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:latest",
+    "auditlog-migration:up": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:up",
+    "auditlog-migration:down": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:down",
+    "auditlog-migration:list": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:list",
+    "auditlog-migration:status": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:status",
+    "auditlog-migration:rollback": "knex --knexfile ./src/db/auditlog-knexfile.ts migrate:rollback",
     "migration:new": "tsx ./scripts/create-migration.ts",
-    "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up",
-    "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down",
-    "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list",
-    "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest",
-    "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status",
-    "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback",
+    "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up && npm run auditlog-migration:up",
+    "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down && npm run auditlog-migration:down",
+    "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list && npm run auditlog-migration:list",
+    "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest && npm run auditlog-migration:latest",
+    "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status && npm run auditlog-migration:status",
+    "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback && npm run auditlog-migration:rollback",
     "seed:new": "tsx ./scripts/create-seed-file.ts",
     "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run",
     "db:reset": "npm run migration:rollback -- --all && npm run migration:latest"

backend/src/@types/knex.d.ts

@@ -170,6 +170,9 @@ import {
   TOrgRoles,
   TOrgRolesInsert,
   TOrgRolesUpdate,
+  TPartitionedAuditLogs,
+  TPartitionedAuditLogsInsert,
+  TPartitionedAuditLogsUpdate,
   TPkiAlerts,
   TPkiAlertsInsert,
   TPkiAlertsUpdate,
@@ -715,6 +718,11 @@ declare module "knex/types/tables" {
       TAuditLogStreamsInsert,
       TAuditLogStreamsUpdate
     >;
+    [TableName.PartitionedAuditLog]: KnexOriginal.CompositeTableType<
+      TPartitionedAuditLogs,
+      TPartitionedAuditLogsInsert,
+      TPartitionedAuditLogsUpdate
+    >;
     [TableName.GitAppInstallSession]: KnexOriginal.CompositeTableType<
       TGitAppInstallSessions,
       TGitAppInstallSessionsInsert,

backend/src/db/auditlog-knexfile.ts

@@ -0,0 +1,73 @@
+// eslint-disable-next-line
+import "ts-node/register";
+
+import dotenv from "dotenv";
+import type { Knex } from "knex";
+import path from "path";
+
+// Update with your config settings. .
+dotenv.config({
+  path: path.join(__dirname, "../../../.env.migration")
+});
+dotenv.config({
+  path: path.join(__dirname, "../../../.env")
+});
+
+if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST) {
+  console.info("Dedicated audit log database not found. No further migrations necessary");
+  process.exit(0);
+}
+
+export default {
+  development: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    seeds: {
+      directory: "./seeds"
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  },
+  production: {
+    client: "postgres",
+    connection: {
+      connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii")
+          }
+        : false
+    },
+    pool: {
+      min: 2,
+      max: 10
+    },
+    migrations: {
+      tableName: "infisical_migrations"
+    }
+  }
+} as Knex.Config;

backend/src/db/index.ts

@@ -1,2 +1,2 @@
 export type { TDbClient } from "./instance";
-export { initDbConnection } from "./instance";
+export { initAuditLogDbConnection, initDbConnection } from "./instance";

backend/src/db/instance.ts

@@ -70,3 +70,45 @@ export const initDbConnection = ({
 
   return db;
 };
+
+export const initAuditLogDbConnection = ({
+  dbConnectionUri,
+  dbRootCert
+}: {
+  dbConnectionUri: string;
+  dbRootCert?: string;
+}) => {
+  // akhilmhdh: the default Knex is knex.Knex<any, any[]>. but when assigned with knex({<config>}) the value is knex.Knex<any, unknown[]>
+  // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[]
+  // eslint-disable-next-line
+  const db: Knex<any, unknown[]> = knex({
+    client: "pg",
+    connection: {
+      connectionString: dbConnectionUri,
+      host: process.env.AUDIT_LOGS_DB_HOST,
+      // @ts-expect-error I have no clue why only for the port there is a type error
+      // eslint-disable-next-line
+      port: process.env.AUDIT_LOGS_DB_PORT,
+      user: process.env.AUDIT_LOGS_DB_USER,
+      database: process.env.AUDIT_LOGS_DB_NAME,
+      password: process.env.AUDIT_LOGS_DB_PASSWORD,
+      ssl: dbRootCert
+        ? {
+            rejectUnauthorized: true,
+            ca: Buffer.from(dbRootCert, "base64").toString("ascii")
+          }
+        : false
+    }
+  });
+
+  // we add these overrides so that auditLogDb and the primary DB are interchangeable
+  db.primaryNode = () => {
+    return db;
+  };
+
+  db.replicaNode = () => {
+    return db;
+  };
+
+  return db;
+};

backend/src/db/migrations/20241003075413_partition-audit-logs.ts

@@ -0,0 +1,170 @@
+import { Knex } from "knex";
+
+import { TableName } from "../schemas";
+
+const formatPartitionDate = (date: Date) => {
+  const year = date.getFullYear();
+  const month = String(date.getMonth() + 1).padStart(2, "0");
+  const day = String(date.getDate()).padStart(2, "0");
+
+  return `${year}-${month}-${day}`;
+};
+
+const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => {
+  const startDateStr = formatPartitionDate(startDate);
+  const endDateStr = formatPartitionDate(endDate);
+
+  const partitionName = `${TableName.PartitionedAuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace(
+    /-/g,
+    ""
+  )}`;
+
+  await knex.schema.raw(
+    `CREATE TABLE ${partitionName} PARTITION OF ${TableName.PartitionedAuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')`
+  );
+};
+
+export async function up(knex: Knex): Promise<void> {
+  // prepare the existing audit log table for it to become a partition
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+    const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+    const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      // remove existing keys
+      t.dropPrimary();
+
+      if (doesOrgIdExist) {
+        t.dropForeign("orgId");
+      }
+
+      if (doesProjectIdExist) {
+        t.dropForeign("projectId");
+      }
+
+      // add normalized fields present in the partition table
+      if (!doesProjectNameExist) {
+        t.string("projectName");
+      }
+    });
+  }
+
+  // create a new partitioned table for audit logs
+  if (!(await knex.schema.hasTable(TableName.PartitionedAuditLog))) {
+    const createTableSql = knex.schema
+      .createTable(TableName.PartitionedAuditLog, (t) => {
+        t.uuid("id").defaultTo(knex.fn.uuid());
+        t.string("actor").notNullable();
+        t.jsonb("actorMetadata").notNullable();
+        t.string("ipAddress");
+        t.string("eventType").notNullable();
+        t.jsonb("eventMetadata");
+        t.string("userAgent");
+        t.string("userAgentType");
+        t.datetime("expiresAt");
+        t.timestamps(true, true, true);
+        t.uuid("orgId");
+        t.string("projectId");
+        t.string("projectName");
+        t.primary(["id", "createdAt"]);
+      })
+      .toString();
+
+    await knex.schema.raw(`
+        ${createTableSql} PARTITION BY RANGE ("createdAt");
+    `);
+
+    await knex.schema.alterTable(TableName.PartitionedAuditLog, (t) => {
+      t.index(["projectId", "createdAt"]);
+      t.index(["orgId", "createdAt"]);
+      t.index("expiresAt");
+      t.index("orgId");
+      t.index("projectId");
+    });
+
+    await knex.raw(
+      `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)`
+    );
+
+    await knex.raw(
+      `CREATE INDEX "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)`
+    );
+
+    // create default partition
+    await knex.schema.raw(
+      `CREATE TABLE ${TableName.PartitionedAuditLog}_default PARTITION OF ${TableName.PartitionedAuditLog} DEFAULT`
+    );
+
+    const nextDate = new Date();
+    nextDate.setDate(nextDate.getDate() + 1);
+    const nextDateStr = formatPartitionDate(nextDate);
+
+    // attach existing audit log table as a partition
+    await knex.schema.raw(`
+    ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old
+    CHECK ( "createdAt" < DATE '${nextDateStr}' );
+
+    ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog}
+    FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' );
+    `);
+
+    // create partition from now until end of month
+    await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1));
+
+    // create partitions 4 years ahead
+    const partitionMonths = 4 * 12;
+    const partitionPromises: Promise<void>[] = [];
+    for (let x = 1; x < partitionMonths; x += 1) {
+      partitionPromises.push(
+        createAuditLogPartition(
+          knex,
+          new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1),
+          new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1)
+        )
+      );
+    }
+
+    await Promise.all(partitionPromises);
+  }
+}
+
+export async function down(knex: Knex): Promise<void> {
+  // detach audit log from partition
+  await knex.schema.raw(`
+    ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog};
+
+    ALTER TABLE ${TableName.AuditLog} DROP CONSTRAINT audit_log_old;
+  `);
+
+  // revert audit log modifications
+  const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId");
+  const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId");
+  const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName");
+
+  if (await knex.schema.hasTable(TableName.AuditLog)) {
+    await knex.schema.alterTable(TableName.AuditLog, (t) => {
+      // we drop this first because adding to the partition results in a new primary key
+      t.dropPrimary();
+
+      // add back the original keys of the audit logs table
+      t.primary(["id"], {
+        constraintName: "audit_logs_pkey"
+      });
+
+      if (doesOrgIdExist) {
+        t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE");
+      }
+      if (doesProjectIdExist) {
+        t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE");
+      }
+
+      // remove normalized fields
+      if (doesProjectNameExist) {
+        t.dropColumn("projectName");
+      }
+    });
+  }
+
+  await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog);
+}

backend/src/db/schemas/audit-logs.ts

@@ -20,7 +20,8 @@ export const AuditLogsSchema = z.object({
   createdAt: z.date(),
   updatedAt: z.date(),
   orgId: z.string().uuid().nullable().optional(),
-  projectId: z.string().nullable().optional()
+  projectId: z.string().nullable().optional(),
+  projectName: z.string().nullable().optional()
 });
 
 export type TAuditLogs = z.infer<typeof AuditLogsSchema>;

backend/src/db/schemas/index.ts

@@ -55,6 +55,7 @@ export * from "./org-bots";
 export * from "./org-memberships";
 export * from "./org-roles";
 export * from "./organizations";
+export * from "./partitioned-audit-logs";
 export * from "./pki-alerts";
 export * from "./pki-collection-items";
 export * from "./pki-collections";

backend/src/db/schemas/models.ts

@@ -90,6 +90,7 @@ export enum TableName {
   OidcConfig = "oidc_configs",
   LdapGroupMap = "ldap_group_maps",
   AuditLog = "audit_logs",
+  PartitionedAuditLog = "partitioned_audit_logs",
   AuditLogStream = "audit_log_streams",
   GitAppInstallSession = "git_app_install_sessions",
   GitAppOrg = "git_app_org",