From 86cb51364ac61b21729e770bd34dab29e4c84974 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 2 Oct 2024 22:30:07 +0800 Subject: [PATCH 01/15] misc: initial setup for migration of audit logs --- .env.migration.example | 1 + backend/package.json | 18 +++-- backend/src/db/auditlog-knexfile.ts | 73 +++++++++++++++++++ backend/src/db/index.ts | 2 +- backend/src/db/instance.ts | 42 +++++++++++ .../20241002092243_audit-log-drop-fk.ts | 53 ++++++++++++++ ...1002110531_add-audit-log-metadata-index.ts | 15 ++++ backend/src/db/schemas/audit-logs.ts | 3 +- .../ee/services/audit-log/audit-log-dal.ts | 26 +------ backend/src/lib/config/env.ts | 6 ++ backend/src/main.ts | 11 ++- backend/src/server/app.ts | 5 +- backend/src/server/routes/index.ts | 5 +- .../server/routes/v1/organization-router.ts | 7 +- frontend/src/hooks/api/auditLogs/types.tsx | 5 +- .../AuditLogsPage/components/LogsTableRow.tsx | 2 +- 16 files changed, 227 insertions(+), 47 deletions(-) create mode 100644 backend/src/db/auditlog-knexfile.ts create mode 100644 backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts create mode 100644 backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts diff --git a/.env.migration.example b/.env.migration.example index 4d1c8f9ef5..2c5f5b9570 100644 --- a/.env.migration.example +++ b/.env.migration.example @@ -1 +1,2 @@ DB_CONNECTION_URI= +AUDIT_LOGS_DB_CONNECTION_URI= diff --git a/backend/package.json b/backend/package.json index 217817edf4..181a3a8684 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,13 +45,19 @@ "test:e2e-coverage": "vitest run --coverage -c vitest.e2e.config.ts", "generate:component": "tsx ./scripts/create-backend-file.ts", "generate:schema": "tsx ./scripts/generate-schema-types.ts", + "auditlog-migration:latest": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:latest", + "auditlog-migration:up": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:up", + "auditlog-migration:down": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:down", + "auditlog-migration:list": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:list", + "auditlog-migration:status": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:status", + "auditlog-migration:rollback": "knex --knexfile ./src/db/auditlog-knexfile.ts migrate:rollback", "migration:new": "tsx ./scripts/create-migration.ts", - "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up", - "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down", - "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list", - "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest", - "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status", - "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback", + "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up && npm run auditlog-migration:up", + "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down && npm run auditlog-migration:down", + "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list && npm run auditlog-migration:list", + "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest && npm run auditlog-migration:latest", + "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status && npm run auditlog-migration:status", + "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback && npm run auditlog-migration:rollback", "seed:new": "tsx ./scripts/create-seed-file.ts", "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run", "db:reset": "npm run migration:rollback -- --all && npm run migration:latest" diff --git a/backend/src/db/auditlog-knexfile.ts b/backend/src/db/auditlog-knexfile.ts new file mode 100644 index 0000000000..f6b84d4534 --- /dev/null +++ b/backend/src/db/auditlog-knexfile.ts @@ -0,0 +1,73 @@ +// eslint-disable-next-line +import "ts-node/register"; + +import dotenv from "dotenv"; +import type { Knex } from "knex"; +import path from "path"; + +// Update with your config settings. . +dotenv.config({ + path: path.join(__dirname, "../../../.env.migration") +}); +dotenv.config({ + path: path.join(__dirname, "../../../.env") +}); + +if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST) { + console.info("Dedicated audit log database not found. No further migrations necessary"); + process.exit(0); +} + +export default { + development: { + client: "postgres", + connection: { + connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI, + host: process.env.AUDIT_LOGS_DB_HOST, + port: process.env.AUDIT_LOGS_DB_PORT, + user: process.env.AUDIT_LOGS_DB_USER, + database: process.env.AUDIT_LOGS_DB_NAME, + password: process.env.AUDIT_LOGS_DB_PASSWORD, + ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT + ? { + rejectUnauthorized: true, + ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii") + } + : false + }, + pool: { + min: 2, + max: 10 + }, + seeds: { + directory: "./seeds" + }, + migrations: { + tableName: "infisical_migrations" + } + }, + production: { + client: "postgres", + connection: { + connectionString: process.env.AUDIT_LOGS_DB_CONNECTION_URI, + host: process.env.AUDIT_LOGS_DB_HOST, + port: process.env.AUDIT_LOGS_DB_PORT, + user: process.env.AUDIT_LOGS_DB_USER, + database: process.env.AUDIT_LOGS_DB_NAME, + password: process.env.AUDIT_LOGS_DB_PASSWORD, + ssl: process.env.AUDIT_LOGS_DB_ROOT_CERT + ? { + rejectUnauthorized: true, + ca: Buffer.from(process.env.AUDIT_LOGS_DB_ROOT_CERT, "base64").toString("ascii") + } + : false + }, + pool: { + min: 2, + max: 10 + }, + migrations: { + tableName: "infisical_migrations" + } + } +} as Knex.Config; diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index 75992e2c69..abebdf65a8 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -1,2 +1,2 @@ export type { TDbClient } from "./instance"; -export { initDbConnection } from "./instance"; +export { initAuditLogDbConnection, initDbConnection } from "./instance"; diff --git a/backend/src/db/instance.ts b/backend/src/db/instance.ts index f6162ad9cc..d4a2a5b2ca 100644 --- a/backend/src/db/instance.ts +++ b/backend/src/db/instance.ts @@ -70,3 +70,45 @@ export const initDbConnection = ({ return db; }; + +export const initAuditLogDbConnection = ({ + dbConnectionUri, + dbRootCert +}: { + dbConnectionUri: string; + dbRootCert?: string; +}) => { + // akhilmhdh: the default Knex is knex.Knex. but when assigned with knex({}) the value is knex.Knex + // this was causing issue with files like `snapshot-dal` `findRecursivelySnapshots` this i am explicitly putting the any and unknown[] + // eslint-disable-next-line + const db: Knex = knex({ + client: "pg", + connection: { + connectionString: dbConnectionUri, + host: process.env.AUDIT_LOGS_DB_HOST, + // @ts-expect-error I have no clue why only for the port there is a type error + // eslint-disable-next-line + port: process.env.AUDIT_LOGS_DB_PORT, + user: process.env.AUDIT_LOGS_DB_USER, + database: process.env.AUDIT_LOGS_DB_NAME, + password: process.env.AUDIT_LOGS_DB_PASSWORD, + ssl: dbRootCert + ? { + rejectUnauthorized: true, + ca: Buffer.from(dbRootCert, "base64").toString("ascii") + } + : false + } + }); + + // we add these overrides so that auditLogDb and the primary DB are interchangeable + db.primaryNode = () => { + return db; + }; + + db.replicaNode = () => { + return db; + }; + + return db; +}; diff --git a/backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts b/backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts new file mode 100644 index 0000000000..b2e1a62810 --- /dev/null +++ b/backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts @@ -0,0 +1,53 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesTableExist = await knex.schema.hasTable(TableName.AuditLog); + + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + if (doesTableExist) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // remove existing FKs + if (doesOrgIdExist) { + t.dropForeign("orgId"); + } + + if (doesProjectIdExist) { + t.dropForeign("projectId"); + } + + // add normalized fields necessary after FK removal + if (!doesProjectNameExist) { + t.string("projectName"); + } + }); + } +} + +export async function down(knex: Knex): Promise { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesTableExist = await knex.schema.hasTable(TableName.AuditLog); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + if (doesTableExist) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // add back FKs + if (doesOrgIdExist) { + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + } + if (doesProjectIdExist) { + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + } + + // remove normalized fields + if (doesProjectNameExist) { + t.dropColumn("projectName"); + } + }); + } +} diff --git a/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts b/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts new file mode 100644 index 0000000000..d07211a90f --- /dev/null +++ b/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts @@ -0,0 +1,15 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasColumn(TableName.AuditLog, "actorMetadata")) { + await knex.raw( + `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)` + ); + } +} + +export async function down(knex: Knex): Promise { + await knex.raw(`DROP INDEX IF EXISTS "audit_logs_actorMetadata_idx"`); +} diff --git a/backend/src/db/schemas/audit-logs.ts b/backend/src/db/schemas/audit-logs.ts index b8906698b6..d1c239724c 100644 --- a/backend/src/db/schemas/audit-logs.ts +++ b/backend/src/db/schemas/audit-logs.ts @@ -20,7 +20,8 @@ export const AuditLogsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), orgId: z.string().uuid().nullable().optional(), - projectId: z.string().nullable().optional() + projectId: z.string().nullable().optional(), + projectName: z.string().nullable().optional() }); export type TAuditLogs = z.infer; diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 5e5e6872ba..4398486e1d 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -1,7 +1,7 @@ import { Knex } from "knex"; import { TDbClient } from "@app/db"; -import { AuditLogsSchema, TableName } from "@app/db/schemas"; +import { TableName } from "@app/db/schemas"; import { DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; @@ -55,11 +55,10 @@ export const auditLogDALFactory = (db: TDbClient) => { try { // Find statements const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) - .leftJoin(TableName.Project, `${TableName.AuditLog}.projectId`, `${TableName.Project}.id`) // eslint-disable-next-line func-names .where(function () { if (orgId) { - void this.where(`${TableName.Project}.orgId`, orgId).orWhere(`${TableName.AuditLog}.orgId`, orgId); + void this.where(`${TableName.AuditLog}.orgId`, orgId); } else if (projectId) { void this.where(`${TableName.AuditLog}.projectId`, projectId); } @@ -72,10 +71,6 @@ export const auditLogDALFactory = (db: TDbClient) => { // Select statements void sqlQuery .select(selectAllTableCols(TableName.AuditLog)) - .select( - db.ref("name").withSchema(TableName.Project).as("projectName"), - db.ref("slug").withSchema(TableName.Project).as("projectSlug") - ) .limit(limit) .offset(offset) .orderBy(`${TableName.AuditLog}.createdAt`, "desc"); @@ -111,21 +106,7 @@ export const auditLogDALFactory = (db: TDbClient) => { } const docs = await sqlQuery; - return docs.map((doc) => { - // Our type system refuses to acknowledge that the project name and slug are present in the doc, due to the disjointed query structure above. - // This is a quick and dirty way to get around the types. - const projectDoc = doc as unknown as { projectName: string; projectSlug: string }; - - return { - ...AuditLogsSchema.parse(doc), - ...(projectDoc?.projectSlug && { - project: { - name: projectDoc.projectName, - slug: projectDoc.projectSlug - } - }) - }; - }); + return docs; } catch (error) { throw new DatabaseError({ error }); } @@ -148,6 +129,7 @@ export const auditLogDALFactory = (db: TDbClient) => { .where("expiresAt", "<", today) .select("id") .limit(AUDIT_LOG_PRUNE_BATCH_SIZE); + // eslint-disable-next-line no-await-in-loop deletedAuditLogIds = await (tx || db)(TableName.AuditLog) .whereIn("id", findExpiredLogSubQuery) diff --git a/backend/src/lib/config/env.ts b/backend/src/lib/config/env.ts index 06b60f27e0..b047de7558 100644 --- a/backend/src/lib/config/env.ts +++ b/backend/src/lib/config/env.ts @@ -34,6 +34,12 @@ const envSchema = z DB_CONNECTION_URI: zpStr(z.string().describe("Postgres database connection string")).default( `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}` ), + AUDIT_LOGS_DB_CONNECTION_URI: zpStr( + z.string().describe("Postgres database connection string for Audit logs").optional() + ), + AUDIT_LOGS_DB_ROOT_CERT: zpStr( + z.string().describe("Postgres database base64-encoded CA cert for Audit logs").optional() + ), MAX_LEASE_LIMIT: z.coerce.number().default(10000), DB_ROOT_CERT: zpStr(z.string().describe("Postgres database base64-encoded CA cert").optional()), DB_HOST: zpStr(z.string().describe("Postgres database host").optional()), diff --git a/backend/src/main.ts b/backend/src/main.ts index a1c5dfd09c..f71a1fe95a 100644 --- a/backend/src/main.ts +++ b/backend/src/main.ts @@ -1,7 +1,7 @@ import dotenv from "dotenv"; import path from "path"; -import { initDbConnection } from "./db"; +import { initAuditLogDbConnection, initDbConnection } from "./db"; import { keyStoreFactory } from "./keystore/keystore"; import { formatSmtpConfig, initEnvConfig, IS_PACKAGED } from "./lib/config/env"; import { isMigrationMode } from "./lib/fn"; @@ -25,6 +25,13 @@ const run = async () => { })) }); + const auditLogDb = appCfg.AUDIT_LOGS_DB_CONNECTION_URI + ? initAuditLogDbConnection({ + dbConnectionUri: appCfg.AUDIT_LOGS_DB_CONNECTION_URI, + dbRootCert: appCfg.AUDIT_LOGS_DB_ROOT_CERT + }) + : undefined; + // Case: App is running in packaged mode (binary), and migration mode is enabled. // Run the migrations and exit the process after completion. if (IS_PACKAGED && isMigrationMode()) { @@ -46,7 +53,7 @@ const run = async () => { const queue = queueServiceFactory(appCfg.REDIS_URL); const keyStore = keyStoreFactory(appCfg.REDIS_URL); - const server = await main({ db, smtp, logger, queue, keyStore }); + const server = await main({ db, auditLogDb, smtp, logger, queue, keyStore }); const bootstrap = await bootstrapCheck({ db }); // eslint-disable-next-line diff --git a/backend/src/server/app.ts b/backend/src/server/app.ts index 8456eed8d8..b768d0db5b 100644 --- a/backend/src/server/app.ts +++ b/backend/src/server/app.ts @@ -30,6 +30,7 @@ import { fastifySwagger } from "./plugins/swagger"; import { registerRoutes } from "./routes"; type TMain = { + auditLogDb?: Knex; db: Knex; smtp: TSmtpService; logger?: Logger; @@ -38,7 +39,7 @@ type TMain = { }; // Run the server! -export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { +export const main = async ({ db, auditLogDb, smtp, logger, queue, keyStore }: TMain) => { const appCfg = getConfig(); const server = fastify({ logger: appCfg.NODE_ENV === "test" ? false : logger, @@ -94,7 +95,7 @@ export const main = async ({ db, smtp, logger, queue, keyStore }: TMain) => { await server.register(maintenanceMode); - await server.register(registerRoutes, { smtp, queue, db, keyStore }); + await server.register(registerRoutes, { smtp, queue, db, auditLogDb, keyStore }); if (appCfg.isProductionMode) { await server.register(registerExternalNextjs, { diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts index addb46b937..30c56faa40 100644 --- a/backend/src/server/routes/index.ts +++ b/backend/src/server/routes/index.ts @@ -213,11 +213,12 @@ import { registerV3Routes } from "./v3"; export const registerRoutes = async ( server: FastifyZodProvider, { + auditLogDb, db, smtp: smtpService, queue: queueService, keyStore - }: { db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory } + }: { auditLogDb?: Knex; db: Knex; smtp: TSmtpService; queue: TQueueServiceFactory; keyStore: TKeyStoreFactory } ) => { const appCfg = getConfig(); if (!appCfg.DISABLE_SECRET_SCANNING) { @@ -282,7 +283,7 @@ export const registerRoutes = async ( const identityOidcAuthDAL = identityOidcAuthDALFactory(db); const identityAzureAuthDAL = identityAzureAuthDALFactory(db); - const auditLogDAL = auditLogDALFactory(db); + const auditLogDAL = auditLogDALFactory(auditLogDb ?? db); const auditLogStreamDAL = auditLogStreamDALFactory(db); const trustedIpDAL = trustedIpDALFactory(db); const telemetryDAL = telemetryDALFactory(db); diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index b113b9f9d2..439950f1cd 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -125,12 +125,6 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { }) .merge( z.object({ - project: z - .object({ - name: z.string(), - slug: z.string() - }) - .optional(), event: z.object({ type: z.string(), metadata: z.any() @@ -168,6 +162,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { actorAuthMethod: req.permission.authMethod, actor: req.permission.type }); + return { auditLogs }; } }); diff --git a/frontend/src/hooks/api/auditLogs/types.tsx b/frontend/src/hooks/api/auditLogs/types.tsx index 764d0b2a53..76eb585177 100644 --- a/frontend/src/hooks/api/auditLogs/types.tsx +++ b/frontend/src/hooks/api/auditLogs/types.tsx @@ -886,8 +886,5 @@ export type AuditLog = { userAgentType: UserAgentType; createdAt: string; updatedAt: string; - project?: { - name: string; - slug: string; - }; + projectName?: string; }; diff --git a/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx b/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx index 014301c567..e4620a477e 100644 --- a/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx +++ b/frontend/src/views/Org/AuditLogsPage/components/LogsTableRow.tsx @@ -573,7 +573,7 @@ export const LogsTableRow = ({ auditLog, isOrgAuditLogs, showActorColumn }: Prop {formatDate(auditLog.createdAt)} {`${eventToNameMap[auditLog.event.type]}`} - {isOrgAuditLogs && {auditLog?.project?.name ?? "N/A"}} + {isOrgAuditLogs && {auditLog?.projectName ?? "N/A"}} {showActorColumn && renderActor(auditLog.actor)} {renderSource()} {renderMetadata(auditLog.event)} From dc8c3a30bd01235709e081fae4887350da6f83a0 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Wed, 2 Oct 2024 22:40:33 +0800 Subject: [PATCH 02/15] misc: added project name to publish log --- backend/src/ee/services/audit-log/audit-log-queue.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/ee/services/audit-log/audit-log-queue.ts b/backend/src/ee/services/audit-log/audit-log-queue.ts index 3fde40c8eb..83a2fafa6f 100644 --- a/backend/src/ee/services/audit-log/audit-log-queue.ts +++ b/backend/src/ee/services/audit-log/audit-log-queue.ts @@ -74,6 +74,7 @@ export const auditLogQueueServiceFactory = ({ actorMetadata: actor.metadata, userAgent, projectId, + projectName: project?.name, ipAddress, orgId, eventType: event.type, From 9b1615f2fb00cc4637e91937aea93308d81d9dee Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 3 Oct 2024 00:31:23 +0800 Subject: [PATCH 03/15] misc: migrated json filters to new op --- .../20241002110531_add-audit-log-metadata-index.ts | 6 ++++++ backend/src/ee/services/audit-log/audit-log-dal.ts | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts b/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts index d07211a90f..96854693a6 100644 --- a/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts +++ b/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts @@ -8,8 +8,14 @@ export async function up(knex: Knex): Promise { `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)` ); } + if (await knex.schema.hasColumn(TableName.AuditLog, "eventMetadata")) { + await knex.raw( + `CREATE INDEX "audit_logs_eventMetadata_idx" ON ${TableName.AuditLog} USING gin("eventMetadata" jsonb_path_ops)` + ); + } } export async function down(knex: Knex): Promise { await knex.raw(`DROP INDEX IF EXISTS "audit_logs_actorMetadata_idx"`); + await knex.raw(`DROP INDEX IF EXISTS "audit_logs_eventMetadata_idx"`); } diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 4398486e1d..e528bfa44d 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -77,13 +77,13 @@ export const auditLogDALFactory = (db: TDbClient) => { // Special case: Filter by actor ID if (actorId) { - void sqlQuery.whereRaw(`"actorMetadata"->>'userId' = ?`, [actorId]); + void sqlQuery.whereRaw(`"actorMetadata" @> jsonb_build_object('userId', ?::text)`, [actorId]); } // Special case: Filter by key/value pairs in eventMetadata field if (eventMetadata && Object.keys(eventMetadata).length) { Object.entries(eventMetadata).forEach(([key, value]) => { - void sqlQuery.whereRaw(`"eventMetadata"->>'${key}' = ?`, [value]); + void sqlQuery.whereRaw(`"eventMetadata" @> jsonb_build_object(?::text, ?::text)`, [key, value]); }); } @@ -104,6 +104,7 @@ export const auditLogDALFactory = (db: TDbClient) => { if (endDate) { void sqlQuery.where(`${TableName.AuditLog}.createdAt`, "<=", endDate); } + const docs = await sqlQuery; return docs; From 2d0433b96c2f39764dc9cd4b4bf33a3a454bb5ab Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Thu, 3 Oct 2024 22:47:16 +0800 Subject: [PATCH 04/15] misc: initial setup for audit log partition: --- backend/src/@types/knex.d.ts | 8 + .../20241002092243_audit-log-drop-fk.ts | 53 ------ ...1002110531_add-audit-log-metadata-index.ts | 21 --- .../20241003075413_partition-audit-logs.ts | 161 ++++++++++++++++++ backend/src/db/schemas/index.ts | 1 + backend/src/db/schemas/models.ts | 1 + .../src/db/schemas/partitioned-audit-logs.ts | 29 ++++ .../ee/services/audit-log/audit-log-dal.ts | 20 +-- 8 files changed, 210 insertions(+), 84 deletions(-) delete mode 100644 backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts delete mode 100644 backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts create mode 100644 backend/src/db/migrations/20241003075413_partition-audit-logs.ts create mode 100644 backend/src/db/schemas/partitioned-audit-logs.ts diff --git a/backend/src/@types/knex.d.ts b/backend/src/@types/knex.d.ts index 6249152762..d40a4f1489 100644 --- a/backend/src/@types/knex.d.ts +++ b/backend/src/@types/knex.d.ts @@ -170,6 +170,9 @@ import { TOrgRoles, TOrgRolesInsert, TOrgRolesUpdate, + TPartitionedAuditLogs, + TPartitionedAuditLogsInsert, + TPartitionedAuditLogsUpdate, TPkiAlerts, TPkiAlertsInsert, TPkiAlertsUpdate, @@ -715,6 +718,11 @@ declare module "knex/types/tables" { TAuditLogStreamsInsert, TAuditLogStreamsUpdate >; + [TableName.PartitionedAuditLog]: KnexOriginal.CompositeTableType< + TPartitionedAuditLogs, + TPartitionedAuditLogsInsert, + TPartitionedAuditLogsUpdate + >; [TableName.GitAppInstallSession]: KnexOriginal.CompositeTableType< TGitAppInstallSessions, TGitAppInstallSessionsInsert, diff --git a/backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts b/backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts deleted file mode 100644 index b2e1a62810..0000000000 --- a/backend/src/db/migrations/20241002092243_audit-log-drop-fk.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Knex } from "knex"; - -import { TableName } from "../schemas"; - -export async function up(knex: Knex): Promise { - const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); - const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesTableExist = await knex.schema.hasTable(TableName.AuditLog); - - const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - - if (doesTableExist) { - await knex.schema.alterTable(TableName.AuditLog, (t) => { - // remove existing FKs - if (doesOrgIdExist) { - t.dropForeign("orgId"); - } - - if (doesProjectIdExist) { - t.dropForeign("projectId"); - } - - // add normalized fields necessary after FK removal - if (!doesProjectNameExist) { - t.string("projectName"); - } - }); - } -} - -export async function down(knex: Knex): Promise { - const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); - const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesTableExist = await knex.schema.hasTable(TableName.AuditLog); - const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - - if (doesTableExist) { - await knex.schema.alterTable(TableName.AuditLog, (t) => { - // add back FKs - if (doesOrgIdExist) { - t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); - } - if (doesProjectIdExist) { - t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); - } - - // remove normalized fields - if (doesProjectNameExist) { - t.dropColumn("projectName"); - } - }); - } -} diff --git a/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts b/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts deleted file mode 100644 index 96854693a6..0000000000 --- a/backend/src/db/migrations/20241002110531_add-audit-log-metadata-index.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Knex } from "knex"; - -import { TableName } from "../schemas"; - -export async function up(knex: Knex): Promise { - if (await knex.schema.hasColumn(TableName.AuditLog, "actorMetadata")) { - await knex.raw( - `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.AuditLog} USING gin("actorMetadata" jsonb_path_ops)` - ); - } - if (await knex.schema.hasColumn(TableName.AuditLog, "eventMetadata")) { - await knex.raw( - `CREATE INDEX "audit_logs_eventMetadata_idx" ON ${TableName.AuditLog} USING gin("eventMetadata" jsonb_path_ops)` - ); - } -} - -export async function down(knex: Knex): Promise { - await knex.raw(`DROP INDEX IF EXISTS "audit_logs_actorMetadata_idx"`); - await knex.raw(`DROP INDEX IF EXISTS "audit_logs_eventMetadata_idx"`); -} diff --git a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts new file mode 100644 index 0000000000..1b18e9e3d1 --- /dev/null +++ b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts @@ -0,0 +1,161 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +const formatDateToYYYYMMDD = (date: Date) => { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); // getMonth() returns 0-based month, so add 1 + const day = String(date.getDate()).padStart(2, "0"); + + return `${year}-${month}-${day}`; +}; + +const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => { + const startDateStr = formatDateToYYYYMMDD(startDate); + const endDateStr = formatDateToYYYYMMDD(endDate); + + const partitionName = `${TableName.PartitionedAuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace( + /-/g, + "" + )}`; + + await knex.schema.raw( + `CREATE TABLE ${partitionName} PARTITION OF ${TableName.PartitionedAuditLog} FOR VALUES FROM ('${startDateStr}') TO ('${endDateStr}')` + ); +}; + +export async function up(knex: Knex): Promise { + // prepare the existing audit log table for it to become a partition + if (await knex.schema.hasTable(TableName.AuditLog)) { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // remove existing keys + t.dropPrimary(); + + if (doesOrgIdExist) { + t.dropForeign("orgId"); + } + + if (doesProjectIdExist) { + t.dropForeign("projectId"); + } + + // add normalized fields present in the partition table + if (!doesProjectNameExist) { + t.string("projectName"); + } + }); + } + + // create a new partitioned table for audit logs + if (!(await knex.schema.hasTable(TableName.PartitionedAuditLog))) { + const createTableSql = knex.schema + .createTable(TableName.PartitionedAuditLog, (t) => { + t.uuid("id").defaultTo(knex.fn.uuid()); + t.string("actor").notNullable(); + t.jsonb("actorMetadata").notNullable(); + t.string("ipAddress"); + t.string("eventType").notNullable(); + t.jsonb("eventMetadata"); + t.string("userAgent"); + t.string("userAgentType"); + t.datetime("expiresAt"); + t.timestamps(true, true, true); + t.uuid("orgId"); + t.string("projectId"); + t.string("projectName"); + t.primary(["id", "createdAt"]); + }) + .toString(); + + await knex.schema.raw(` + ${createTableSql} PARTITION BY RANGE ("createdAt"); + `); + + // add indices + await knex.raw( + `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` + ); + + await knex.raw( + `CREATE INDEX "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` + ); + + // create default partition + await knex.schema.raw( + `CREATE TABLE ${TableName.PartitionedAuditLog}_default PARTITION OF ${TableName.PartitionedAuditLog} DEFAULT` + ); + + const nextDate = new Date(); + nextDate.setDate(nextDate.getDate() + 1); + const nextDateStr = formatDateToYYYYMMDD(nextDate); + + // attach existing audit log table as a partition + await knex.schema.raw(` + ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old + CHECK ( "createdAt" < DATE '${nextDateStr}' ); + + ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog} + FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); + `); + + // create partitions 3 months ahead + await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1, 1)); + + await createAuditLogPartition( + knex, + new Date(nextDate.getFullYear(), nextDate.getMonth() + 1, 1), + new Date(nextDate.getFullYear(), nextDate.getMonth() + 2, 1) + ); + + await createAuditLogPartition( + knex, + new Date(nextDate.getFullYear(), nextDate.getMonth() + 2, 1), + new Date(nextDate.getFullYear(), nextDate.getMonth() + 3, 1) + ); + } +} + +export async function down(knex: Knex): Promise { + // detach audit log from partition + await knex.schema.raw(` + ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog}; + + ALTER TABLE ${TableName.AuditLog} DROP CONSTRAINT audit_log_old; + `); + + // revert audit log modifications + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesTableExist = await knex.schema.hasTable(TableName.AuditLog); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + if (doesTableExist) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // we drop this first because adding to the partition results in a new primary key + t.dropPrimary(); + + // add back the original keys of the audit logs table + t.primary(["id"], { + constraintName: "audit_logs_pkey" + }); + + if (doesOrgIdExist) { + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + } + if (doesProjectIdExist) { + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + } + + // remove normalized fields + if (doesProjectNameExist) { + t.dropColumn("projectName"); + } + }); + } + + await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog); +} diff --git a/backend/src/db/schemas/index.ts b/backend/src/db/schemas/index.ts index 4fcf26c1a3..86ff5e612d 100644 --- a/backend/src/db/schemas/index.ts +++ b/backend/src/db/schemas/index.ts @@ -55,6 +55,7 @@ export * from "./org-bots"; export * from "./org-memberships"; export * from "./org-roles"; export * from "./organizations"; +export * from "./partitioned-audit-logs"; export * from "./pki-alerts"; export * from "./pki-collection-items"; export * from "./pki-collections"; diff --git a/backend/src/db/schemas/models.ts b/backend/src/db/schemas/models.ts index 08f3e79cee..4e241439ec 100644 --- a/backend/src/db/schemas/models.ts +++ b/backend/src/db/schemas/models.ts @@ -90,6 +90,7 @@ export enum TableName { OidcConfig = "oidc_configs", LdapGroupMap = "ldap_group_maps", AuditLog = "audit_logs", + PartitionedAuditLog = "partitioned_audit_logs", AuditLogStream = "audit_log_streams", GitAppInstallSession = "git_app_install_sessions", GitAppOrg = "git_app_org", diff --git a/backend/src/db/schemas/partitioned-audit-logs.ts b/backend/src/db/schemas/partitioned-audit-logs.ts new file mode 100644 index 0000000000..dd9500e7a7 --- /dev/null +++ b/backend/src/db/schemas/partitioned-audit-logs.ts @@ -0,0 +1,29 @@ +// Code generated by automation script, DO NOT EDIT. +// Automated by pulling database and generating zod schema +// To update. Just run npm run generate:schema +// Written by akhilmhdh. + +import { z } from "zod"; + +import { TImmutableDBKeys } from "./models"; + +export const PartitionedAuditLogsSchema = z.object({ + id: z.string().uuid(), + actor: z.string(), + actorMetadata: z.unknown(), + ipAddress: z.string().nullable().optional(), + eventType: z.string(), + eventMetadata: z.unknown().nullable().optional(), + userAgent: z.string().nullable().optional(), + userAgentType: z.string().nullable().optional(), + expiresAt: z.date().nullable().optional(), + createdAt: z.date(), + updatedAt: z.date(), + orgId: z.string().uuid().nullable().optional(), + projectId: z.string().nullable().optional(), + projectName: z.string().nullable().optional() +}); + +export type TPartitionedAuditLogs = z.infer; +export type TPartitionedAuditLogsInsert = Omit, TImmutableDBKeys>; +export type TPartitionedAuditLogsUpdate = Partial, TImmutableDBKeys>>; diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index e528bfa44d..c26942a6c9 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -25,7 +25,7 @@ type TFindQuery = { }; export const auditLogDALFactory = (db: TDbClient) => { - const auditLogOrm = ormify(db, TableName.AuditLog); + const auditLogOrm = ormify(db, TableName.PartitionedAuditLog); const find = async ( { @@ -54,13 +54,13 @@ export const auditLogDALFactory = (db: TDbClient) => { try { // Find statements - const sqlQuery = (tx || db.replicaNode())(TableName.AuditLog) + const sqlQuery = (tx || db.replicaNode())(TableName.PartitionedAuditLog) // eslint-disable-next-line func-names .where(function () { if (orgId) { - void this.where(`${TableName.AuditLog}.orgId`, orgId); + void this.where(`${TableName.PartitionedAuditLog}.orgId`, orgId); } else if (projectId) { - void this.where(`${TableName.AuditLog}.projectId`, projectId); + void this.where(`${TableName.PartitionedAuditLog}.projectId`, projectId); } }); @@ -70,10 +70,10 @@ export const auditLogDALFactory = (db: TDbClient) => { // Select statements void sqlQuery - .select(selectAllTableCols(TableName.AuditLog)) + .select(selectAllTableCols(TableName.PartitionedAuditLog)) .limit(limit) .offset(offset) - .orderBy(`${TableName.AuditLog}.createdAt`, "desc"); + .orderBy(`${TableName.PartitionedAuditLog}.createdAt`, "desc"); // Special case: Filter by actor ID if (actorId) { @@ -99,10 +99,10 @@ export const auditLogDALFactory = (db: TDbClient) => { // Filter by date range if (startDate) { - void sqlQuery.where(`${TableName.AuditLog}.createdAt`, ">=", startDate); + void sqlQuery.where(`${TableName.PartitionedAuditLog}.createdAt`, ">=", startDate); } if (endDate) { - void sqlQuery.where(`${TableName.AuditLog}.createdAt`, "<=", endDate); + void sqlQuery.where(`${TableName.PartitionedAuditLog}.createdAt`, "<=", endDate); } const docs = await sqlQuery; @@ -126,13 +126,13 @@ export const auditLogDALFactory = (db: TDbClient) => { logger.info(`${QueueName.DailyResourceCleanUp}: audit log started`); do { try { - const findExpiredLogSubQuery = (tx || db)(TableName.AuditLog) + const findExpiredLogSubQuery = (tx || db)(TableName.PartitionedAuditLog) .where("expiresAt", "<", today) .select("id") .limit(AUDIT_LOG_PRUNE_BATCH_SIZE); // eslint-disable-next-line no-await-in-loop - deletedAuditLogIds = await (tx || db)(TableName.AuditLog) + deletedAuditLogIds = await (tx || db)(TableName.PartitionedAuditLog) .whereIn("id", findExpiredLogSubQuery) .del() .returning("id"); From 723f0e862dc9fc501267e9aca9e1b1500c579278 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 01:42:24 +0800 Subject: [PATCH 05/15] misc: finalized partition script --- .../20241003075413_partition-audit-logs.ts | 53 +++++++++++-------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts index 1b18e9e3d1..3fd9bf200c 100644 --- a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts +++ b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts @@ -2,17 +2,17 @@ import { Knex } from "knex"; import { TableName } from "../schemas"; -const formatDateToYYYYMMDD = (date: Date) => { +const formatPartitionDate = (date: Date) => { const year = date.getFullYear(); - const month = String(date.getMonth() + 1).padStart(2, "0"); // getMonth() returns 0-based month, so add 1 + const month = String(date.getMonth() + 1).padStart(2, "0"); const day = String(date.getDate()).padStart(2, "0"); return `${year}-${month}-${day}`; }; const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Date) => { - const startDateStr = formatDateToYYYYMMDD(startDate); - const endDateStr = formatDateToYYYYMMDD(endDate); + const startDateStr = formatPartitionDate(startDate); + const endDateStr = formatPartitionDate(endDate); const partitionName = `${TableName.PartitionedAuditLog}_${startDateStr.replace(/-/g, "")}_${endDateStr.replace( /-/g, @@ -75,7 +75,14 @@ export async function up(knex: Knex): Promise { ${createTableSql} PARTITION BY RANGE ("createdAt"); `); - // add indices + await knex.schema.alterTable(TableName.PartitionedAuditLog, (t) => { + t.index(["projectId", "createdAt"]); + t.index(["orgId", "createdAt"]); + t.index("expiresAt"); + t.index("orgId"); + t.index("projectId"); + }); + await knex.raw( `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` ); @@ -91,7 +98,7 @@ export async function up(knex: Knex): Promise { const nextDate = new Date(); nextDate.setDate(nextDate.getDate() + 1); - const nextDateStr = formatDateToYYYYMMDD(nextDate); + const nextDateStr = formatPartitionDate(nextDate); // attach existing audit log table as a partition await knex.schema.raw(` @@ -102,20 +109,23 @@ export async function up(knex: Knex): Promise { FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); `); - // create partitions 3 months ahead - await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1, 1)); - - await createAuditLogPartition( - knex, - new Date(nextDate.getFullYear(), nextDate.getMonth() + 1, 1), - new Date(nextDate.getFullYear(), nextDate.getMonth() + 2, 1) - ); - - await createAuditLogPartition( - knex, - new Date(nextDate.getFullYear(), nextDate.getMonth() + 2, 1), - new Date(nextDate.getFullYear(), nextDate.getMonth() + 3, 1) - ); + // create partition from now until end of month + await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); + + // create partitions 4 years ahead + const partitionMonths = 4 * 12; + const partitionPromises: Promise[] = []; + for (let x = 1; x < partitionMonths; x += 1) { + partitionPromises.push( + createAuditLogPartition( + knex, + new Date(nextDate.getFullYear(), nextDate.getMonth() + x, 1), + new Date(nextDate.getFullYear(), nextDate.getMonth() + (x + 1), 1) + ) + ); + } + + await Promise.all(partitionPromises); } } @@ -130,10 +140,9 @@ export async function down(knex: Knex): Promise { // revert audit log modifications const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesTableExist = await knex.schema.hasTable(TableName.AuditLog); const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - if (doesTableExist) { + if (await knex.schema.hasTable(TableName.AuditLog)) { await knex.schema.alterTable(TableName.AuditLog, (t) => { // we drop this first because adding to the partition results in a new primary key t.dropPrimary(); From 81846d9c67b038bb1b055aea2147282bf7ab7425 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 02:25:02 +0800 Subject: [PATCH 06/15] misc: added timeout for db queries --- backend/src/ee/services/audit-log/audit-log-dal.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index c26942a6c9..e818f2c83f 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -105,7 +105,8 @@ export const auditLogDALFactory = (db: TDbClient) => { void sqlQuery.where(`${TableName.PartitionedAuditLog}.createdAt`, "<=", endDate); } - const docs = await sqlQuery; + // we timeout long running queries to prevent DB resource issues (2 minutes) + const docs = await sqlQuery.timeout(1000 * 120); return docs; } catch (error) { From e05f05f9edac18e37d35c0852a0c6f2ccbce808b Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 02:41:21 +0800 Subject: [PATCH 07/15] misc: added timeout error prompt --- .../ee/services/audit-log/audit-log-dal.ts | 11 +++- frontend/src/hooks/api/auditLogs/queries.tsx | 50 ++++++++++++------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index e818f2c83f..ad509a743d 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -1,8 +1,8 @@ -import { Knex } from "knex"; +import { Knex, KnexTimeoutError } from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; -import { DatabaseError } from "@app/lib/errors"; +import { BadRequestError, DatabaseError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; import { QueueName } from "@app/queue"; @@ -110,6 +110,13 @@ export const auditLogDALFactory = (db: TDbClient) => { return docs; } catch (error) { + if (error instanceof KnexTimeoutError) { + throw new BadRequestError({ + error, + message: "Failed to fetch audit logs due to timeout. Add more search filters." + }); + } + throw new DatabaseError({ error }); } }; diff --git a/frontend/src/hooks/api/auditLogs/queries.tsx b/frontend/src/hooks/api/auditLogs/queries.tsx index 1c74a79ba2..5ec69b0e72 100644 --- a/frontend/src/hooks/api/auditLogs/queries.tsx +++ b/frontend/src/hooks/api/auditLogs/queries.tsx @@ -1,5 +1,7 @@ import { useInfiniteQuery, UseInfiniteQueryOptions, useQuery } from "@tanstack/react-query"; +import { AxiosError } from "axios"; +import { createNotification } from "@app/components/notifications"; import { apiRequest } from "@app/config/request"; import { Actor, AuditLog, TGetAuditLogsFilter } from "./types"; @@ -28,27 +30,37 @@ export const useGetAuditLogs = ( return useInfiniteQuery({ queryKey: auditLogKeys.getAuditLogs(projectId, filters), queryFn: async ({ pageParam }) => { - const { data } = await apiRequest.get<{ auditLogs: AuditLog[] }>( - "/api/v1/organization/audit-logs", - { - params: { - ...filters, - offset: pageParam, - startDate: filters?.startDate?.toISOString(), - endDate: filters?.endDate?.toISOString(), - ...(filters.eventMetadata && Object.keys(filters.eventMetadata).length - ? { - eventMetadata: Object.entries(filters.eventMetadata) - .map(([key, value]) => `${key}=${value}`) - .join(",") - } - : {}), - ...(filters.eventType?.length ? { eventType: filters.eventType.join(",") } : {}), - ...(projectId ? { projectId } : {}) + try { + const { data } = await apiRequest.get<{ auditLogs: AuditLog[] }>( + "/api/v1/organization/audit-logs", + { + params: { + ...filters, + offset: pageParam, + startDate: filters?.startDate?.toISOString(), + endDate: filters?.endDate?.toISOString(), + ...(filters.eventMetadata && Object.keys(filters.eventMetadata).length + ? { + eventMetadata: Object.entries(filters.eventMetadata) + .map(([key, value]) => `${key}=${value}`) + .join(",") + } + : {}), + ...(filters.eventType?.length ? { eventType: filters.eventType.join(",") } : {}), + ...(projectId ? { projectId } : {}) + } } + ); + return data.auditLogs; + } catch (error) { + if (error instanceof AxiosError) { + createNotification({ + type: "error", + text: error.response?.data.message + }); } - ); - return data.auditLogs; + return []; + } }, getNextPageParam: (lastPage, pages) => lastPage.length !== 0 ? pages.length * filters.limit : undefined, From 8dbdb798333709dac0592719e58a05e980bef899 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 21:43:33 +0800 Subject: [PATCH 08/15] misc: finalized partition migration script --- backend/src/db/auditlog-knexfile.ts | 2 + .../20241003075413_partition-audit-logs.ts | 92 +++++++++++-------- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/backend/src/db/auditlog-knexfile.ts b/backend/src/db/auditlog-knexfile.ts index f6b84d4534..3ceef65a0d 100644 --- a/backend/src/db/auditlog-knexfile.ts +++ b/backend/src/db/auditlog-knexfile.ts @@ -18,6 +18,8 @@ if (!process.env.AUDIT_LOGS_DB_CONNECTION_URI && !process.env.AUDIT_LOGS_DB_HOST process.exit(0); } +console.info("Executing migration on audit log database..."); + export default { development: { client: "postgres", diff --git a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts index 3fd9bf200c..d0c9231039 100644 --- a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts +++ b/backend/src/db/migrations/20241003075413_partition-audit-logs.ts @@ -24,9 +24,11 @@ const createAuditLogPartition = async (knex: Knex, startDate: Date, endDate: Dat ); }; +const isUsingDedicatedAuditLogDb = Boolean(process.env.AUDIT_LOGS_DB_CONNECTION_URI); + export async function up(knex: Knex): Promise { - // prepare the existing audit log table for it to become a partition - if (await knex.schema.hasTable(TableName.AuditLog)) { + if (!isUsingDedicatedAuditLogDb && (await knex.schema.hasTable(TableName.AuditLog))) { + // prepare the existing audit log table for it to become a partition const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); @@ -84,11 +86,11 @@ export async function up(knex: Knex): Promise { }); await knex.raw( - `CREATE INDEX "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` + `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` ); await knex.raw( - `CREATE INDEX "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` + `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` ); // create default partition @@ -100,14 +102,16 @@ export async function up(knex: Knex): Promise { nextDate.setDate(nextDate.getDate() + 1); const nextDateStr = formatPartitionDate(nextDate); - // attach existing audit log table as a partition - await knex.schema.raw(` - ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old - CHECK ( "createdAt" < DATE '${nextDateStr}' ); - - ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog} - FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); - `); + // attach existing audit log table as a partition ONLY if using the same DB + if (!isUsingDedicatedAuditLogDb) { + await knex.schema.raw(` + ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old + CHECK ( "createdAt" < DATE '${nextDateStr}' ); + + ALTER TABLE ${TableName.PartitionedAuditLog} ATTACH PARTITION ${TableName.AuditLog} + FOR VALUES FROM (MINVALUE) TO ('${nextDateStr}' ); + `); + } // create partition from now until end of month await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); @@ -130,40 +134,50 @@ export async function up(knex: Knex): Promise { } export async function down(knex: Knex): Promise { - // detach audit log from partition - await knex.schema.raw(` + const result = await knex.raw(` + SELECT inhrelid::regclass::text + FROM pg_inherits + WHERE inhparent::regclass::text = '${TableName.PartitionedAuditLog}' + AND inhrelid::regclass::text = '${TableName.AuditLog}' + `); + + const isAuditLogAPartition = result.rows.length > 0; + if (isAuditLogAPartition) { + // detach audit log from partition + await knex.schema.raw(` ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog}; ALTER TABLE ${TableName.AuditLog} DROP CONSTRAINT audit_log_old; `); - // revert audit log modifications - const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); - const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - - if (await knex.schema.hasTable(TableName.AuditLog)) { - await knex.schema.alterTable(TableName.AuditLog, (t) => { - // we drop this first because adding to the partition results in a new primary key - t.dropPrimary(); + // revert audit log modifications + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - // add back the original keys of the audit logs table - t.primary(["id"], { - constraintName: "audit_logs_pkey" + if (await knex.schema.hasTable(TableName.AuditLog)) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + // we drop this first because adding to the partition results in a new primary key + t.dropPrimary(); + + // add back the original keys of the audit logs table + t.primary(["id"], { + constraintName: "audit_logs_pkey" + }); + + if (doesOrgIdExist) { + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + } + if (doesProjectIdExist) { + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + } + + // remove normalized fields + if (doesProjectNameExist) { + t.dropColumn("projectName"); + } }); - - if (doesOrgIdExist) { - t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); - } - if (doesProjectIdExist) { - t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); - } - - // remove normalized fields - if (doesProjectNameExist) { - t.dropColumn("projectName"); - } - }); + } } await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog); From d66932038593dc354b367a9f972517137cf05357 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 22:06:32 +0800 Subject: [PATCH 09/15] misc: addressed type issue with knex --- backend/src/ee/services/audit-log/audit-log-dal.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index ad509a743d..a12429e01c 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -1,4 +1,4 @@ -import { Knex, KnexTimeoutError } from "knex"; +import { Knex, knex } from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; @@ -110,7 +110,7 @@ export const auditLogDALFactory = (db: TDbClient) => { return docs; } catch (error) { - if (error instanceof KnexTimeoutError) { + if (error instanceof knex.KnexTimeoutError) { throw new BadRequestError({ error, message: "Failed to fetch audit logs due to timeout. Add more search filters." From 6bfeac5e981b1332c24411fe4fc1c6bf0dd9e9f9 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 22:15:39 +0800 Subject: [PATCH 10/15] misc: addressed import knex issue --- backend/src/ee/services/audit-log/audit-log-dal.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index a12429e01c..035527a6f5 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -1,4 +1,5 @@ -import { Knex, knex } from "knex"; +// weird commonjs-related error in the CI requires us to use this kind of import +import * as kx from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; @@ -46,7 +47,7 @@ export const auditLogDALFactory = (db: TDbClient) => { eventType?: EventType[]; eventMetadata?: Record; }, - tx?: Knex + tx?: kx.Knex ) => { if (!orgId && !projectId) { throw new Error("Either orgId or projectId must be provided"); @@ -110,7 +111,7 @@ export const auditLogDALFactory = (db: TDbClient) => { return docs; } catch (error) { - if (error instanceof knex.KnexTimeoutError) { + if (error instanceof kx.KnexTimeoutError) { throw new BadRequestError({ error, message: "Failed to fetch audit logs due to timeout. Add more search filters." @@ -122,7 +123,7 @@ export const auditLogDALFactory = (db: TDbClient) => { }; // delete all audit log that have expired - const pruneAuditLog = async (tx?: Knex) => { + const pruneAuditLog = async (tx?: kx.Knex) => { const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000; const MAX_RETRY_ON_FAILURE = 3; From cf446a38b316d168a648f8b491477c286bb5268c Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 22:27:11 +0800 Subject: [PATCH 11/15] misc: improved knex import --- backend/src/ee/services/audit-log/audit-log-dal.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 035527a6f5..37247ee300 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -1,5 +1,5 @@ -// weird commonjs-related error in the CI requires us to use this kind of import -import * as kx from "knex"; +// weird commonjs-related error in the CI requires us to do the import like this +import knex from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; @@ -47,7 +47,7 @@ export const auditLogDALFactory = (db: TDbClient) => { eventType?: EventType[]; eventMetadata?: Record; }, - tx?: kx.Knex + tx?: knex.Knex ) => { if (!orgId && !projectId) { throw new Error("Either orgId or projectId must be provided"); @@ -111,7 +111,7 @@ export const auditLogDALFactory = (db: TDbClient) => { return docs; } catch (error) { - if (error instanceof kx.KnexTimeoutError) { + if (error instanceof knex.KnexTimeoutError) { throw new BadRequestError({ error, message: "Failed to fetch audit logs due to timeout. Add more search filters." @@ -123,7 +123,7 @@ export const auditLogDALFactory = (db: TDbClient) => { }; // delete all audit log that have expired - const pruneAuditLog = async (tx?: kx.Knex) => { + const pruneAuditLog = async (tx?: knex.Knex) => { const AUDIT_LOG_PRUNE_BATCH_SIZE = 10000; const MAX_RETRY_ON_FAILURE = 3; From 1687d66a0ee08cfbbce45239bc38f9c7ce5f70e0 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Fri, 4 Oct 2024 22:37:13 +0800 Subject: [PATCH 12/15] misc: ignore partitions in generate schema --- backend/scripts/generate-schema-types.ts | 2 +- backend/src/db/schemas/audit-logs.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/scripts/generate-schema-types.ts b/backend/scripts/generate-schema-types.ts index 43984ecfac..c7b1fcc7a9 100644 --- a/backend/scripts/generate-schema-types.ts +++ b/backend/scripts/generate-schema-types.ts @@ -90,7 +90,7 @@ const main = async () => { .whereRaw("table_schema = current_schema()") .select<{ tableName: string }[]>("table_name as tableName") .orderBy("table_name") - ).filter((el) => !el.tableName.includes("_migrations")); + ).filter((el) => !el.tableName.includes("_migrations") && !el.tableName.includes("partitioned_audit_logs_")); for (let i = 0; i < tables.length; i += 1) { const { tableName } = tables[i]; diff --git a/backend/src/db/schemas/audit-logs.ts b/backend/src/db/schemas/audit-logs.ts index d1c239724c..b8906698b6 100644 --- a/backend/src/db/schemas/audit-logs.ts +++ b/backend/src/db/schemas/audit-logs.ts @@ -20,8 +20,7 @@ export const AuditLogsSchema = z.object({ createdAt: z.date(), updatedAt: z.date(), orgId: z.string().uuid().nullable().optional(), - projectId: z.string().nullable().optional(), - projectName: z.string().nullable().optional() + projectId: z.string().nullable().optional() }); export type TAuditLogs = z.infer; From 98cca7039cc02beb1e773df7d211d77b27d7395e Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Mon, 7 Oct 2024 14:00:20 +0800 Subject: [PATCH 13/15] misc: addressed comments --- backend/package.json | 12 ++--- ...241007052025_make-audit-log-independent.ts | 48 +++++++++++++++++++ ...=> 20241007052449_partition-audit-logs.ts} | 40 ++-------------- .../ee/services/audit-log/audit-log-dal.ts | 4 +- backend/src/lib/errors/index.ts | 12 +++++ backend/src/server/plugins/error-handler.ts | 8 +++- 6 files changed, 78 insertions(+), 46 deletions(-) create mode 100644 backend/src/db/migrations/20241007052025_make-audit-log-independent.ts rename backend/src/db/migrations/{20241003075413_partition-audit-logs.ts => 20241007052449_partition-audit-logs.ts} (77%) diff --git a/backend/package.json b/backend/package.json index 886b80893d..332c5a9610 100644 --- a/backend/package.json +++ b/backend/package.json @@ -52,12 +52,12 @@ "auditlog-migration:status": "knex --knexfile ./src/db/auditlog-knexfile.ts --client pg migrate:status", "auditlog-migration:rollback": "knex --knexfile ./src/db/auditlog-knexfile.ts migrate:rollback", "migration:new": "tsx ./scripts/create-migration.ts", - "migration:up": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:up && npm run auditlog-migration:up", - "migration:down": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:down && npm run auditlog-migration:down", - "migration:list": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:list && npm run auditlog-migration:list", - "migration:latest": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest && npm run auditlog-migration:latest", - "migration:status": "knex --knexfile ./src/db/knexfile.ts --client pg migrate:status && npm run auditlog-migration:status", - "migration:rollback": "knex --knexfile ./src/db/knexfile.ts migrate:rollback && npm run auditlog-migration:rollback", + "migration:up": "npm run auditlog-migration:up && knex --knexfile ./src/db/knexfile.ts --client pg migrate:up", + "migration:down": "npm run auditlog-migration:down && knex --knexfile ./src/db/knexfile.ts --client pg migrate:down", + "migration:list": "npm run auditlog-migration:list && knex --knexfile ./src/db/knexfile.ts --client pg migrate:list", + "migration:latest": "npm run auditlog-migration:latest && knex --knexfile ./src/db/knexfile.ts --client pg migrate:latest", + "migration:status": "npm run auditlog-migration:status && knex --knexfile ./src/db/knexfile.ts --client pg migrate:status", + "migration:rollback": "npm run auditlog-migration:rollback && knex --knexfile ./src/db/knexfile.ts migrate:rollback", "seed:new": "tsx ./scripts/create-seed-file.ts", "seed": "knex --knexfile ./src/db/knexfile.ts --client pg seed:run", "db:reset": "npm run migration:rollback -- --all && npm run migration:latest" diff --git a/backend/src/db/migrations/20241007052025_make-audit-log-independent.ts b/backend/src/db/migrations/20241007052025_make-audit-log-independent.ts new file mode 100644 index 0000000000..b6b98b9bc0 --- /dev/null +++ b/backend/src/db/migrations/20241007052025_make-audit-log-independent.ts @@ -0,0 +1,48 @@ +import { Knex } from "knex"; + +import { TableName } from "../schemas"; + +export async function up(knex: Knex): Promise { + if (await knex.schema.hasTable(TableName.AuditLog)) { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + await knex.schema.alterTable(TableName.AuditLog, (t) => { + if (doesOrgIdExist) { + t.dropForeign("orgId"); + } + + if (doesProjectIdExist) { + t.dropForeign("projectId"); + } + + // add normalized field + if (!doesProjectNameExist) { + t.string("projectName"); + } + }); + } +} + +export async function down(knex: Knex): Promise { + const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); + const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); + const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); + + if (await knex.schema.hasTable(TableName.AuditLog)) { + await knex.schema.alterTable(TableName.AuditLog, (t) => { + if (doesOrgIdExist) { + t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); + } + if (doesProjectIdExist) { + t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); + } + + // remove normalized field + if (doesProjectNameExist) { + t.dropColumn("projectName"); + } + }); + } +} diff --git a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts b/backend/src/db/migrations/20241007052449_partition-audit-logs.ts similarity index 77% rename from backend/src/db/migrations/20241003075413_partition-audit-logs.ts rename to backend/src/db/migrations/20241007052449_partition-audit-logs.ts index d0c9231039..a8b68ef83e 100644 --- a/backend/src/db/migrations/20241003075413_partition-audit-logs.ts +++ b/backend/src/db/migrations/20241007052449_partition-audit-logs.ts @@ -28,27 +28,9 @@ const isUsingDedicatedAuditLogDb = Boolean(process.env.AUDIT_LOGS_DB_CONNECTION_ export async function up(knex: Knex): Promise { if (!isUsingDedicatedAuditLogDb && (await knex.schema.hasTable(TableName.AuditLog))) { - // prepare the existing audit log table for it to become a partition - const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); - const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - await knex.schema.alterTable(TableName.AuditLog, (t) => { // remove existing keys t.dropPrimary(); - - if (doesOrgIdExist) { - t.dropForeign("orgId"); - } - - if (doesProjectIdExist) { - t.dropForeign("projectId"); - } - - // add normalized fields present in the partition table - if (!doesProjectNameExist) { - t.string("projectName"); - } }); } @@ -119,7 +101,7 @@ export async function up(knex: Knex): Promise { // create partitions 4 years ahead const partitionMonths = 4 * 12; const partitionPromises: Promise[] = []; - for (let x = 1; x < partitionMonths; x += 1) { + for (let x = 1; x <= partitionMonths; x += 1) { partitionPromises.push( createAuditLogPartition( knex, @@ -134,14 +116,14 @@ export async function up(knex: Knex): Promise { } export async function down(knex: Knex): Promise { - const result = await knex.raw(` + const partitionSearchResult = await knex.raw(` SELECT inhrelid::regclass::text FROM pg_inherits WHERE inhparent::regclass::text = '${TableName.PartitionedAuditLog}' AND inhrelid::regclass::text = '${TableName.AuditLog}' `); - const isAuditLogAPartition = result.rows.length > 0; + const isAuditLogAPartition = partitionSearchResult.rows.length > 0; if (isAuditLogAPartition) { // detach audit log from partition await knex.schema.raw(` @@ -151,10 +133,6 @@ export async function down(knex: Knex): Promise { `); // revert audit log modifications - const doesProjectIdExist = await knex.schema.hasColumn(TableName.AuditLog, "projectId"); - const doesOrgIdExist = await knex.schema.hasColumn(TableName.AuditLog, "orgId"); - const doesProjectNameExist = await knex.schema.hasColumn(TableName.AuditLog, "projectName"); - if (await knex.schema.hasTable(TableName.AuditLog)) { await knex.schema.alterTable(TableName.AuditLog, (t) => { // we drop this first because adding to the partition results in a new primary key @@ -164,18 +142,6 @@ export async function down(knex: Knex): Promise { t.primary(["id"], { constraintName: "audit_logs_pkey" }); - - if (doesOrgIdExist) { - t.foreign("orgId").references("id").inTable(TableName.Organization).onDelete("CASCADE"); - } - if (doesProjectIdExist) { - t.foreign("projectId").references("id").inTable(TableName.Project).onDelete("CASCADE"); - } - - // remove normalized fields - if (doesProjectNameExist) { - t.dropColumn("projectName"); - } }); } } diff --git a/backend/src/ee/services/audit-log/audit-log-dal.ts b/backend/src/ee/services/audit-log/audit-log-dal.ts index 37247ee300..214fc7b4f5 100644 --- a/backend/src/ee/services/audit-log/audit-log-dal.ts +++ b/backend/src/ee/services/audit-log/audit-log-dal.ts @@ -3,7 +3,7 @@ import knex from "knex"; import { TDbClient } from "@app/db"; import { TableName } from "@app/db/schemas"; -import { BadRequestError, DatabaseError } from "@app/lib/errors"; +import { DatabaseError, GatewayTimeoutError } from "@app/lib/errors"; import { ormify, selectAllTableCols } from "@app/lib/knex"; import { logger } from "@app/lib/logger"; import { QueueName } from "@app/queue"; @@ -112,7 +112,7 @@ export const auditLogDALFactory = (db: TDbClient) => { return docs; } catch (error) { if (error instanceof knex.KnexTimeoutError) { - throw new BadRequestError({ + throw new GatewayTimeoutError({ error, message: "Failed to fetch audit logs due to timeout. Add more search filters." }); diff --git a/backend/src/lib/errors/index.ts b/backend/src/lib/errors/index.ts index fd56317888..01d1c8b8e8 100644 --- a/backend/src/lib/errors/index.ts +++ b/backend/src/lib/errors/index.ts @@ -23,6 +23,18 @@ export class InternalServerError extends Error { } } +export class GatewayTimeoutError extends Error { + name: string; + + error: unknown; + + constructor({ name, error, message }: { message?: string; name?: string; error?: unknown }) { + super(message || "Timeout error"); + this.name = name || "GatewayTimeoutError"; + this.error = error; + } +} + export class UnauthorizedError extends Error { name: string; diff --git a/backend/src/server/plugins/error-handler.ts b/backend/src/server/plugins/error-handler.ts index 8ea8b82236..03681b1c89 100644 --- a/backend/src/server/plugins/error-handler.ts +++ b/backend/src/server/plugins/error-handler.ts @@ -7,6 +7,7 @@ import { BadRequestError, DatabaseError, ForbiddenRequestError, + GatewayTimeoutError, InternalServerError, NotFoundError, ScimRequestError, @@ -25,7 +26,8 @@ enum HttpStatusCodes { Unauthorized = 401, Forbidden = 403, // eslint-disable-next-line @typescript-eslint/no-shadow - InternalServerError = 500 + InternalServerError = 500, + GatewayTimeout = 504 } export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider) => { @@ -47,6 +49,10 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider void res .status(HttpStatusCodes.InternalServerError) .send({ statusCode: HttpStatusCodes.InternalServerError, message: "Something went wrong", error: error.name }); + } else if (error instanceof GatewayTimeoutError) { + void res + .status(HttpStatusCodes.GatewayTimeout) + .send({ statusCode: HttpStatusCodes.GatewayTimeout, message: error.message, error: error.name }); } else if (error instanceof ZodError) { void res .status(HttpStatusCodes.Unauthorized) From 5bf6f69fca714956a9d21e7154fcbb3251e3198c Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 02:44:24 +0800 Subject: [PATCH 14/15] misc: moved to partitionauditlogs schema --- backend/src/ee/routes/v1/project-router.ts | 4 ++-- backend/src/server/routes/v1/organization-router.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/ee/routes/v1/project-router.ts b/backend/src/ee/routes/v1/project-router.ts index e3956731eb..aefca8a7bc 100644 --- a/backend/src/ee/routes/v1/project-router.ts +++ b/backend/src/ee/routes/v1/project-router.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { AuditLogsSchema, SecretSnapshotsSchema } from "@app/db/schemas"; +import { PartitionedAuditLogsSchema, SecretSnapshotsSchema } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; import { AUDIT_LOGS, PROJECTS } from "@app/lib/api-docs"; import { getLastMidnightDateISO, removeTrailingSlash } from "@app/lib/fn"; @@ -120,7 +120,7 @@ export const registerProjectRouter = async (server: FastifyZodProvider) => { }), response: { 200: z.object({ - auditLogs: AuditLogsSchema.omit({ + auditLogs: PartitionedAuditLogsSchema.omit({ eventMetadata: true, eventType: true, actor: true, diff --git a/backend/src/server/routes/v1/organization-router.ts b/backend/src/server/routes/v1/organization-router.ts index 439950f1cd..68e1864684 100644 --- a/backend/src/server/routes/v1/organization-router.ts +++ b/backend/src/server/routes/v1/organization-router.ts @@ -1,12 +1,12 @@ import { z } from "zod"; import { - AuditLogsSchema, GroupsSchema, IncidentContactsSchema, OrganizationsSchema, OrgMembershipsSchema, OrgRolesSchema, + PartitionedAuditLogsSchema, UsersSchema } from "@app/db/schemas"; import { EventType, UserAgentType } from "@app/ee/services/audit-log/audit-log-types"; @@ -117,7 +117,7 @@ export const registerOrgRouter = async (server: FastifyZodProvider) => { response: { 200: z.object({ - auditLogs: AuditLogsSchema.omit({ + auditLogs: PartitionedAuditLogsSchema.omit({ eventMetadata: true, eventType: true, actor: true, From f6fcef24c668b49f265b888c2f9285a23a330c19 Mon Sep 17 00:00:00 2001 From: Sheen Capadngan Date: Tue, 8 Oct 2024 02:56:10 +0800 Subject: [PATCH 15/15] misc: added console statement to partition migration --- .../20241007052449_partition-audit-logs.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/src/db/migrations/20241007052449_partition-audit-logs.ts b/backend/src/db/migrations/20241007052449_partition-audit-logs.ts index a8b68ef83e..e132bb5efa 100644 --- a/backend/src/db/migrations/20241007052449_partition-audit-logs.ts +++ b/backend/src/db/migrations/20241007052449_partition-audit-logs.ts @@ -28,6 +28,7 @@ const isUsingDedicatedAuditLogDb = Boolean(process.env.AUDIT_LOGS_DB_CONNECTION_ export async function up(knex: Knex): Promise { if (!isUsingDedicatedAuditLogDb && (await knex.schema.hasTable(TableName.AuditLog))) { + console.info("Dropping primary key of Audit Log table..."); await knex.schema.alterTable(TableName.AuditLog, (t) => { // remove existing keys t.dropPrimary(); @@ -55,10 +56,12 @@ export async function up(knex: Knex): Promise { }) .toString(); + console.info("Creating partition table..."); await knex.schema.raw(` ${createTableSql} PARTITION BY RANGE ("createdAt"); `); + console.log("Adding indices..."); await knex.schema.alterTable(TableName.PartitionedAuditLog, (t) => { t.index(["projectId", "createdAt"]); t.index(["orgId", "createdAt"]); @@ -67,15 +70,20 @@ export async function up(knex: Knex): Promise { t.index("projectId"); }); + console.log("Adding GIN indices..."); + await knex.raw( `CREATE INDEX IF NOT EXISTS "audit_logs_actorMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("actorMetadata" jsonb_path_ops)` ); + console.log("GIN index for actorMetadata done"); await knex.raw( `CREATE INDEX IF NOT EXISTS "audit_logs_eventMetadata_idx" ON ${TableName.PartitionedAuditLog} USING gin("eventMetadata" jsonb_path_ops)` ); + console.log("GIN index for eventMetadata done"); // create default partition + console.log("Creating default partition..."); await knex.schema.raw( `CREATE TABLE ${TableName.PartitionedAuditLog}_default PARTITION OF ${TableName.PartitionedAuditLog} DEFAULT` ); @@ -86,6 +94,7 @@ export async function up(knex: Knex): Promise { // attach existing audit log table as a partition ONLY if using the same DB if (!isUsingDedicatedAuditLogDb) { + console.log("Attaching existing audit log table as a partition..."); await knex.schema.raw(` ALTER TABLE ${TableName.AuditLog} ADD CONSTRAINT audit_log_old CHECK ( "createdAt" < DATE '${nextDateStr}' ); @@ -96,6 +105,7 @@ export async function up(knex: Knex): Promise { } // create partition from now until end of month + console.log("Creating audit log partitions ahead of time... next date:", nextDateStr); await createAuditLogPartition(knex, nextDate, new Date(nextDate.getFullYear(), nextDate.getMonth() + 1)); // create partitions 4 years ahead @@ -112,6 +122,7 @@ export async function up(knex: Knex): Promise { } await Promise.all(partitionPromises); + console.log("Partition migration complete"); } } @@ -126,6 +137,7 @@ export async function down(knex: Knex): Promise { const isAuditLogAPartition = partitionSearchResult.rows.length > 0; if (isAuditLogAPartition) { // detach audit log from partition + console.log("Detaching original audit log table from new partition table..."); await knex.schema.raw(` ALTER TABLE ${TableName.PartitionedAuditLog} DETACH PARTITION ${TableName.AuditLog}; @@ -133,6 +145,7 @@ export async function down(knex: Knex): Promise { `); // revert audit log modifications + console.log("Reverting changes made to the audit log table..."); if (await knex.schema.hasTable(TableName.AuditLog)) { await knex.schema.alterTable(TableName.AuditLog, (t) => { // we drop this first because adding to the partition results in a new primary key @@ -147,4 +160,5 @@ export async function down(knex: Knex): Promise { } await knex.schema.dropTableIfExists(TableName.PartitionedAuditLog); + console.log("Partition rollback complete"); }