From 4a50347aa883d3ca798e7ec2d7e52cd2b0c314a5 Mon Sep 17 00:00:00 2001 From: Mike Silva Date: Thu, 10 Oct 2024 09:47:08 -0700 Subject: [PATCH 1/2] fixed bucket caching issue for multiple keys on audit log. --- apps/workflows/jobs/refill-daily.ts | 40 +++++++++++++++++++------- apps/workflows/jobs/refill-monthly.ts | 41 ++++++++++++++++++++------- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/apps/workflows/jobs/refill-daily.ts b/apps/workflows/jobs/refill-daily.ts index edbcb9f530..eca82f7578 100644 --- a/apps/workflows/jobs/refill-daily.ts +++ b/apps/workflows/jobs/refill-daily.ts @@ -15,7 +15,11 @@ client.defineJob({ const db = connectDatabase(); const t = new Date(); t.setUTCHours(t.getUTCHours() - 24); + const BUCKET_NAME = "unkey_mutations"; + type Key = `${string}::${string}`; + type BucketId = string; + const bucketCache = new Map(); const keys = await io.runTask("list keys", () => db.query.keys.findMany({ where: (table, { isNotNull, isNull, eq, and, gt, or }) => @@ -35,17 +39,33 @@ client.defineJob({ io.logger.info(`found ${keys.length} keys with daily refill set`); for (const key of keys) { - const bucket = await io.runTask(`get bucket for ${key.workspaceId}`, async () => { - return await db.query.auditLogBucket.findFirst({ + const cacheKey: Key = `${key.workspaceId}::${BUCKET_NAME}`; + let bucketId = ""; + const cachedBucketId = bucketCache.get(cacheKey); + if (cachedBucketId) { + bucketId = cachedBucketId; + } else { + const bucket = await db.query.auditLogBucket.findFirst({ where: (table, { eq, and }) => - and(eq(table.workspaceId, key.workspaceId), eq(table.name, "unkey_mutations")), + and(eq(table.workspaceId, key.workspaceId), eq(table.name, BUCKET_NAME)), + columns: { + id: true, + }, }); - }); - if (!bucket) { - io.logger.error(`bucket for ${key.workspaceId} does not exist`); - continue; + + if (bucket) { + bucketId = bucket.id; + } else { + bucketId = newId("auditLogBucket"); + await db.insert(schema.auditLogBucket).values({ + id: bucketId, + workspaceId: key.workspaceId, + name: BUCKET_NAME, + }); + } } + bucketCache.set(cacheKey, bucketId); await io.runTask(`refill for ${key.id}`, async () => { await db.transaction(async (tx) => { await tx @@ -60,7 +80,7 @@ client.defineJob({ await tx.insert(schema.auditLog).values({ id: auditLogId, workspaceId: key.workspaceId, - bucketId: bucket.id, + bucketId: bucketId, time: Date.now(), event: "key.update", actorId: "trigger", @@ -72,7 +92,7 @@ client.defineJob({ type: "workspace", id: key.workspaceId, workspaceId: key.workspaceId, - bucketId: bucket.id, + bucketId: bucketId, auditLogId, displayName: `workspace ${key.workspaceId}`, }, @@ -80,7 +100,7 @@ client.defineJob({ type: "key", id: key.id, workspaceId: key.workspaceId, - bucketId: bucket.id, + bucketId: bucketId, auditLogId, displayName: `key ${key.id}`, }, diff --git a/apps/workflows/jobs/refill-monthly.ts b/apps/workflows/jobs/refill-monthly.ts index 452752ae2e..ccbb2f791a 100644 --- a/apps/workflows/jobs/refill-monthly.ts +++ b/apps/workflows/jobs/refill-monthly.ts @@ -14,7 +14,11 @@ client.defineJob({ const db = connectDatabase(); const t = new Date(); t.setUTCMonth(t.getUTCMonth() - 1); + const BUCKET_NAME = "unkey_mutations"; + type Key = `${string}::${string}`; + type BucketId = string; + const bucketCache = new Map(); const keys = await io.runTask("list keys", () => db.query.keys.findMany({ where: (table, { isNotNull, isNull, eq, and, or }) => @@ -33,17 +37,34 @@ client.defineJob({ ); io.logger.info(`found ${keys.length} keys with monthly refill set`); for (const key of keys) { - const bucket = await io.runTask(`get bucket for ${key.workspaceId}`, async () => { - return await db.query.auditLogBucket.findFirst({ + + const cacheKey: Key = `${key.workspaceId}::${BUCKET_NAME}`; + let bucketId = ""; + const cachedBucketId = bucketCache.get(cacheKey); + if (cachedBucketId) { + bucketId = cachedBucketId; + } else { + const bucket = await db.query.auditLogBucket.findFirst({ where: (table, { eq, and }) => - and(eq(table.workspaceId, key.workspaceId), eq(table.name, "unkey_mutations")), + and(eq(table.workspaceId, key.workspaceId), eq(table.name, BUCKET_NAME)), + columns: { + id: true, + }, }); - }); - if (!bucket) { - io.logger.error(`bucket for ${key.workspaceId} does not exist`); - continue; + + if (bucket) { + bucketId = bucket.id; + } else { + bucketId = newId("auditLogBucket"); + await db.insert(schema.auditLogBucket).values({ + id: bucketId, + workspaceId: key.workspaceId, + name: BUCKET_NAME, + }); + } } + bucketCache.set(cacheKey, bucketId); await io.runTask(`refill for ${key.id}`, async () => { await db.transaction(async (tx) => { await tx @@ -58,7 +79,7 @@ client.defineJob({ await tx.insert(schema.auditLog).values({ id: auditLogId, workspaceId: key.workspaceId, - bucketId: bucket.id, + bucketId: bucketId, time: Date.now(), event: "key.update", actorId: "trigger", @@ -70,7 +91,7 @@ client.defineJob({ type: "workspace", id: key.workspaceId, workspaceId: key.workspaceId, - bucketId: bucket.id, + bucketId: bucketId, auditLogId, displayName: `workspace ${key.workspaceId}`, }, @@ -78,7 +99,7 @@ client.defineJob({ type: "key", id: key.id, workspaceId: key.workspaceId, - bucketId: bucket.id, + bucketId: bucketId, auditLogId, displayName: `key ${key.id}`, }, From a97a6cb78a49d4c42f6ae5717f3872b91d4e1657 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:50:46 +0000 Subject: [PATCH 2/2] [autofix.ci] apply automated fixes --- apps/workflows/jobs/refill-monthly.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/workflows/jobs/refill-monthly.ts b/apps/workflows/jobs/refill-monthly.ts index ccbb2f791a..47c9cd1369 100644 --- a/apps/workflows/jobs/refill-monthly.ts +++ b/apps/workflows/jobs/refill-monthly.ts @@ -37,7 +37,6 @@ client.defineJob({ ); io.logger.info(`found ${keys.length} keys with monthly refill set`); for (const key of keys) { - const cacheKey: Key = `${key.workspaceId}::${BUCKET_NAME}`; let bucketId = ""; const cachedBucketId = bucketCache.get(cacheKey);