feat: return errors as values for clickhouse (#2665)

unkeyed · Nov 18, 2024 · 707f0d4 · 707f0d4
1 parent b21b1c4
commit 707f0d4
Show file tree

Hide file tree

Showing 21 changed files with 135 additions and 79 deletions.
diff --git a/apps/api/src/routes/v1_keys_getVerifications.ts b/apps/api/src/routes/v1_keys_getVerifications.ts
@@ -221,13 +221,17 @@ export const registerV1KeysGetVerifications = (app: App) =>
     const verificationsFromAllKeys = await Promise.all(
       ids.map(({ keyId, keySpaceId }) => {
         return cache.verificationsByKeyId.swr(`${keyId}:${start}-${end}`, async () => {
-          return await analytics.getVerificationsDaily({
+          const res = await analytics.getVerificationsDaily({
             workspaceId: authorizedWorkspaceId,
             keySpaceId: keySpaceId,
             keyId: keyId,
             start: start ? start : now - 24 * 60 * 60 * 1000,
             end: end ? end : now,
           });
+          if (res.err) {
+            throw new Error(res.err.message);
+          }
+          return res.val;
         });
       }),
     );

diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/page.tsx b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/page.tsx
@@ -97,7 +97,7 @@ export default async function APIKeyDetailPage(props: {
     }),
     clickhouse.verifications
       .latest({ workspaceId: key.workspaceId, keySpaceId: key.keyAuthId, keyId: key.id })
-      .then((res) => res.at(0)?.time ?? 0),
+      .then((res) => res.val?.at(0)?.time ?? 0),
   ]);
 
   const successOverTime: { x: string; y: number }[] = [];
@@ -108,7 +108,7 @@ export default async function APIKeyDetailPage(props: {
   const expiredOverTime: { x: string; y: number }[] = [];
   const forbiddenOverTime: { x: string; y: number }[] = [];
 
-  for (const d of verifications.sort((a, b) => a.time - b.time)) {
+  for (const d of verifications.val!.sort((a, b) => a.time - b.time)) {
     const x = new Date(d.time).toISOString();
     switch (d.outcome) {
       case "":
@@ -174,7 +174,7 @@ export default async function APIKeyDetailPage(props: {
     expired: 0,
     forbidden: 0,
   };
-  verifications.forEach((v) => {
+  verifications.val!.forEach((v) => {
     switch (v.outcome) {
       case "VALID":
         stats.valid += v.count;
@@ -317,7 +317,7 @@ export default async function APIKeyDetailPage(props: {
           </EmptyPlaceholder>
         )}
 
-        {latestVerifications.length > 0 ? (
+        {latestVerifications.val && latestVerifications.val.length > 0 ? (
           <>
             <Separator className="my-8" />
             <h2 className="text-2xl font-semibold leading-none tracking-tight mt-8">

diff --git a/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/verification-table.tsx b/apps/dashboard/app/(app)/apis/[apiId]/keys/[keyAuthId]/[keyId]/verification-table.tsx
@@ -34,7 +34,7 @@ export const VerificationTable = ({ verifications }: Props) => {
           </TableRow>
         </TableHeader>
         <TableBody className={"font-mono"}>
-          {verifications.map((verification, i) => {
+          {verifications.val?.map((verification, i) => {
             /**
              * Instead of rounding every row individually, we want to round consecutive colored rows together.
              * For example:
@@ -54,10 +54,10 @@ export const VerificationTable = ({ verifications }: Props) => {
              */
             const isStartOfColoredBlock =
               verification.outcome !== "VALID" &&
-              (i === 0 || verifications[i - 1].outcome === "VALID");
+              (i === 0 || verifications.val[i - 1].outcome === "VALID");
             const isEndOfColoredBlock =
               verification.outcome !== "VALID" &&
-              (i === verifications.length - 1 || verifications[i + 1].outcome === "VALID");
+              (i === verifications.val.length - 1 || verifications.val[i + 1].outcome === "VALID");
 
             return (
               <TableRow

diff --git a/apps/dashboard/app/(app)/apis/[apiId]/page.tsx b/apps/dashboard/app/(app)/apis/[apiId]/page.tsx
@@ -54,22 +54,22 @@ export default async function ApiPage(props: {
         .where(and(eq(schema.keys.keyAuthId, api.keyAuthId!), isNull(schema.keys.deletedAt)))
         .execute()
         .then((res) => res.at(0)?.count ?? 0),
-      getVerificationsPerInterval(query),
-      getActiveKeysPerInterval(query),
+      getVerificationsPerInterval(query).then((res) => res.val!),
+      getActiveKeysPerInterval(query).then((res) => res.val!),
       clickhouse.activeKeys
         .perMonth({
           workspaceId: api.workspaceId,
           keySpaceId: api.keyAuthId!,
           start: billingCycleStart,
           end: billingCycleEnd,
         })
-        .then((res) => res.at(0)),
+        .then((res) => res.val!.at(0)),
       getVerificationsPerInterval({
         workspaceId: api.workspaceId,
         keySpaceId: api.keyAuthId!,
         start: billingCycleStart,
         end: billingCycleEnd,
-      }),
+      }).then((res) => res.val!),
     ]);
 
   const successOverTime: { x: string; y: number }[] = [];

diff --git a/apps/dashboard/app/(app)/identities/[identityId]/page.tsx b/apps/dashboard/app/(app)/identities/[identityId]/page.tsx
@@ -168,7 +168,7 @@ const LastUsed: React.FC<{ workspaceId: string; keySpaceId: string; keyId: strin
       keySpaceId: props.keySpaceId,
       keyId: props.keyId,
     })
-    .then((res) => res.at(0)?.time ?? null);
+    .then((res) => res.val?.at(0)?.time ?? null);
 
   return (
     <TableCell>

diff --git a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/logs/page.tsx b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/logs/page.tsx
@@ -122,10 +122,7 @@ const AuditLogTable: React.FC<{
 
     success: selected.success ?? undefined,
   };
-  const logs = await clickhouse.ratelimits.logs(query).catch((err) => {
-    console.error(err);
-    throw err;
-  });
+  const logs = await clickhouse.ratelimits.logs(query).then((res) => res.val!);
 
   if (logs.length === 0) {
     return (

diff --git a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/overrides/table.tsx b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/overrides/table.tsx
@@ -92,7 +92,7 @@ const LastUsed: React.FC<{
     identifier: [identifier],
   });
 
-  const unixMilli = lastUsed.at(0)?.time;
+  const unixMilli = lastUsed.val?.at(0)?.time;
   if (unixMilli) {
     return <span className="text-sm text-content-subtle">{ms(Date.now() - unixMilli)} ago</span>;
   }

diff --git a/apps/dashboard/app/(app)/ratelimits/[namespaceId]/page.tsx b/apps/dashboard/app/(app)/ratelimits/[namespaceId]/page.tsx
@@ -68,17 +68,17 @@ export default async function RatelimitNamespacePage(props: {
       .from(schema.ratelimitOverrides)
       .where(eq(schema.ratelimitOverrides.namespaceId, namespace.id))
       .execute()
-      .then((res) => res.at(0)?.count ?? 0),
-    getRatelimitsPerInterval(query),
+      .then((res) => res?.at(0)?.count ?? 0),
+    getRatelimitsPerInterval(query).then((res) => res.val!),
     getRatelimitsPerInterval({
       workspaceId: namespace.workspaceId,
       namespaceId: namespace.id,
       start: billingCycleStart,
       end: billingCycleEnd,
-    }),
+    }).then((res) => res.val!),
     clickhouse.ratelimits
       .latest({ workspaceId: namespace.workspaceId, namespaceId: namespace.id })
-      .then((res) => res.at(0)?.time),
+      .then((res) => res.val?.at(0)?.time),
   ]);
 
   const passedOverTime: { x: string; y: number }[] = [];

diff --git a/apps/dashboard/app/(app)/ratelimits/card.tsx b/apps/dashboard/app/(app)/ratelimits/card.tsx
@@ -17,15 +17,17 @@ export const RatelimitCard: React.FC<Props> = async ({ workspace, namespace }) =
   const intervalMs = 1000 * 60 * 60;
 
   const [history, lastUsed] = await Promise.all([
-    clickhouse.ratelimits.perMinute({
-      workspaceId: workspace.id,
-      namespaceId: namespace.id,
-      start: end - intervalMs,
-      end,
-    }),
+    clickhouse.ratelimits
+      .perMinute({
+        workspaceId: workspace.id,
+        namespaceId: namespace.id,
+        start: end - intervalMs,
+        end,
+      })
+      .then((res) => res.val!),
     clickhouse.ratelimits
       .latest({ workspaceId: workspace.id, namespaceId: namespace.id })
-      .then((res) => res.at(0)?.time),
+      .then((res) => res.val?.at(0)?.time),
   ]);
 
   const totalRequests = history.reduce((sum, d) => sum + d.total, 0);

diff --git a/apps/dashboard/app/(app)/settings/root-keys/[keyId]/history/page.tsx b/apps/dashboard/app/(app)/settings/root-keys/[keyId]/history/page.tsx
@@ -39,11 +39,13 @@ export default async function HistoryPage(props: {
   if (!key?.keyAuth?.api) {
     return notFound();
   }
-  const history = await clickhouse.verifications.logs({
-    workspaceId: UNKEY_WORKSPACE_ID,
-    keySpaceId: key.keyAuthId,
-    keyId: key.id,
-  });
+  const history = await clickhouse.verifications
+    .logs({
+      workspaceId: UNKEY_WORKSPACE_ID,
+      keySpaceId: key.keyAuthId,
+      keyId: key.id,
+    })
+    .then((res) => res.val!);
 
   return <AccessTable verifications={history} />;
 }
diff --git a/apps/dashboard/app/(app)/settings/root-keys/[keyId]/layout.tsx b/apps/dashboard/app/(app)/settings/root-keys/[keyId]/layout.tsx
@@ -84,7 +84,7 @@ const LastUsed: React.FC<{ workspaceId: string; keySpaceId: string; keyId: strin
 }) => {
   const lastUsed = await clickhouse.verifications
     .latest({ workspaceId, keySpaceId, keyId })
-    .then((res) => res.at(0)?.time ?? 0);
+    .then((res) => res.val?.at(0)?.time ?? 0);
 
   return (
     <Metric label="Last Used" value={lastUsed ? `${ms(Date.now() - lastUsed)} ago` : "Never"} />

diff --git a/apps/dashboard/app/(app)/settings/root-keys/[keyId]/page.tsx b/apps/dashboard/app/(app)/settings/root-keys/[keyId]/page.tsx
@@ -91,11 +91,13 @@ export default async function RootKeyPage(props: {
   if (!keyForHistory?.keyAuth?.api) {
     return notFound();
   }
-  const history = await clickhouse.verifications.latest({
-    workspaceId: UNKEY_WORKSPACE_ID,
-    keySpaceId: key.keyAuthId,
-    keyId: key.id,
-  });
+  const history = await clickhouse.verifications
+    .latest({
+      workspaceId: UNKEY_WORKSPACE_ID,
+      keySpaceId: key.keyAuthId,
+      keyId: key.id,
+    })
+    .then((res) => res.val!);
 
   const apis = workspace.apis.map((api) => {
     const apiPermissionsStructure = apiPermissions(api.id);

diff --git a/apps/dashboard/app/(app)/settings/root-keys/[keyId]/permissions/add-permission-for-api.tsx b/apps/dashboard/app/(app)/settings/root-keys/[keyId]/permissions/add-permission-for-api.tsx
@@ -9,7 +9,7 @@ import {
   SelectValue,
 } from "@/components/ui/select";
 import type { Permission } from "@unkey/db";
-import { type PropsWithChildren, useMemo, useState } from "react";
+import { type PropsWithChildren, useState } from "react";
 import { PermissionToggle } from "./permission_toggle";
 import { apiPermissions } from "./permissions";
 
@@ -38,10 +38,7 @@ export function DialogAddPermissionsForAPI(
   });
 
   const [selectedApiId, setSelectedApiId] = useState<string>("");
-  const selectedApi = useMemo(
-    () => props.apis.find((api) => api.id === selectedApiId),
-    [selectedApiId],
-  );
+  const selectedApi = props.apis.find((api) => api.id === selectedApiId);
 
   const isSelectionDisabled =
     selectedApi && !apisWithoutPermission.some((api) => api.id === selectedApi.id);

diff --git a/apps/dashboard/app/(app)/success/page.tsx b/apps/dashboard/app/(app)/success/page.tsx
@@ -64,7 +64,7 @@ export default async function SuccessPage() {
     });
   }
 
-  const activeWorkspaces = await clickhouse.business.activeWorkspaces();
+  const activeWorkspaces = await clickhouse.business.activeWorkspaces().then((res) => res.val!);
   const chartData = activeWorkspaces.map(({ time, workspaces }) => ({
     x: new Date(time).toLocaleDateString(),
     y: workspaces,

diff --git a/internal/clickhouse/package.json b/internal/clickhouse/package.json
@@ -18,6 +18,7 @@
   },
   "dependencies": {
     "@clickhouse/client-web": "^1.6.0",
+    "@unkey/error": "workspace:^",
     "zod": "^3.23.8"
   }
 }
diff --git a/internal/clickhouse/src/billing.ts b/internal/clickhouse/src/billing.ts
@@ -30,10 +30,10 @@ export function getBillableRatelimits(ch: Querier) {
     });
 
     const res = await query(args);
-    if (!res) {
+    if (res.err || res.val.length === 0) {
       return 0;
     }
-    return res.at(0)?.count ?? 0;
+    return res.val.at(0)?.count ?? 0;
   };
 }
 // get the billable verifications for a workspace in a specific month.
@@ -65,9 +65,9 @@ export function getBillableVerifications(ch: Querier) {
     });
 
     const res = await query(args);
-    if (!res) {
+    if (res.err || res.val.length === 0) {
       return 0;
     }
-    return res.at(0)?.count ?? 0;
+    return res.val.at(0)?.count ?? 0;
   };
 }
diff --git a/internal/clickhouse/src/client/client.ts b/internal/clickhouse/src/client/client.ts
@@ -1,7 +1,8 @@
 import { type ClickHouseClient, createClient } from "@clickhouse/client-web";
+import { Err, Ok, type Result } from "@unkey/error";
 import { z } from "zod";
+import { InsertError, QueryError } from "./error";
 import type { Inserter, Querier } from "./interface";
-
 export type Config = {
   url: string;
 };
@@ -12,6 +13,7 @@ export class Client implements Querier, Inserter {
   constructor(config: Config) {
     this.client = createClient({
       url: config.url,
+
       clickhouse_settings: {
         async_insert: 1,
         wait_for_async_insert: 1,
@@ -32,11 +34,11 @@ export class Client implements Querier, Inserter {
     // The schema of the output of each row
     // Example: z.object({ id: z.string() })
     schema: TOut;
-  }): (params: z.input<TIn>) => Promise<z.output<TOut>[]> {
-    return async (params: z.input<TIn>): Promise<z.output<TOut>[]> => {
+  }): (params: z.input<TIn>) => Promise<Result<z.output<TOut>[], QueryError>> {
+    return async (params: z.input<TIn>): Promise<Result<z.output<TOut>[], QueryError>> => {
       const validParams = req.params?.safeParse(params);
       if (validParams?.error) {
-        throw new Error(`Bad params: ${validParams.error.message}`);
+        return Err(new QueryError(`Bad params: ${validParams.error.message}`, { query: "" }));
       }
       const res = await this.client
         .query({
@@ -48,7 +50,11 @@ export class Client implements Querier, Inserter {
           throw new Error(`${err.message} ${req.query}, params: ${JSON.stringify(params)}`);
         });
       const rows = await res.json();
-      return z.array(req.schema).parse(rows);
+      const parsed = z.array(req.schema).safeParse(rows);
+      if (parsed.error) {
+        return Err(new QueryError(`Malformed data: ${parsed.error.message}`, { query: req.query }));
+      }
+      return Ok(parsed.data);
     };
   }
 
@@ -57,22 +63,40 @@ export class Client implements Querier, Inserter {
     schema: TSchema;
   }): (
     events: z.input<TSchema> | z.input<TSchema>[],
-  ) => Promise<{ executed: boolean; query_id: string }> {
+  ) => Promise<Result<{ executed: boolean; query_id: string }, InsertError>> {
     return async (events: z.input<TSchema> | z.input<TSchema>[]) => {
       let validatedEvents: z.output<TSchema> | z.output<TSchema>[] | undefined = undefined;
       const v = Array.isArray(events)
         ? req.schema.array().safeParse(events)
         : req.schema.safeParse(events);
       if (!v.success) {
-        throw new Error(v.error.message);
+        return Err(new InsertError(v.error.message));
       }
       validatedEvents = v.data;
 
-      return await this.client.insert({
-        table: req.table,
-        format: "JSONEachRow",
-        values: Array.isArray(validatedEvents) ? validatedEvents : [validatedEvents],
-      });
+      return this.retry(() =>
+        this.client
+          .insert({
+            table: req.table,
+            format: "JSONEachRow",
+            values: Array.isArray(validatedEvents) ? validatedEvents : [validatedEvents],
+          })
+          .then((res) => Ok(res))
+          .catch((err) => Err(new InsertError(err.message))),
+      );
     };
   }
+
+  private async retry<T>(fn: (attempt: number) => Promise<T>): Promise<T> {
+    let err: Error | undefined = undefined;
+    for (let i = 1; i <= 3; i++) {
+      try {
+        return fn(i);
+      } catch (e) {
+        console.warn(e);
+        err = e as Error;
+      }
+    }
+    throw err;
+  }
 }