Merge branch 'main' into harshbhat/unkey-firecrawl-streamlit

.github/workflows/pr-alerts-campsite.yml

@@ -0,0 +1,53 @@
+name: Campsite PR Alerts
+
+on:
+  pull_request:
+    types: [opened, closed, merged, ready_for_review, reopened]
+
+jobs:
+  post_to_campsite:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '20'
+
+      - name: Post to Campsite
+        id: post_to_campsite
+        run: |
+          if [[ "${{ github.event.pull_request.draft }}" == "true" ]]; then
+            STATUS_EMOJI="⚪"
+          elif [[ "${{ github.event.action }}" == "opened" || "${{ github.event.action }}" == "reopened" || "${{ github.event.action }}" == "ready_for_review" ]]; then
+            STATUS_EMOJI="🟢"
+          elif [[ "${{ github.event.action }}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
+            STATUS_EMOJI="🟣"
+          elif [[ "${{ github.event.action }}" == "closed" ]]; then
+            STATUS_EMOJI="🔴"
+          fi
+
+          ACTION=${{ github.event.action }}
+          if [[ "${ACTION}" == "ready_for_review" ]]; then
+            ACTION="ready for review"
+          elif [[ "${ACTION}" == "reopened" ]]; then
+            ACTION="reopened"
+          elif [[ "${ACTION}" == "closed" && "${{ github.event.pull_request.merged }}" == "true" ]]; then
+            ACTION="merged"
+          fi
+
+          ESCAPED_TITLE=$(echo "${{ github.event.pull_request.title }}" | jq -Rr @json)
+
+          CONTENT="${STATUS_EMOJI} Pull request ${ACTION} by ${{ github.event.pull_request.user.login }}: [#${{ github.event.pull_request.number }} ${ESCAPED_TITLE}](${{ github.event.pull_request.html_url }})"
+
+          echo "content=${CONTENT}" >> $GITHUB_OUTPUT
+      - name: Create Campsite message
+        uses: campsite/campsite-github-action@v1
+        with:
+          api_key: ${{ secrets.CAMPSITE_API_KEY }}
+          action_type: create_message
+          thread_id: ${{ secrets.CAMPSITE_PR_ALERTS_THREAD_ID }}
+          content: ${{ steps.post_to_campsite.outputs.content }}

apps/api/CHANGELOG.md

@@ -1,5 +1,11 @@
 # api
 
+## 0.1.0
+
+### Minor Changes
+
+- 09d36ad: add /v1/keys.whoami route
+
 ## 0.0.14
 
 ### Patch Changes

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "api",
-  "version": "0.0.14",
+  "version": "0.1.0",
   "private": true,
   "scripts": {
     "build": "tsc",

apps/api/src/pkg/env.ts

@@ -1,6 +1,10 @@
 import { z } from "zod";
 import type { MessageBody } from "./key_migration/message";
 
+export const cloudflareRatelimiter = z.custom<{
+  limit: (opts: { key: string }) => Promise<{ success: boolean }>;
+}>((r) => typeof r.limit === "function");
+
 export const zEnv = z.object({
   VERSION: z.string().default("unknown"),
   DATABASE_HOST: z.string(),
@@ -42,6 +46,11 @@ export const zEnv = z.object({
         return 0;
       }
     }),
+  RL_10_60s: cloudflareRatelimiter,
+  RL_30_60s: cloudflareRatelimiter,
+  RL_200_60s: cloudflareRatelimiter,
+  RL_500_10s: cloudflareRatelimiter,
+  RL_200_10s: cloudflareRatelimiter,
 });
 
 export type Env = z.infer<typeof zEnv>;

apps/api/src/pkg/key_migration/handler.ts

@@ -96,6 +96,7 @@ export async function migrateKey(
         expires: message.expires ? new Date(message.expires) : null,
         refillInterval: message.refill?.interval,
         refillAmount: message.refill?.amount,
+        refillDay: message.refill?.refillDay,
         enabled: message.enabled,
         remaining: message.remaining,
         ratelimitAsync: message.ratelimit?.async,

apps/api/src/pkg/key_migration/message.ts

@@ -14,7 +14,7 @@ export type MessageBody = {
   permissions?: string[];
   expires?: number;
   remaining?: number;
-  refill?: { interval: "daily" | "monthly"; amount: number };
+  refill?: { interval: "daily" | "monthly"; amount: number; refillDay?: number };
   ratelimit?: { async: boolean; limit: number; duration: number };
   enabled: boolean;
   environment?: string;

apps/api/src/pkg/keys/service.ts

@@ -57,7 +57,11 @@ type InvalidResponse = {
     | "DISABLED"
     | "INSUFFICIENT_PERMISSIONS";
   key: Key;
-  identity: { id: string; externalId: string; meta: Record<string, unknown> | null } | null;
+  identity: {
+    id: string;
+    externalId: string;
+    meta: Record<string, unknown> | null;
+  } | null;
   api: Api;
   ratelimit?: {
     remaining: number;
@@ -73,7 +77,11 @@ type ValidResponse = {
   code?: never;
   valid: true;
   key: Key;
-  identity: { id: string; externalId: string; meta: Record<string, unknown> | null } | null;
+  identity: {
+    id: string;
+    externalId: string;
+    meta: Record<string, unknown> | null;
+  } | null;
   api: Api;
   ratelimit?: {
     remaining: number;
@@ -451,6 +459,7 @@ export class KeyService {
 
     if (data.api.ipWhitelist) {
       const ip = c.req.header("True-Client-IP") ?? c.req.header("CF-Connecting-IP");
+
       if (!ip) {
         return Ok({
           key: data.key,
@@ -461,7 +470,8 @@ export class KeyService {
           permissions: data.permissions,
         });
       }
-      const ipWhitelist = JSON.parse(data.api.ipWhitelist) as string[];
+
+      const ipWhitelist = data.api.ipWhitelist.split(",").map((s) => s.trim());
       if (!ipWhitelist.includes(ip)) {
         return Ok({
           key: data.key,

apps/api/src/pkg/ratelimit/client.ts

@@ -1,8 +1,8 @@
 import { Err, Ok, type Result } from "@unkey/error";
 import type { Logger } from "@unkey/worker-logging";
-import type { Metrics } from "../metrics";
-
+import { cloudflareRatelimiter } from "../env";
 import type { Context } from "../hono/app";
+import type { Metrics } from "../metrics";
 import { retry } from "../util/retry";
 import { Agent } from "./agent";
 import {
@@ -11,7 +11,6 @@ import {
   type RatelimitRequest,
   type RatelimitResponse,
 } from "./interface";
-
 export class AgentRatelimiter implements RateLimiter {
   private readonly logger: Logger;
   private readonly metrics: Metrics;
@@ -67,7 +66,34 @@ export class AgentRatelimiter implements RateLimiter {
     req: RatelimitRequest,
   ): Promise<Result<RatelimitResponse, RatelimitError>> {
     const start = performance.now();
+    try {
+      if (req.async) {
+        // Construct a binding key that could match a configured ratelimiter
+        const lookup = `RL_${req.limit}_${Math.round(req.interval / 1000)}s` as keyof typeof c.env;
+        const binding = c.env[lookup];
+
+        if (binding) {
+          const res = await cloudflareRatelimiter.parse(binding).limit({ key: req.identifier });
 
+          this.metrics.emit({
+            metric: "metric.ratelimit",
+            workspaceId: req.workspaceId,
+            namespaceId: req.namespaceId,
+            latency: performance.now() - start,
+            identifier: req.identifier,
+            mode: "async",
+            error: false,
+            success: res.success,
+            source: "cloudflare",
+          });
+          return Ok({ pass: res.success, reset: -1, current: -1, remaining: -1, triggered: null });
+        }
+      }
+    } catch (err) {
+      this.logger.error("cfrl failed, falling back to agent", {
+        error: (err as Error).message,
+      });
+    }
     const res = await this._limit(c, req);
     this.metrics.emit({
       metric: "metric.ratelimit",

apps/api/src/routes/legacy_keys_verifyKey.test.ts

@@ -129,7 +129,7 @@ describe("with ip whitelist", () => {
         name: "test",
         authType: "key",
         keyAuthId: keyAuthId,
-        ipWhitelist: JSON.stringify(["100.100.100.100"]),
+        ipWhitelist: ["100.100.100.100"].join(","),
         createdAt: new Date(),
         deletedAt: null,
       });
@@ -177,7 +177,7 @@ describe("with ip whitelist", () => {
         name: "test",
         authType: "key",
         keyAuthId: keyAuthid,
-        ipWhitelist: JSON.stringify(["100.100.100.100"]),
+        ipWhitelist: ["100.100.100.100"].join(","),
         createdAt: new Date(),
         deletedAt: null,
       });

apps/api/src/routes/schema.ts

@@ -59,13 +59,19 @@ export const keySchema = z
     refill: z
       .object({
         interval: z.enum(["daily", "monthly"]).openapi({
-          description: "Determines the rate at which verifications will be refilled.",
+          description:
+            "Determines the rate at which verifications will be refilled. When 'daily' is set for 'interval' 'refillDay' will be set to null.",
           example: "daily",
         }),
         amount: z.number().int().openapi({
           description: "Resets `remaining` to this value every interval.",
           example: 100,
         }),
+        refillDay: z.number().min(1).max(31).default(1).nullable().openapi({
+          description:
+            "The day verifications will refill each month, when interval is set to 'monthly'. Value is not zero-indexed making 1 the first day of the month. If left blank it will default to the first day of the month. When 'daily' is set for 'interval' 'refillDay' will be set to null.",
+          example: 15,
+        }),
         lastRefillAt: z.number().int().optional().openapi({
           description: "The unix timestamp in miliseconds when the key was last refilled.",
           example: 100,
@@ -76,10 +82,12 @@ export const keySchema = z
         description:
           "Unkey allows you to refill remaining verifications on a key on a regular interval.",
         example: {
-          interval: "daily",
+          interval: "monthly",
           amount: 10,
+          refillDay: 10,
         },
       }),
+
     ratelimit: z
       .object({
         async: z.boolean().openapi({

apps/api/src/routes/v1_apis_getApi.happy.test.ts

@@ -31,7 +31,7 @@ test("with ip whitelist", async (t) => {
     id: newId("api"),
     name: "with ip whitelist",
     workspaceId: h.resources.userWorkspace.id,
-    ipWhitelist: JSON.stringify(["127.0.0.1"]),
+    ipWhitelist: ["127.0.0.1"].join(","),
     createdAt: new Date(),
     deletedAt: null,
   };

apps/api/src/routes/v1_apis_listKeys.ts

@@ -317,6 +317,7 @@ export const registerV1ApisListKeys = (app: App) =>
             ? {
                 interval: k.refillInterval,
                 amount: k.refillAmount,
+                refillDay: k.refillInterval === "monthly" && k.refillDay ? k.refillDay : null,
                 lastRefillAt: k.lastRefillAt?.getTime(),
               }
             : undefined,

apps/api/src/routes/v1_keys_createKey.error.test.ts

@@ -11,7 +11,6 @@ import type { V1KeysCreateKeyRequest, V1KeysCreateKeyResponse } from "./v1_keys_
 test("when the api does not exist", async (t) => {
   const h = await IntegrationHarness.init(t);
   const apiId = newId("api");
-
   const root = await h.createRootKey([`api.${apiId}.create_key`]);
   /* The code snippet is making a POST request to the "/v1/keys.createKey" endpoint with the specified headers. It is using the `h.post` method from the `Harness` instance to send the request. The generic types `<V1KeysCreateKeyRequest, V1KeysCreateKeyResponse>` specify the request payload and response types respectively. */
 
@@ -119,3 +118,35 @@ test("when key recovery is not enabled", async (t) => {
     },
   });
 });
+
+test("reject invalid refill config when daily interval has non-null refillDay", async (t) => {
+  const h = await IntegrationHarness.init(t);
+
+  const root = await h.createRootKey([`api.${h.resources.userApi.id}.create_key`]);
+
+  const res = await h.post<V1KeysCreateKeyRequest, V1KeysCreateKeyResponse>({
+    url: "/v1/keys.createKey",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${root.key}`,
+    },
+    body: {
+      byteLength: 16,
+      apiId: h.resources.userApi.id,
+      remaining: 10,
+      refill: {
+        amount: 100,
+        refillDay: 4,
+        interval: "daily",
+      },
+    },
+  });
+  expect(res.status).toEqual(400);
+  expect(res.body).toMatchObject({
+    error: {
+      code: "BAD_REQUEST",
+      docs: "https://unkey.dev/docs/api-reference/errors/code/BAD_REQUEST",
+      message: "when interval is set to 'daily', 'refillDay' must be null.",
+    },
+  });
+});

apps/api/src/routes/v1_keys_createKey.happy.test.ts

@@ -467,4 +467,35 @@ describe("with externalId", () => {
       expect(key!.identity!.id).toEqual(identity.id);
     });
   });
+  describe("Should default first day of month if none provided", () => {
+    test("should provide default value", async (t) => {
+      const h = await IntegrationHarness.init(t);
+      const root = await h.createRootKey([`api.${h.resources.userApi.id}.create_key`]);
+
+      const res = await h.post<V1KeysCreateKeyRequest, V1KeysCreateKeyResponse>({
+        url: "/v1/keys.createKey",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${root.key}`,
+        },
+        body: {
+          apiId: h.resources.userApi.id,
+          remaining: 10,
+          refill: {
+            interval: "monthly",
+            amount: 20,
+            refillDay: undefined,
+          },
+        },
+      });
+
+      expect(res.status, `expected 200, received: ${JSON.stringify(res, null, 2)}`).toBe(200);
+
+      const key = await h.db.primary.query.keys.findFirst({
+        where: (table, { eq }) => eq(table.id, res.body.keyId),
+      });
+      expect(key).toBeDefined();
+      expect(key!.refillDay).toEqual(1);
+    });
+  });
 });