feat: ingesting analytics tags (#2698)

* wip: analytics tags

* [autofix.ci] apply automated fixes

* feat: add verifications_per_hour_v2

* feat: insert tags

* docs: specify 10 tags per request

* test: run serially cause gh actions suck

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

.github/workflows/deploy.yaml

@@ -129,6 +129,7 @@ jobs:
     needs:
       - api_canary_test
       - agent_production_deployment
+
     uses: ./.github/workflows/job_deploy_api_production.yaml
     secrets:
       CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
@@ -170,3 +171,17 @@ jobs:
     uses: ./.github/workflows/job_deploy_api_enterprise.yaml
     secrets:
       CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+
+  clickhouse_migration_preview:
+    needs: api_local_test
+    uses: ./.github/workflows/job_clickhouse_migration_preview.yaml
+    secrets:
+      CLICKHOUSE_URL: ${{ secrets.CLICKHOUSE_URL }}
+
+  clickhouse_migration_production:
+    needs:
+      - api_preview_test
+      - agent_staging_deployment
+    uses: ./.github/workflows/job_clickhouse_migration_production.yaml
+    secrets:
+      CLICKHOUSE_URL: ${{ secrets.CLICKHOUSE_URL }}

.github/workflows/job_clickhouse_migration_preview.yaml

@@ -0,0 +1,31 @@
+name: ClickHouse Migration Production
+on:
+  workflow_call:
+    secrets:
+      CLICKHOUSE_URL:
+        required: true
+
+
+
+jobs:
+  deploy:
+    environment: ClickHouse Production Migration
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install
+        uses: ./.github/actions/install
+        with:
+          go: true
+
+      - name: Install goose
+        run: go install github.com/pressly/goose/v3/cmd/goose@latest
+
+
+      - name: Deploy
+        run: goose up
+        working-directory: internal/clickhouse/schema
+        env:
+          GOOSE_DRIVER: clickhouse
+          GOOSE_DB_STRING: ${{ secrets.CLICKHOUSE_URL }}

.github/workflows/job_clickhouse_migration_production.yaml

@@ -0,0 +1,31 @@
+name: ClickHouse Migration Preview
+on:
+  workflow_call:
+    secrets:
+      CLICKHOUSE_URL:
+        required: true
+
+
+
+jobs:
+  deploy:
+    environment: Preview
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install
+        uses: ./.github/actions/install
+        with:
+          go: true
+
+      - name: Install gooes
+        run: go install github.com/pressly/goose/v3/cmd/goose@latest
+
+
+      - name: Deploy
+        run: goose up
+        working-directory: internal/clickhouse/schema
+        env:
+          GOOSE_DRIVER: clickhouse
+          GOOSE_DB_STRING: ${{ secrets.CLICKHOUSE_URL }}

.github/workflows/job_deploy_api_production.yaml

@@ -1,11 +1,11 @@
-name: Deploy API Production
+name: ClickHouse Migration
 on:
   workflow_call:
-    secrets: 
-      CLOUDFLARE_API_TOKEN: 
+    secrets:
+      CLOUDFLARE_API_TOKEN:
         required: true
 
-    
+
 
 jobs:
   deploy:
@@ -26,4 +26,4 @@ jobs:
         run: wrangler deploy --env=production --var VERSION:$(git rev-parse --short HEAD)
         working-directory: apps/api
         env:
-          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}

Taskfile.yml

@@ -26,6 +26,13 @@ tasks:
       GOOSE_MIGRATION_DIR: ./internal/clickhouse/schema
     cmds:
       - goose up
+  migrate-clickhouse-reset:
+    env:
+      GOOSE_DRIVER: clickhouse
+      GOOSE_DBSTRING: "tcp://default:password@127.0.0.1:9000"
+      GOOSE_MIGRATION_DIR: ./internal/clickhouse/schema
+    cmds:
+      - goose down-to 0
 
 
   migrate-db:

apps/api/src/pkg/auth/root_key.ts

@@ -62,6 +62,7 @@ export async function rootKeyAuth(c: Context<HonoEnv>, permissionQuery?: Permiss
       // @ts-expect-error - the cf object will be there on cloudflare
       region: c.req.cf?.region,
       request_id: c.get("requestId"),
+      tags: [],
     }),
   );
 

apps/api/src/routes/v1_keys_verifyKey.ts

@@ -30,6 +30,22 @@ The key will be verified against the api's configuration. If the key does not be
                 description: "The key to verify",
                 example: "sk_1234",
               }),
+
+              tags: z
+                .array(z.string().min(1).max(128))
+                .max(10)
+                .optional()
+                .openapi({
+                  description: `Tags do not influence the outcome of a verification.
+                They can be added to filter or aggregate historical verification data for your analytics needs.
+
+                To unkey, a tag is simply a string, we don't enforce any schema but leave that up to you.
+                The only exception is that each tag must be between 1 and 128 characters long.
+
+                A typical setup would be to add key-value pairs of resources or locations, that you need later when querying.
+                `,
+                  example: ["path=/v1/users/123", "region=us-east-1"],
+                }),
               authorization: z
                 .object({
                   permissions: z.any(permissionQuerySchema).openapi("PermissionQuery", {
@@ -362,6 +378,7 @@ export const registerV1KeysVerifyKey = (app: App) =>
         region: c.req.raw.cf.colo ?? "",
         outcome: val.code ?? "VALID",
         identity_id: val.identity?.id,
+        tags: req.tags ?? [],
       }),
     );
 

apps/docs/apis/features/analytics.mdx

@@ -21,3 +21,61 @@ Our per key analytics give you a deep dive into each individual key, giving usag
 <Frame caption="Per key Analytics">
   <img src="/images/per-key-analytics.png" alt="Per key analytics"/>
 </Frame>
+
+
+## Tags
+
+You can add tags to verification requests to aggregate or filter data when querying.
+
+For example you might want to add the path of your API as a tag: `path=/v1/my-path` and then later retrieve a breakdown of api key usage per unique path.
+
+Unkey does not parse tags in any special way.
+The only limitations are:
+- Tags are strings.
+- You can specify up to 10 tags per request.
+- Each tag must be between (including) 1 and 128 characters.
+
+That being said, having some structure for your tags could be benefitial for you.
+A common theme is treating them as key-value pairs and specifying them with either a colon or equal-sign as delimiter.
+```
+key:value
+key=value
+```
+
+Since Unkey does not know what API routes your user has called or which resources they interact with, we encourage you to record these in the tags.
+Here's an example tags for a fictional blog API:
+
+
+```json
+[
+  "path=/v1/posts.createPost",
+  "postId=post_1asofijqknslkfqWF",
+  "region=us-east-1",
+  "apiVersion=f8ad21bd", // a git sha of your deployment or semver
+]
+```
+
+### Using tags
+
+Tags can be added in the request body.
+
+```bash Providing Tags {7-10}
+curl --request POST \
+  --url https://api.unkey.dev/v1/keys.verifyKey \
+  --header 'Content-Type: application/json' \
+  --data '{
+  "apiId": "api_1234",
+  "key": "sk_1234",
+  "tags": [
+    "tag1",
+    "path=/v1/my-resource/123"
+  ]
+}'
+```
+
+
+### Querying tags
+
+We have only rolled out tag ingestion so far to allow you to start recording data as early as possible.
+
+We're working on new query capabilities including filtering and aggregating by tags.

apps/docs/mint.json

@@ -413,6 +413,9 @@
   "api": {
     "baseUrl": "https://api.unkey.dev"
   },
+  "codeBlock": {
+    "mode": "auto"
+  },
   "redirects": [
     {
       "source": "/onboarding",

apps/docs/package.json

@@ -8,7 +8,7 @@
   "keywords": [],
   "author": "Andreas Thomas & James Perkins",
   "devDependencies": {
-    "mintlify": "^4.0.182"
+    "mintlify": "^4.0.289"
   },
   "dependencies": {
     "sharp": "^0.33.4"

internal/clickhouse/schema/027_add_tags_to_verifications.raw_key_verifications_v1.sql

@@ -0,0 +1,11 @@
+-- +goose up
+ALTER TABLE verifications.raw_key_verifications_v1
+ADD COLUMN IF NOT EXISTS tags Array(String) DEFAULT [];
+
+
+-- +goose down
+
+
+
+ALTER TABLE verifications.raw_key_verifications_v1
+DROP COLUMN IF EXISTS tags;

internal/clickhouse/schema/028_create_verifications.key_verifications_per_hour_v2.sql

@@ -0,0 +1,19 @@
+-- +goose up
+CREATE TABLE verifications.key_verifications_per_hour_v2
+(
+  time          DateTime,
+  workspace_id  String,
+  key_space_id  String,
+  identity_id   String,
+  key_id        String,
+  outcome       LowCardinality(String),
+  tags          Array(String),
+  count         Int64
+)
+ENGINE = SummingMergeTree()
+ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags)
+;
+
+
+-- +goose down
+DROP TABLE verifications.key_verifications_per_hour_v2;

internal/clickhouse/schema/029_create_verifications.key_verifications_per_hour_mv_v2.sql

@@ -0,0 +1,27 @@
+-- +goose up
+CREATE MATERIALIZED VIEW verifications.key_verifications_per_hour_mv_v2
+TO verifications.key_verifications_per_hour_v2
+AS
+SELECT
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  count(*) as count,
+  toStartOfHour(fromUnixTimestamp64Milli(time)) AS time,
+  tags
+FROM verifications.raw_key_verifications_v1
+GROUP BY
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  time,
+  tags
+;
+
+
+-- +goose down
+DROP VIEW verifications.key_verifications_per_hour_mv_v2;

internal/clickhouse/schema/030_create_verifications.key_verifications_per_day_v2.sql

@@ -0,0 +1,19 @@
+-- +goose up
+CREATE TABLE verifications.key_verifications_per_day_v2
+(
+  time          DateTime,
+  workspace_id  String,
+  key_space_id  String,
+  identity_id   String,
+  key_id        String,
+  outcome       LowCardinality(String),
+  tags          Array(String),
+  count         Int64
+)
+ENGINE = SummingMergeTree()
+ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags)
+;
+
+
+-- +goose down
+DROP TABLE verifications.key_verifications_per_day_v2;

internal/clickhouse/schema/031_create_verifications.key_verifications_per_day_mv_v2.sql

@@ -0,0 +1,27 @@
+-- +goose up
+CREATE MATERIALIZED VIEW verifications.key_verifications_per_day_mv_v2
+TO verifications.key_verifications_per_day_v2
+AS
+SELECT
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  count(*) as count,
+  toStartOfDay(fromUnixTimestamp64Milli(time)) AS time,
+  tags
+FROM verifications.raw_key_verifications_v1
+GROUP BY
+  workspace_id,
+  key_space_id,
+  identity_id,
+  key_id,
+  outcome,
+  time,
+  tags
+;
+
+
+-- +goose down
+DROP VIEW verifications.key_verifications_per_day_mv_v2;

internal/clickhouse/schema/032_create_verifications.key_verifications_per_month_v2.sql

@@ -0,0 +1,19 @@
+-- +goose up
+CREATE TABLE verifications.key_verifications_per_month_v2
+(
+  time          DateTime,
+  workspace_id  String,
+  key_space_id  String,
+  identity_id   String,
+  key_id        String,
+  outcome       LowCardinality(String),
+  tags          Array(String),
+  count         Int64
+)
+ENGINE = SummingMergeTree()
+ORDER BY (workspace_id, key_space_id, identity_id, key_id, time, tags)
+;
+
+
+-- +goose down
+DROP TABLE verifications.key_verifications_per_month_v2;