From 11df938932069f0231c6a0ecb45c5b1105798d32 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:59:06 +0400 Subject: [PATCH 01/13] adds logging and tracking through workers and DO --- apps/mail/app/(routes)/demo/logging/page.tsx | 62 ++++++ apps/mail/app/(routes)/demo/page.tsx | 19 ++ apps/mail/app/routes.ts | 2 + apps/mail/components/logging-dashboard.tsx | 198 ++++++++++++++++++ apps/mail/lib/trpc.ts | 27 +++ apps/server/src/env.ts | 2 + apps/server/src/lib/logging-durable-object.ts | 141 +++++++++++++ apps/server/src/lib/server-utils.ts | 18 +- apps/server/src/lib/trpc-logging.ts | 89 ++++++++ apps/server/src/main.ts | 3 + apps/server/src/trpc/index.ts | 4 + apps/server/src/trpc/routes/logging.ts | 39 ++++ apps/server/src/trpc/trpc.ts | 67 +++++- apps/server/wrangler.jsonc | 5 + 14 files changed, 667 insertions(+), 9 deletions(-) create mode 100644 apps/mail/app/(routes)/demo/logging/page.tsx create mode 100644 apps/mail/app/(routes)/demo/page.tsx create mode 100644 apps/mail/components/logging-dashboard.tsx create mode 100644 apps/mail/lib/trpc.ts create mode 100644 apps/server/src/lib/logging-durable-object.ts create mode 100644 apps/server/src/lib/trpc-logging.ts create mode 100644 apps/server/src/trpc/routes/logging.ts diff --git a/apps/mail/app/(routes)/demo/logging/page.tsx b/apps/mail/app/(routes)/demo/logging/page.tsx new file mode 100644 index 0000000000..df2a661f31 --- /dev/null +++ b/apps/mail/app/(routes)/demo/logging/page.tsx @@ -0,0 +1,62 @@ +import { api } from '@/lib/trpc'; +import { useEffect, useState } from 'react'; + +export default function LoggingDemoPage() { + const [sessionStats, setSessionStats] = useState(null); + const [callHistory, setCallHistory] = useState(null); + const [loading, setLoading] = useState(true); + + useEffect(() => { + const fetchData = async () => { + try { + // Fetch session stats + const statsResponse = await fetch('http://localhost:8787/api/trpc/logging.getSessionStats', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + }); + const statsData = await statsResponse.json(); + setSessionStats(statsData); + + // Fetch call history + const historyResponse = await fetch('http://localhost:8787/api/trpc/logging.getCallHistory', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'include', + body: JSON.stringify({ + json: { limit: 100 } + }), + }); + const historyData = await historyResponse.json(); + setCallHistory(historyData); + } catch (error) { + console.error('Error fetching data:', error); + } finally { + setLoading(false); + } + }; + + fetchData(); + }, []); + + if (loading) { + return
Loading...
; + } + + const jsonData = { + sessionStats, + callHistory, + timestamp: new Date().toISOString(), + }; + + return ( +
+

Session Logging JSON Output

+
+ 
+                    {JSON.stringify(jsonData, null, 2)}
+
+
+
+ ); +} \ No newline at end of file diff --git a/apps/mail/app/(routes)/demo/page.tsx b/apps/mail/app/(routes)/demo/page.tsx new file mode 100644 index 0000000000..c5b1e1d637 --- /dev/null +++ b/apps/mail/app/(routes)/demo/page.tsx @@ -0,0 +1,19 @@ +import { Link } from 'react-router'; + +export default function DemoPage() { + return ( +
+

TRPC Call Logging

+

+ Simple JSON output of all TRPC calls with routes, inputs, and outputs. +

+ +
+

Session Logging

+ + /demo/logging - View all TRPC calls JSON + +
+
+ ); +} \ No newline at end of file diff --git a/apps/mail/app/routes.ts b/apps/mail/app/routes.ts index 57d9fbe23f..82a5cb3e40 100644 --- a/apps/mail/app/routes.ts +++ b/apps/mail/app/routes.ts @@ -23,6 +23,8 @@ export default [ layout('(routes)/layout.tsx', [ route('/developer', '(routes)/developer/page.tsx'), + route('/demo', '(routes)/demo/page.tsx'), + route('/demo/logging', '(routes)/demo/logging/page.tsx'), layout( '(routes)/mail/layout.tsx', prefix('/mail', [ diff --git a/apps/mail/components/logging-dashboard.tsx b/apps/mail/components/logging-dashboard.tsx new file mode 100644 index 0000000000..efda45094e --- /dev/null +++ b/apps/mail/components/logging-dashboard.tsx @@ -0,0 +1,198 @@ +'use client'; + +import { api } from '@/lib/trpc'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Button } from '@/components/ui/button'; +import { Badge } from '@/components/ui/badge'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { formatDistanceToNow } from 'date-fns'; +import { RefreshCw, Trash2, Activity } from 'lucide-react'; +import { useState } from 'react'; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from '@/components/ui/alert-dialog'; + +export function LoggingDashboard() { + const [limit, setLimit] = useState(50); + + const { data: stats, refetch: refetchStats } = api.logging.getSessionStats.useQuery(); + const { data: history, refetch: refetchHistory } = api.logging.getCallHistory.useQuery({ limit }); + const clearSessionMutation = api.logging.clearSession.useMutation({ + onSuccess: () => { + refetchStats(); + refetchHistory(); + }, + }); + + const handleClearSession = () => { + clearSessionMutation.mutate(); + }; + + if (!stats || !history) { + return
Loading...
; + } + + return ( +
+
+

Session Logging Dashboard

+
+ + + + + + + + Clear Session Logs + + Are you sure you want to clear all session logs? This action cannot be undone. + + + + Cancel + + Clear Logs + + + + +
+
+ + {/* Stats Cards */} +
+ + + Total Calls + + + +
{stats.totalCalls}
+ + + + + + Errors + 5 ? 'destructive' : 'secondary'}> + {stats.totalErrors} + + + +
{stats.errorRate.toFixed(1)}%
+ + + + + + Avg Duration + + +
{stats.averageDuration.toFixed(0)}ms
+ + + + + + Session Duration + + +
+ {formatDistanceToNow(Date.now() - stats.sessionDuration, { addSuffix: true })} +
+ + +
+ + {/* Call History */} + + + Recent Calls + + +
+ + + +
+ + +
+ {history.map((call) => ( +
+
+
+ + {call.metadata.method} + + {call.procedure} +
+
+ {call.duration}ms +
+
+ +
+ {new Date(call.timestamp).toLocaleString()} +
+ + {call.error && ( +
+ Error: {call.error} +
+ )} + + {call.input && Object.keys(call.input).length > 0 && ( +
+ Input + 
+                                                {JSON.stringify(call.input, null, 2)}
+
+
+ )} +
+ ))} +
+ + + +
+ ); +} \ No newline at end of file diff --git a/apps/mail/lib/trpc.ts b/apps/mail/lib/trpc.ts new file mode 100644 index 0000000000..aeb1b81394 --- /dev/null +++ b/apps/mail/lib/trpc.ts @@ -0,0 +1,27 @@ +import { createTRPCClient, httpBatchLink } from '@trpc/client'; +import type { AppRouter } from '@zero/server/trpc'; +import superjson from 'superjson'; + +const getUrl = () => 'http://localhost:8787/api/trpc'; + +export const api = createTRPCClient({ + links: [ + httpBatchLink({ + maxItems: 1, + url: getUrl(), + transformer: superjson, + fetch: (url, options) => + fetch(url, { ...options, credentials: 'include' }).then((res) => { + if (typeof window !== 'undefined') { + const currentPath = new URL(window.location.href).pathname; + const redirectPath = res.headers.get('X-Zero-Redirect'); + if (!!redirectPath && redirectPath !== currentPath) { + window.location.href = redirectPath; + res.headers.delete('X-Zero-Redirect'); + } + } + return res; + }), + }), + ], +}); \ No newline at end of file diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts index 8fdd37f6f7..0f5b3f663e 100644 --- a/apps/server/src/env.ts +++ b/apps/server/src/env.ts @@ -1,5 +1,6 @@ import type { ThinkingMCP, ThreadSyncWorker, WorkflowRunner, ZeroDB, ZeroMCP } from './main'; import type { ShardRegistry, ZeroAgent, ZeroDriver } from './routes/agent'; +import type { LoggingDurableObject } from './lib/logging-durable-object'; import { env as _env } from 'cloudflare:workers'; import type { QueryableHandler } from 'dormroom'; @@ -11,6 +12,7 @@ export type ZeroEnv = { ZERO_MCP: DurableObjectNamespace; THINKING_MCP: DurableObjectNamespace; WORKFLOW_RUNNER: DurableObjectNamespace; + LOGGING: DurableObjectNamespace; THREAD_SYNC_WORKER: DurableObjectNamespace; SYNC_THREADS_WORKFLOW: Workflow; SYNC_THREADS_COORDINATOR_WORKFLOW: Workflow; diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts new file mode 100644 index 0000000000..75e03f9f9d --- /dev/null +++ b/apps/server/src/lib/logging-durable-object.ts @@ -0,0 +1,141 @@ +import { DurableObject } from 'cloudflare:workers'; +import { Queryable } from 'dormroom'; +import type { ZeroEnv } from '../env'; + +export interface TRPCCallLog { + id: string; + timestamp: number; + userId: string; + sessionId: string; + procedure: string; + input: any; + output?: any; + error?: string; + duration: number; + metadata: { + userAgent?: string; + ip?: string; + method: 'query' | 'mutation' | 'subscription'; + }; +} + +export interface LoggingState { + sessionId: string; + userId: string; + calls: TRPCCallLog[]; + startedAt: number; + lastActivity: number; + totalCalls: number; + totalErrors: number; + totalDuration: number; +} + +@Queryable() +export class LoggingDurableObject extends DurableObject { + private state: DurableObjectState; + protected env: ZeroEnv; + + constructor(state: DurableObjectState, env: ZeroEnv) { + super(state, env); + this.state = state; + this.env = env; + } + + async logCall(callData: Omit): Promise { + const log: TRPCCallLog = { + ...callData, + id: crypto.randomUUID(), + timestamp: Date.now(), + }; + + // Get current state + const currentState = await this.getState(); + + // Add the new call + currentState.calls.push(log); + currentState.lastActivity = log.timestamp; + currentState.totalCalls++; + currentState.totalDuration += log.duration; + + if (log.error) { + currentState.totalErrors++; + } + + // Keep only last 1000 calls to prevent memory issues + if (currentState.calls.length > 1000) { + currentState.calls = currentState.calls.slice(-1000); + } + + // Store updated state + await this.state.storage.put('state', currentState); + } + + async getState(): Promise { + const state = await this.state.storage.get('state'); + if (!state) { + // Initialize new state + const newState: LoggingState = { + sessionId: crypto.randomUUID(), + userId: '', + calls: [], + startedAt: Date.now(), + lastActivity: Date.now(), + totalCalls: 0, + totalErrors: 0, + totalDuration: 0, + }; + await this.state.storage.put('state', newState); + return newState; + } + return state; + } + + async initializeSession(userId: string): Promise { + const state = await this.getState(); + state.userId = userId; + state.sessionId = crypto.randomUUID(); + state.startedAt = Date.now(); + state.lastActivity = Date.now(); + await this.state.storage.put('state', state); + } + + async getCallHistory(limit: number = 100): Promise { + const state = await this.getState(); + return state.calls.slice(-limit); + } + + async getSessionStats(): Promise<{ + totalCalls: number; + totalErrors: number; + totalDuration: number; + averageDuration: number; + errorRate: number; + sessionDuration: number; + }> { + const state = await this.getState(); + const sessionDuration = Date.now() - state.startedAt; + + return { + totalCalls: state.totalCalls, + totalErrors: state.totalErrors, + totalDuration: state.totalDuration, + averageDuration: state.totalCalls > 0 ? state.totalDuration / state.totalCalls : 0, + errorRate: state.totalCalls > 0 ? (state.totalErrors / state.totalCalls) * 100 : 0, + sessionDuration, + }; + } + + async clearSession(): Promise { + const newState: LoggingState = { + sessionId: crypto.randomUUID(), + userId: '', + calls: [], + startedAt: Date.now(), + lastActivity: Date.now(), + totalCalls: 0, + totalErrors: 0, + totalDuration: 0, + }; + await this.state.storage.put('state', newState); + } +} \ No newline at end of file diff --git a/apps/server/src/lib/server-utils.ts b/apps/server/src/lib/server-utils.ts index b5ee8d7b1e..54106d962a 100644 --- a/apps/server/src/lib/server-utils.ts +++ b/apps/server/src/lib/server-utils.ts @@ -30,7 +30,7 @@ class MockExecutionContext implements ExecutionContext { console.error('MockExecutionContext: Error in waitUntil', error); } } - passThroughOnException(): void {} + passThroughOnException(): void { } props: any; } @@ -604,6 +604,22 @@ export const verifyToken = async (token: string) => { return !!data; }; +// Logging utility functions +export const getLoggingDO = async (sessionId: string) => { + const stub = env.LOGGING.get(env.LOGGING.idFromName(sessionId)); + return stub; +}; + +export const logTRPCCall = async (sessionId: string, callData: any) => { + const loggingDO = await getLoggingDO(sessionId); + await loggingDO.logCall(callData); +}; + +export const initializeLoggingSession = async (sessionId: string, userId: string) => { + const loggingDO = await getLoggingDO(sessionId); + await loggingDO.initializeSession(userId); +}; + export const resetConnection = async (connectionId: string) => { const { db, conn } = createDb(env.HYPERDRIVE.connectionString); await db diff --git a/apps/server/src/lib/trpc-logging.ts b/apps/server/src/lib/trpc-logging.ts new file mode 100644 index 0000000000..3689163480 --- /dev/null +++ b/apps/server/src/lib/trpc-logging.ts @@ -0,0 +1,89 @@ +import type { TRPCError } from '@trpc/server'; +import type { TRPCCallLog } from './logging-durable-object'; +import { logTRPCCall, initializeLoggingSession } from './server-utils'; +import { getContext } from 'hono/context-storage'; +import type { HonoContext } from '../ctx'; + +export interface LoggingContext { + sessionId: string; + userId?: string; +} + +export const createLoggingMiddleware = () => { + return async (opts: { + path: string; + type: 'query' | 'mutation' | 'subscription'; + next: () => Promise; + input: any; + ctx: any; + }) => { + const startTime = Date.now(); + const c = getContext(); + const sessionId = c.var.sessionUser?.id || 'anonymous'; + const userId = c.var.sessionUser?.id; + + // Initialize session if this is the first call + if (userId) { + try { + await initializeLoggingSession(sessionId, userId); + } catch (error) { + console.error('Failed to initialize logging session:', error); + } + } + + let output: any; + let error: string | undefined; + + try { + // Execute the TRPC call + output = await opts.next(); + + // Log successful call + const callData: Omit = { + userId: userId || 'anonymous', + sessionId, + procedure: opts.path, + input: opts.input, + output: output, + duration: Date.now() - startTime, + metadata: { + method: opts.type, + userAgent: c.req.header('User-Agent'), + ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + }, + }; + + // Log asynchronously to avoid blocking the response + logTRPCCall(sessionId, callData).catch((err) => { + console.error('Failed to log TRPC call:', err); + }); + + } catch (err) { + error = err instanceof Error ? err.message : 'Unknown error'; + + // Log failed call + const callData: Omit = { + userId: userId || 'anonymous', + sessionId, + procedure: opts.path, + input: opts.input, + error, + duration: Date.now() - startTime, + metadata: { + method: opts.type, + userAgent: c.req.header('User-Agent'), + ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + }, + }; + + // Log asynchronously to avoid blocking the response + logTRPCCall(sessionId, callData).catch((logErr) => { + console.error('Failed to log TRPC error:', logErr); + }); + + throw err; + } + + return output; + }; +}; \ No newline at end of file diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index a9bb9f51d7..fd98a8e06d 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -31,6 +31,8 @@ import { oAuthDiscoveryMetadata } from 'better-auth/plugins'; import { EProviders, type IEmailSendBatch } from './types'; import { eq, and, desc, asc, inArray } from 'drizzle-orm'; import { ThinkingMCP } from './lib/sequential-thinking'; +import { ZeroAgent, ZeroDriver } from './routes/agent'; +import { LoggingDurableObject } from './lib/logging-durable-object'; import { contextStorage } from 'hono/context-storage'; import { defaultUserSettings } from './lib/schemas'; import { createLocalJWKSet, jwtVerify } from 'jose'; @@ -1148,4 +1150,5 @@ export { SyncThreadsWorkflow, SyncThreadsCoordinatorWorkflow, ShardRegistry, + LoggingDurableObject, }; diff --git a/apps/server/src/trpc/index.ts b/apps/server/src/trpc/index.ts index 5a5a433b89..3b473c6543 100644 --- a/apps/server/src/trpc/index.ts +++ b/apps/server/src/trpc/index.ts @@ -17,6 +17,9 @@ import { bimiRouter } from './routes/bimi'; import type { HonoContext } from '../ctx'; import { aiRouter } from './routes/ai'; import { router } from './trpc'; +import { categoriesRouter } from './routes/categories'; +import { templatesRouter } from './routes/templates'; +import { loggingRouter } from './routes/logging'; export const appRouter = router({ ai: aiRouter, @@ -34,6 +37,7 @@ export const appRouter = router({ user: userRouter, templates: templatesRouter, meet: meetRouter, + logging: loggingRouter, }); export type AppRouter = typeof appRouter; diff --git a/apps/server/src/trpc/routes/logging.ts b/apps/server/src/trpc/routes/logging.ts new file mode 100644 index 0000000000..fb6df991ae --- /dev/null +++ b/apps/server/src/trpc/routes/logging.ts @@ -0,0 +1,39 @@ +import { privateProcedure, router } from '../trpc'; +import { getLoggingDO } from '../../lib/server-utils'; +import { z } from 'zod'; + +export const loggingRouter = router({ + getSessionStats: privateProcedure + .query(async ({ ctx }) => { + const sessionId = ctx.sessionUser?.id || 'anonymous'; + const loggingDO = await getLoggingDO(sessionId); + return await loggingDO.getSessionStats(); + }), + + getCallHistory: privateProcedure + .input( + z.object({ + limit: z.number().min(1).max(1000).default(100), + }) + ) + .query(async ({ ctx, input }) => { + const sessionId = ctx.sessionUser?.id || 'anonymous'; + const loggingDO = await getLoggingDO(sessionId); + return await loggingDO.getCallHistory(input.limit); + }), + + clearSession: privateProcedure + .mutation(async ({ ctx }) => { + const sessionId = ctx.sessionUser?.id || 'anonymous'; + const loggingDO = await getLoggingDO(sessionId); + await loggingDO.clearSession(); + return { success: true }; + }), + + getSessionState: privateProcedure + .query(async ({ ctx }) => { + const sessionId = ctx.sessionUser?.id || 'anonymous'; + const loggingDO = await getLoggingDO(sessionId); + return await loggingDO.getState(); + }), +}); \ No newline at end of file diff --git a/apps/server/src/trpc/trpc.ts b/apps/server/src/trpc/trpc.ts index 3bb2e55f3e..baf6c12ee2 100644 --- a/apps/server/src/trpc/trpc.ts +++ b/apps/server/src/trpc/trpc.ts @@ -1,4 +1,4 @@ -import { getActiveConnection, getZeroDB } from '../lib/server-utils'; +import { getActiveConnection, getZeroDB, logTRPCCall, initializeLoggingSession } from '../lib/server-utils'; import { Ratelimit, type RatelimitConfig } from '@upstash/ratelimit'; import type { HonoContext, HonoVariables } from '../ctx'; import { getConnInfo } from 'hono/cloudflare-workers'; @@ -14,8 +14,59 @@ type TrpcContext = { const t = initTRPC.context().create({ transformer: superjson }); +// Logging middleware +const createLoggingMiddleware = () => { + return t.middleware(async ({ ctx, next, path, type, input }) => { + const startTime = Date.now(); + const sessionId = ctx.sessionUser?.id || 'anonymous'; + + try { + // Initialize session if needed + await initializeLoggingSession(sessionId, sessionId); + + const result = await next(); + const duration = Date.now() - startTime; + + // Log the call asynchronously (don't block the response) + logTRPCCall(sessionId, { + procedure: path, + input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size + output: result.ok ? JSON.stringify(result.data).slice(0, 1000) : undefined, // Limit output size + error: !result.ok ? result.error?.message : undefined, + duration, + metadata: { + userAgent: ctx.c.req.header('user-agent'), + ip: getConnInfo(ctx.c).remote.address, + method: type, + }, + }).catch(err => console.error('Failed to log TRPC call:', err)); + + return result; + } catch (error) { + const duration = Date.now() - startTime; + + // Log the error asynchronously + logTRPCCall(sessionId, { + procedure: path, + input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size + error: error instanceof Error ? error.message : 'Unknown error', + duration, + metadata: { + userAgent: ctx.c.req.header('user-agent'), + ip: getConnInfo(ctx.c).remote.address, + method: type, + }, + }).catch(err => console.error('Failed to log TRPC call:', err)); + + throw error; + } + }); +}; + +const loggingMiddleware = createLoggingMiddleware(); + export const router = t.router; -export const publicProcedure = t.procedure; +export const publicProcedure = t.procedure.use(loggingMiddleware); export const privateProcedure = publicProcedure.use(async ({ ctx, next }) => { if (!ctx.sessionUser) { @@ -69,12 +120,12 @@ export const activeDriverProcedure = activeConnectionProcedure.use(async ({ ctx, accessToken: null, refreshToken: null, }); - if (activeConnection.accessToken) { - ctx.c.header( - 'X-Zero-Redirect', - `/settings/connections?disconnectedConnectionId=${activeConnection.id}`, - ); - } + + ctx.c.header( + 'X-Zero-Redirect', + `/settings/connections?disconnectedConnectionId=${activeConnection.id}`, + ); + throw new TRPCError({ code: 'UNAUTHORIZED', message: 'Connection expired. Please reconnect.', diff --git a/apps/server/wrangler.jsonc b/apps/server/wrangler.jsonc index 309d984e66..76c6488e0d 100644 --- a/apps/server/wrangler.jsonc +++ b/apps/server/wrangler.jsonc @@ -58,6 +58,10 @@ "name": "WORKFLOW_RUNNER", "class_name": "WorkflowRunner", }, + { + "name": "LOGGING", + "class_name": "LoggingDurableObject", + }, { "name": "THREAD_SYNC_WORKER", "class_name": "ThreadSyncWorker", @@ -143,6 +147,7 @@ { "tag": "v9", "new_sqlite_classes": ["ShardRegistry"], + "new_classes": ["LoggingDurableObject"], }, ], From fa657aa2de8a02e91fdac721f5e8c512a133893c Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Wed, 6 Aug 2025 16:59:22 +0400 Subject: [PATCH 02/13] linting fixes --- apps/mail/app/(routes)/demo/logging/page.tsx | 1 - apps/server/src/lib/trpc-logging.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/apps/mail/app/(routes)/demo/logging/page.tsx b/apps/mail/app/(routes)/demo/logging/page.tsx index df2a661f31..bacbf62e11 100644 --- a/apps/mail/app/(routes)/demo/logging/page.tsx +++ b/apps/mail/app/(routes)/demo/logging/page.tsx @@ -1,4 +1,3 @@ -import { api } from '@/lib/trpc'; import { useEffect, useState } from 'react'; export default function LoggingDemoPage() { diff --git a/apps/server/src/lib/trpc-logging.ts b/apps/server/src/lib/trpc-logging.ts index 3689163480..5f0611439f 100644 --- a/apps/server/src/lib/trpc-logging.ts +++ b/apps/server/src/lib/trpc-logging.ts @@ -1,4 +1,3 @@ -import type { TRPCError } from '@trpc/server'; import type { TRPCCallLog } from './logging-durable-object'; import { logTRPCCall, initializeLoggingSession } from './server-utils'; import { getContext } from 'hono/context-storage'; From dc1a70f6882d1bfc6cc5509df935a0090a1a5d36 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Fri, 8 Aug 2025 17:42:23 +0400 Subject: [PATCH 03/13] Integrate Datadog API client and enhance logging functionality - Added '@datadog/datadog-api-client' dependency to the project. - Updated logging durable object to export session data to Datadog upon session end or expiration. - Implemented new TRPC routes for exporting current session logs to Datadog and ending sessions. --- apps/mail/app/(routes)/demo/logging/page.tsx | 166 ++++++++---- apps/server/package.json | 1 + apps/server/src/env.ts | 3 + apps/server/src/lib/datadog-service.ts | 115 +++++++++ apps/server/src/lib/logging-durable-object.ts | 44 ++++ apps/server/src/trpc/routes/logging.ts | 16 ++ apps/server/wrangler.jsonc | 3 + pnpm-lock.yaml | 237 +++++++++++------- 8 files changed, 447 insertions(+), 138 deletions(-) create mode 100644 apps/server/src/lib/datadog-service.ts diff --git a/apps/mail/app/(routes)/demo/logging/page.tsx b/apps/mail/app/(routes)/demo/logging/page.tsx index bacbf62e11..0db0ce8618 100644 --- a/apps/mail/app/(routes)/demo/logging/page.tsx +++ b/apps/mail/app/(routes)/demo/logging/page.tsx @@ -1,60 +1,134 @@ import { useEffect, useState } from 'react'; -export default function LoggingDemoPage() { +export default function LoggingDemo() { const [sessionStats, setSessionStats] = useState(null); const [callHistory, setCallHistory] = useState(null); - const [loading, setLoading] = useState(true); + const [loading, setLoading] = useState(false); - useEffect(() => { - const fetchData = async () => { - try { - // Fetch session stats - const statsResponse = await fetch('http://localhost:8787/api/trpc/logging.getSessionStats', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - credentials: 'include', - }); - const statsData = await statsResponse.json(); - setSessionStats(statsData); - - // Fetch call history - const historyResponse = await fetch('http://localhost:8787/api/trpc/logging.getCallHistory', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - credentials: 'include', - body: JSON.stringify({ - json: { limit: 100 } - }), - }); - const historyData = await historyResponse.json(); - setCallHistory(historyData); - } catch (error) { - console.error('Error fetching data:', error); - } finally { - setLoading(false); - } - }; + const fetchData = async () => { + setLoading(true); + try { + // Fetch session stats + const statsResponse = await fetch('http://localhost:8787/api/trpc/logging.getSessionStats', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Cookie': document.cookie // Include auth cookies + }, + credentials: 'include', + body: JSON.stringify({ json: {} }), + }); + const statsData = await statsResponse.json(); + setSessionStats(statsData); - fetchData(); - }, []); + // Fetch call history + const historyResponse = await fetch('http://localhost:8787/api/trpc/logging.getCallHistory', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Cookie': document.cookie // Include auth cookies + }, + credentials: 'include', + body: JSON.stringify({ json: { limit: 100 } }), + }); + const historyData = await historyResponse.json(); + setCallHistory(historyData); + } catch (error) { + console.error('Error fetching data:', error); + } finally { + setLoading(false); + } + }; - if (loading) { - return
Loading...
; - } + const exportToDatadog = async () => { + try { + const response = await fetch('http://localhost:8787/api/trpc/logging.exportToDatadog', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Cookie': document.cookie + }, + credentials: 'include', + body: JSON.stringify({ json: {} }), + }); + const result = await response.json(); + console.log('Datadog export result:', result); + console.log('Session exported to Datadog!'); + } catch (error) { + console.error('Error exporting to Datadog:', error); + console.error('Failed to export to Datadog'); + } + }; - const jsonData = { - sessionStats, - callHistory, - timestamp: new Date().toISOString(), + const endSession = async () => { + try { + const response = await fetch('http://localhost:8787/api/trpc/logging.endSession', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Cookie': document.cookie + }, + credentials: 'include', + body: JSON.stringify({ json: {} }), + }); + const result = await response.json(); + console.log('Session end result:', result); + console.log('Session ended and exported to Datadog!'); + fetchData(); // Refresh data + } catch (error) { + console.error('Error ending session:', error); + console.error('Failed to end session'); + } }; + useEffect(() => { + fetchData(); + }, []); + return ( -
-

Session Logging JSON Output

-
- 
-                    {JSON.stringify(jsonData, null, 2)}
-
+
+

TRPC Logging Demo

+ +
+ + + +
+ +
+
+

Session Stats

+
+ 
+                            {sessionStats ? JSON.stringify(sessionStats, null, 2) : 'Loading...'}
+
+
+
+ +
+

Call History

+
+ 
+                            {callHistory ? JSON.stringify(callHistory, null, 2) : 'Loading...'}
+
+
+
); diff --git a/apps/server/package.json b/apps/server/package.json index eefdde724c..78af5f4010 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -31,6 +31,7 @@ "@arcadeai/arcadejs": "1.8.1", "@barkleapp/css-sanitizer": "1.0.0", "@coinbase/cookie-manager": "1.1.8", + "@datadog/datadog-api-client": "1.40.0", "@dub/better-auth": "0.0.3", "@googleapis/gmail": "12.0.0", "@googleapis/people": "3.0.9", diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts index 0f5b3f663e..25cbc5c56d 100644 --- a/apps/server/src/env.ts +++ b/apps/server/src/env.ts @@ -99,6 +99,9 @@ export type ZeroEnv = { OTEL_EXPORTER_OTLP_ENDPOINT?: string; OTEL_EXPORTER_OTLP_HEADERS?: string; OTEL_SERVICE_NAME?: string; + DD_API_KEY: string; + DD_APP_KEY: string; + DD_SITE: string; }; const env = _env as ZeroEnv; diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts new file mode 100644 index 0000000000..6662f5c183 --- /dev/null +++ b/apps/server/src/lib/datadog-service.ts @@ -0,0 +1,115 @@ +import { client, v2 } from '@datadog/datadog-api-client'; +import type { TRPCCallLog } from './logging-durable-object'; +import type { ZeroEnv } from '../env'; + +export class DatadogService { + private apiInstance: v2.LogsApi; + + constructor(env?: ZeroEnv) { + const configuration = client.createConfiguration({ + authMethods: { + apiKeyAuth: env?.DD_API_KEY || '', + appKeyAuth: env?.DD_APP_KEY || '', + }, + }); + + // Set the site for the configuration + if (env?.DD_SITE) { + configuration.setServerVariables({ site: env.DD_SITE }); + } + + this.apiInstance = new v2.LogsApi(configuration); + } + + async exportSessionLogs(sessionId: string, userId: string, logs: TRPCCallLog[]): Promise { + if (logs.length === 0) return; + + try { + const logEntries = logs.map(log => ({ + message: `TRPC call: ${log.procedure}`, + service: 'zero-mail-app', + ddsource: 'trpc-logging', + ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure}`, + hostname: 'cloudflare-worker', + timestamp: Date.now(), + // Custom fields for TRPC data + trpc: { + procedure: log.procedure, + input: log.input, + output: log.output, + error: log.error, + duration: log.duration, + sessionId, + userId, + }, + })); + + console.log('📊 Sending to Datadog:', { + sessionId, + userId, + logCount: logs.length, + // logEntries: logEntries + }); + + const params = { + body: logEntries, + }; + + await this.apiInstance.submitLog(params); + + console.log('✅ Successfully exported to Datadog:', { + sessionId, + userId, + logCount: logs.length + }); + } catch (error) { + console.error('❌ Failed to export session logs to Datadog:', error); + } + } + + async exportBatchLogs(logs: Array<{ sessionId: string; userId: string; logs: TRPCCallLog[] }>): Promise { + if (logs.length === 0) return; + + try { + const allLogEntries: any[] = []; + + for (const { sessionId, userId, logs: sessionLogs } of logs) { + const logEntries = sessionLogs.map(log => ({ + message: `TRPC call: ${log.procedure}`, + service: 'zero-mail-app', + ddsource: 'trpc-logging', + ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure}`, + hostname: 'cloudflare-worker', + timestamp: Date.now(), + trpc: { + procedure: log.procedure, + input: log.input, + output: log.output, + error: log.error, + duration: log.duration, + sessionId, + userId, + }, + })); + + allLogEntries.push(...logEntries); + } + + console.log('📊 Sending batch to Datadog:', { + sessionCount: logs.length, + totalLogCount: allLogEntries.length, + sessions: logs.map(l => ({ sessionId: l.sessionId, userId: l.userId, logCount: l.logs.length })) + }); + + if (allLogEntries.length > 0) { + const params = { + body: allLogEntries, + }; + + await this.apiInstance.submitLog(params); + } + } catch (error) { + console.error('Failed to export batch logs to Datadog:', error); + } + } +} \ No newline at end of file diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts index 75e03f9f9d..d348d2bc0e 100644 --- a/apps/server/src/lib/logging-durable-object.ts +++ b/apps/server/src/lib/logging-durable-object.ts @@ -1,6 +1,7 @@ import { DurableObject } from 'cloudflare:workers'; import { Queryable } from 'dormroom'; import type { ZeroEnv } from '../env'; +import { DatadogService } from './datadog-service'; export interface TRPCCallLog { id: string; @@ -34,11 +35,14 @@ export interface LoggingState { export class LoggingDurableObject extends DurableObject { private state: DurableObjectState; protected env: ZeroEnv; + private datadogService: DatadogService; + private readonly SESSION_TIMEOUT = 30 * 60 * 1000; // 30 minutes constructor(state: DurableObjectState, env: ZeroEnv) { super(state, env); this.state = state; this.env = env; + this.datadogService = new DatadogService(env); } async logCall(callData: Omit): Promise { @@ -51,6 +55,19 @@ export class LoggingDurableObject extends DurableObject { // Get current state const currentState = await this.getState(); + // Check if session has expired + const timeSinceLastActivity = Date.now() - currentState.lastActivity; + if (timeSinceLastActivity > this.SESSION_TIMEOUT && currentState.calls.length > 0) { + // Export expired session to Datadog + await this.exportSessionToDatadog(currentState); + + // Start new session + await this.clearSession(); + const newState = await this.getState(); + newState.userId = currentState.userId; // Preserve user ID + await this.state.storage.put('state', newState); + } + // Add the new call currentState.calls.push(log); currentState.lastActivity = log.timestamp; @@ -138,4 +155,31 @@ export class LoggingDurableObject extends DurableObject { }; await this.state.storage.put('state', newState); } + + async exportSessionToDatadog(state: LoggingState): Promise { + if (state.calls.length === 0) return; + + try { + await this.datadogService.exportSessionLogs( + state.sessionId, + state.userId, + state.calls + ); + } catch (error) { + console.error('Failed to export session to Datadog:', error); + } + } + + async exportCurrentSessionToDatadog(): Promise { + const state = await this.getState(); + await this.exportSessionToDatadog(state); + } + + async endSession(): Promise { + const state = await this.getState(); + if (state.calls.length > 0) { + await this.exportSessionToDatadog(state); + } + await this.clearSession(); + } } \ No newline at end of file diff --git a/apps/server/src/trpc/routes/logging.ts b/apps/server/src/trpc/routes/logging.ts index fb6df991ae..e5629e9935 100644 --- a/apps/server/src/trpc/routes/logging.ts +++ b/apps/server/src/trpc/routes/logging.ts @@ -36,4 +36,20 @@ export const loggingRouter = router({ const loggingDO = await getLoggingDO(sessionId); return await loggingDO.getState(); }), + + exportToDatadog: privateProcedure + .mutation(async ({ ctx }) => { + const sessionId = ctx.sessionUser?.id || 'anonymous'; + const loggingDO = await getLoggingDO(sessionId); + await loggingDO.exportCurrentSessionToDatadog(); + return { success: true }; + }), + + endSession: privateProcedure + .mutation(async ({ ctx }) => { + const sessionId = ctx.sessionUser?.id || 'anonymous'; + const loggingDO = await getLoggingDO(sessionId); + await loggingDO.endSession(); + return { success: true }; + }), }); \ No newline at end of file diff --git a/apps/server/wrangler.jsonc b/apps/server/wrangler.jsonc index 76c6488e0d..a8bc512e67 100644 --- a/apps/server/wrangler.jsonc +++ b/apps/server/wrangler.jsonc @@ -182,6 +182,9 @@ "MEET_AUTH_HEADER": "", "OTEL_EXPORTER_OTLP_ENDPOINT": "https://api.axiom.co/v1/traces", "OTEL_SERVICE_NAME": "zero-email-server-local", + "DD_API_KEY": "", + "DD_APP_KEY": "", + "DD_SITE": "datadoghq.com", }, "kv_namespaces": [ { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a30e1ff6b8..eae1df6689 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,7 +20,7 @@ catalogs: version: 0.0.48 better-auth: specifier: ^1.3.4 - version: 1.3.4 + version: 1.3.7 drizzle-kit: specifier: ^0.31.1 version: 0.31.4 @@ -41,7 +41,7 @@ catalogs: version: 5.8.3 wrangler: specifier: ^4.28.1 - version: 4.28.1 + version: 4.30.0 zod: specifier: ^3.25.42 version: 3.25.67 @@ -142,7 +142,7 @@ importers: version: 1.1.0(react-dom@19.1.0(react@19.1.0))(react@19.1.0) '@react-router/dev': specifier: ^7.6.1 - version: 7.6.3(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(react-router@7.6.3(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(tsx@4.19.4)(typescript@5.8.3)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(wrangler@4.28.1(@cloudflare/workers-types@4.20250628.0))(yaml@2.8.0) + version: 7.6.3(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(react-router@7.6.3(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(tsx@4.19.4)(typescript@5.8.3)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(wrangler@4.30.0(@cloudflare/workers-types@4.20250628.0))(yaml@2.8.0) '@sentry/react': specifier: 9.40.0 version: 9.40.0(react@19.1.0) @@ -232,7 +232,7 @@ importers: version: 19.1.0-rc.2 better-auth: specifier: 'catalog:' - version: 1.3.4(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + version: 1.3.7(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(zod@3.25.67) canvas-confetti: specifier: 1.9.3 version: 1.9.3 @@ -419,7 +419,7 @@ importers: devDependencies: '@cloudflare/vite-plugin': specifier: ^1.3.1 - version: 1.7.5(rollup@4.44.1)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(workerd@1.20250803.0)(wrangler@4.28.1(@cloudflare/workers-types@4.20250628.0)) + version: 1.7.5(rollup@4.44.1)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(workerd@1.20250813.0)(wrangler@4.30.0(@cloudflare/workers-types@4.20250628.0)) '@inlang/cli': specifier: ^3.0.0 version: 3.0.12 @@ -485,7 +485,7 @@ importers: version: 5.1.4(typescript@5.8.3)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0)) wrangler: specifier: 'catalog:' - version: 4.28.1(@cloudflare/workers-types@4.20250628.0) + version: 4.30.0(@cloudflare/workers-types@4.20250628.0) apps/server: dependencies: @@ -516,6 +516,9 @@ importers: '@coinbase/cookie-manager': specifier: 1.1.8 version: 1.1.8 + '@datadog/datadog-api-client': + specifier: 1.40.0 + version: 1.40.0 '@dub/better-auth': specifier: 0.0.3 version: 0.0.3 @@ -581,7 +584,7 @@ importers: version: 1.5.1 better-auth: specifier: 'catalog:' - version: 1.3.4(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + version: 1.3.7(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(zod@3.25.67) cheerio: specifier: 1.1.0 version: 1.1.0 @@ -599,7 +602,7 @@ importers: version: 1.0.1 drizzle-orm: specifier: 'catalog:' - version: 0.43.1(@cloudflare/workers-types@4.20250628.0)(@opentelemetry/api@1.9.0)(@types/pg@8.6.1)(better-sqlite3@11.10.0)(kysely@0.28.2)(postgres@3.4.5) + version: 0.43.1(@cloudflare/workers-types@4.20250628.0)(@opentelemetry/api@1.9.0)(@types/pg@8.6.1)(better-sqlite3@11.10.0)(kysely@0.28.5)(postgres@3.4.5) dub: specifier: 0.64.2 version: 0.64.2(@modelcontextprotocol/sdk@1.15.1)(zod@3.25.67) @@ -677,7 +680,7 @@ importers: version: 11.1.0 wrangler: specifier: 'catalog:' - version: 4.28.1(@cloudflare/workers-types@4.20250628.0) + version: 4.30.0(@cloudflare/workers-types@4.20250628.0) zod: specifier: 'catalog:' version: 3.25.67 @@ -1089,8 +1092,8 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} - '@better-auth/utils@0.2.5': - resolution: {integrity: sha512-uI2+/8h/zVsH8RrYdG8eUErbuGBk16rZKQfz8CjxQOyCE6v7BqFYEbFwvOkvl1KbUdxhqOnXp78+uE5h8qVEgQ==} + '@better-auth/utils@0.2.6': + resolution: {integrity: sha512-3y/vaL5Ox33dBwgJ6ub3OPkVqr6B5xL2kgxNHG8eHZuryLyG/4JSPGqjbdRSgjuy9kALUZYDFl+ORIAxlWMSuA==} '@better-fetch/fetch@1.1.18': resolution: {integrity: sha512-rEFOE1MYIsBmoMJtQbl32PGHHXuG2hDxvEd7rUHE0vCBoFQVSDqaVs9hkZEtHCxRoY+CljXKFCOuJ8uxqw1LcA==} @@ -1129,8 +1132,8 @@ packages: workerd: optional: true - '@cloudflare/unenv-preset@2.6.0': - resolution: {integrity: sha512-h7Txw0WbDuUbrvZwky6+x7ft+U/Gppfn/rWx6IdR+e9gjygozRJnV26Y2TOr3yrIFa6OsZqqR2lN+jWTrakHXg==} + '@cloudflare/unenv-preset@2.6.1': + resolution: {integrity: sha512-48rC6jo9CkSRkImfu5KU4zKyoPJx7b9GTUpZn0Emr6J+jkmrLhwCY3BI10QS+fhOt1NkJNlxIcYrBgvWeCpKOw==} peerDependencies: unenv: 2.0.0-rc.19 workerd: ^1.20250802.0 @@ -1150,8 +1153,8 @@ packages: cpu: [x64] os: [darwin] - '@cloudflare/workerd-darwin-64@1.20250803.0': - resolution: {integrity: sha512-6QciMnJp1p3F1qUiN0LaLfmw7SuZA/gfUBOe8Ft81pw16JYZ3CyiqIKPJvc1SV8jgDx8r+gz/PRi1NwOMt329A==} + '@cloudflare/workerd-darwin-64@1.20250813.0': + resolution: {integrity: sha512-Pka37/jqLy7ZaQlwpBy79A/BLH+qpRPSEX2h/zWND+qRfoCVCCaZQPdknHZO0pcvHPzK8E2Z4j5QI1IafPA5UA==} engines: {node: '>=16'} cpu: [x64] os: [darwin] @@ -1162,8 +1165,8 @@ packages: cpu: [arm64] os: [darwin] - '@cloudflare/workerd-darwin-arm64@1.20250803.0': - resolution: {integrity: sha512-DoIgghDowtqoNhL6OoN/F92SKtrk7mRQKc4YSs/Dst8IwFZq+pCShOlWfB0MXqHKPSoiz5xLSrUKR9H6gQMPvw==} + '@cloudflare/workerd-darwin-arm64@1.20250813.0': + resolution: {integrity: sha512-QnaJbmhcA32+4uZ+or1hXZjdxGqrFUuh6Ye+skEGu3iB/xzq9CmyVyoKoshiUOcWGKndQb7KRo56dq0bVvVLFw==} engines: {node: '>=16'} cpu: [arm64] os: [darwin] @@ -1174,8 +1177,8 @@ packages: cpu: [x64] os: [linux] - '@cloudflare/workerd-linux-64@1.20250803.0': - resolution: {integrity: sha512-mYdz4vNWX3+PoqRjssepVQqgh42IBiSrl+wb7vbh7VVWUVzBnQKtW3G+UFiBF62hohCLexGIEi7L0cFfRlcKSQ==} + '@cloudflare/workerd-linux-64@1.20250813.0': + resolution: {integrity: sha512-6pokgBQmujJsAuqOme2wBX5ol/1YW3d7kV7wp0Y1/tFi46TnmWcEy08B4FD5t2AARQJ68a7XMxIJKWChcaJ9Cg==} engines: {node: '>=16'} cpu: [x64] os: [linux] @@ -1186,8 +1189,8 @@ packages: cpu: [arm64] os: [linux] - '@cloudflare/workerd-linux-arm64@1.20250803.0': - resolution: {integrity: sha512-RmrtUYLRUg6djKU7Z6yebS6YGJVnaDVY6bbXca+2s26vw4ibJDOTPLuBHFQF62Grw3fAfsNbjQh5i14vG2mqUg==} + '@cloudflare/workerd-linux-arm64@1.20250813.0': + resolution: {integrity: sha512-lFwqohi8fkR98OwjHT69sbThx4BJem7vu6N8kqrge7wuKJWrMDNbzOTdyBA8adV9DmE07ELuN2vcbbu8ZjaL2Q==} engines: {node: '>=16'} cpu: [arm64] os: [linux] @@ -1198,8 +1201,8 @@ packages: cpu: [x64] os: [win32] - '@cloudflare/workerd-windows-64@1.20250803.0': - resolution: {integrity: sha512-uLV8gdudz36o9sUaAKbBxxTwZwLFz1KyW7QpBvOo4+r3Ib8yVKXGiySIMWGD7A0urSMrjf3e5LlLcJKgZUOjMA==} + '@cloudflare/workerd-windows-64@1.20250813.0': + resolution: {integrity: sha512-Fs62NvUajtoXb+4W8jaRXzw64Nbmb8X+PbRLZbxUFv68sGhxKPw1nB1YEmNNZ215ma47hTlSdF3UQh4FOmz7NA==} engines: {node: '>=16'} cpu: [x64] os: [win32] @@ -1242,6 +1245,10 @@ packages: resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==} engines: {node: '>=18'} + '@datadog/datadog-api-client@1.40.0': + resolution: {integrity: sha512-b8XxPEy0pnsmKed/4aejWMXkzV2IXolUreK7K7KYGJW/bYPpQLJpwBgv2W+GJTaHWX1YfH1veQnm3CW3Qvs04Q==} + engines: {node: '>=12.0.0'} + '@daybrush/utils@1.13.0': resolution: {integrity: sha512-ALK12C6SQNNHw1enXK+UO8bdyQ+jaWNQ1Af7Z3FNxeAwjYhQT7do+TRE4RASAJ3ObaS2+TJ7TXR3oz2Gzbw0PQ==} @@ -4309,11 +4316,11 @@ packages: engines: {node: '>= 8.0.0'} hasBin: true - '@simplewebauthn/browser@13.1.0': - resolution: {integrity: sha512-WuHZ/PYvyPJ9nxSzgHtOEjogBhwJfC8xzYkPC+rR/+8chl/ft4ngjiK8kSU5HtRJfczupyOh33b25TjYbvwAcg==} + '@simplewebauthn/browser@13.1.2': + resolution: {integrity: sha512-aZnW0KawAM83fSBUgglP5WofbrLbLyr7CoPqYr66Eppm7zO86YX6rrCjRB3hQKPrL7ATvY4FVXlykZ6w6FwYYw==} - '@simplewebauthn/server@13.1.1': - resolution: {integrity: sha512-1hsLpRHfSuMB9ee2aAdh0Htza/X3f4djhYISrggqGe3xopNjOcePiSDkDDoPzDYaaMCrbqGP1H2TYU7bgL9PmA==} + '@simplewebauthn/server@13.1.2': + resolution: {integrity: sha512-VwoDfvLXSCaRiD+xCIuyslU0HLxVggeE5BL06+GbsP2l1fGf5op8e0c3ZtKoi+vSg1q4ikjtAghC23ze2Q3H9g==} engines: {node: '>=20.0.0'} '@sinclair/typebox@0.27.8': @@ -4750,6 +4757,9 @@ packages: '@types/babel__traverse@7.28.0': resolution: {integrity: sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==} + '@types/buffer-from@1.1.3': + resolution: {integrity: sha512-2lq4YC9uLUMGHkl2IDtX4tCXSo2+hwMpOJcY1qiIk1kybc31rIlPyM1HCVJhkPFIo75a/pOVxqyvwuf5TpCG/w==} + '@types/canvas-confetti@1.9.0': resolution: {integrity: sha512-aBGj/dULrimR1XDZLtG9JwxX1b4HPRF6CX9Yfwh3NvstZEm1ZL7RBnel4keCPSqs1ANRu1u2Aoz9R+VmtjYuTg==} @@ -4864,6 +4874,9 @@ packages: '@types/node@22.15.29': resolution: {integrity: sha512-LNdjOkUDlU1RZb8e1kOIUpN1qQUlzGkEtbVNo53vbrwDg5om6oduhm4SiUaPW5ASTXhAiP0jInWG8Qx9fVlOeQ==} + '@types/pako@1.0.7': + resolution: {integrity: sha512-YBtzT2ztNF6R/9+UXj2wTGFnC9NklAnASt3sC0h2m1bbH7G6FyBIkt4AN8ThZpNfxUo1b2iMVO0UawiJymEt8A==} + '@types/pg-pool@2.0.6': resolution: {integrity: sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==} @@ -5305,11 +5318,12 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - better-auth@1.3.4: - resolution: {integrity: sha512-JbZYam6Cs3Eu5CSoMK120zSshfaKvrCftSo/+v7524H1RvhryQ7UtMbzagBcXj0Digjj8hZtVkkR4tTZD/wK2g==} + better-auth@1.3.7: + resolution: {integrity: sha512-/1fEyx2SGgJQM5ujozDCh9eJksnVkNU/J7Fk/tG5Y390l8nKbrPvqiFlCjlMM+scR+UABJbQzA6An7HT50LHyQ==} peerDependencies: react: ^18.0.0 || ^19.0.0 react-dom: ^18.0.0 || ^19.0.0 + zod: ^3.25.0 || ^4.0.0 peerDependenciesMeta: react: optional: true @@ -5649,6 +5663,9 @@ packages: resolution: {integrity: sha512-nH0a49E/kSVk6BeFgKZy4uUsy6D2A16p120h5bYD9ILBhQu7o2sJFH+WI4R731TSBQ0dB1Ik7inB/dRAB4C8QQ==} engines: {node: '>=18'} + cross-fetch@3.2.0: + resolution: {integrity: sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q==} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -6169,6 +6186,9 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} + es6-promise@4.2.8: + resolution: {integrity: sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==} + esbuild-register@3.6.0: resolution: {integrity: sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg==} peerDependencies: @@ -7314,9 +7334,9 @@ packages: resolution: {integrity: sha512-FIyV/64EkKhJmjgC0g2hygpBv5RNWVPyNCqSAD7eTCv6eFWNIi4PN1UvdSJGicN/o35bnevgis4Y0UDC0qi8jQ==} engines: {node: '>=14.0.0'} - kysely@0.28.2: - resolution: {integrity: sha512-4YAVLoF0Sf0UTqlhgQMFU9iQECdah7n+13ANkiuVfRvlK+uI0Etbgd7bVP36dKlG+NXWbhGua8vnGt+sdhvT7A==} - engines: {node: '>=18.0.0'} + kysely@0.28.5: + resolution: {integrity: sha512-rlB0I/c6FBDWPcQoDtkxi9zIvpmnV5xoIalfCMSMCa7nuA6VGA3F54TW9mEgX4DVf10sXAWCF5fDbamI/5ZpKA==} + engines: {node: '>=20.0.0'} leac@0.6.0: resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==} @@ -7451,6 +7471,10 @@ packages: lodash@4.17.21: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + loglevel@1.9.2: + resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} + engines: {node: '>= 0.6.0'} + long@5.3.2: resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} @@ -7691,8 +7715,8 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - miniflare@4.20250803.0: - resolution: {integrity: sha512-1tmCLfmMw0SqRBF9PPII9CVLQRzOrO7uIBmSng8BMSmtgs2kos7OeoM0sg6KbR9FrvP/zAniLyZuCAMAjuu4fQ==} + miniflare@4.20250813.1: + resolution: {integrity: sha512-6PyXwR4pZmH9ukO0jR5LmhlFVMktsVVGVcUjD9Lpev5QwnqjTRPEv73cnXCe0+oTbIm5TYnvXsAklaWxQuxstA==} engines: {node: '>=18.0.0'} hasBin: true @@ -8053,6 +8077,9 @@ packages: pako@1.0.11: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} + pako@2.1.0: + resolution: {integrity: sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -8450,8 +8477,8 @@ packages: proto-list@1.2.4: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} - protobufjs@7.5.3: - resolution: {integrity: sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw==} + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} engines: {node: '>=12.0.0'} proxy-addr@2.0.7: @@ -9244,8 +9271,8 @@ packages: resolution: {integrity: sha512-5JRxVqC8I8NuOUjzBbvVJAKNM8qoVuH0O77h4WInc/qC2q5IreqKxYwgkga3PfA22OayK2ikceb/B26dztPl+Q==} engines: {node: '>=16'} - supports-color@10.1.0: - resolution: {integrity: sha512-GBuewsPrhJPftT+fqDa9oI/zc5HNsG9nREqwzoSFDOIqf0NggOZbHQj2TE1P1CDJK8ZogFnlZY9hWoUiur7I/A==} + supports-color@10.2.0: + resolution: {integrity: sha512-5eG9FQjEjDbAlI5+kdpdyPIBMRH4GfTVDGREVupaZHmVoppknhM29b/S9BkQz7cathp85BVgRi/As3Siln7e0Q==} engines: {node: '>=18'} supports-color@7.2.0: @@ -9991,20 +10018,20 @@ packages: engines: {node: '>=16'} hasBin: true - workerd@1.20250803.0: - resolution: {integrity: sha512-oYH29mE/wNolPc32NHHQbySaNorj6+KASUtOvQHySxB5mO1NWdGuNv49woxNCF5971UYceGQndY+OLT+24C3wQ==} + workerd@1.20250813.0: + resolution: {integrity: sha512-bDlPGSnb/KESpGFE57cDjgP8mEKDM4WBTd/uGJBsQYCB6Aokk1eK3ivtHoxFx3MfJNo3v6/hJy6KK1b6rw1gvg==} engines: {node: '>=16'} hasBin: true workers-og@0.0.25: resolution: {integrity: sha512-OkTyqCkUCUpGHwMwGmVCMtFPUASf9oBEiCYyOVMBDnUidTQt7AwvDx5EIuCMuQELUGm/tIyvvC8OU/hBsxlBUw==} - wrangler@4.28.1: - resolution: {integrity: sha512-B1w6XS3o1q1Icyx1CyirY5GNyYhucd63Jqml/EYSbB5dgv0VT8ir7L8IkCdbICEa4yYTETIgvTTZqffM6tBulA==} + wrangler@4.30.0: + resolution: {integrity: sha512-NXJUObuXxgG8/ChQ4yXkWLmDQ5ZcO98gyq1yFKYVntJ884C0IpDQrVnAv2RA0ZEz5eB8zal+4OKnr26P3N7ItA==} engines: {node: '>=18.0.0'} hasBin: true peerDependencies: - '@cloudflare/workers-types': ^4.20250803.0 + '@cloudflare/workers-types': ^4.20250813.0 peerDependenciesMeta: '@cloudflare/workers-types': optional: true @@ -10123,9 +10150,6 @@ packages: zod@3.25.67: resolution: {integrity: sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==} - zod@4.0.15: - resolution: {integrity: sha512-2IVHb9h4Mt6+UXkyMs0XbfICUh1eUrlJJAOupBHUhLRnKkruawyDddYRCs0Eizt900ntIMk9/4RksYl+FgSpcQ==} - zustand@4.5.7: resolution: {integrity: sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==} engines: {node: '>=12.7.0'} @@ -10515,9 +10539,8 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} - '@better-auth/utils@0.2.5': + '@better-auth/utils@0.2.6': dependencies: - typescript: 5.8.3 uncrypto: 0.1.3 '@better-fetch/fetch@1.1.18': {} @@ -10556,21 +10579,21 @@ snapshots: '@cloudflare/playwright@0.0.11': {} - '@cloudflare/unenv-preset@2.3.3(unenv@2.0.0-rc.17)(workerd@1.20250803.0)': + '@cloudflare/unenv-preset@2.3.3(unenv@2.0.0-rc.17)(workerd@1.20250813.0)': dependencies: unenv: 2.0.0-rc.17 optionalDependencies: - workerd: 1.20250803.0 + workerd: 1.20250813.0 - '@cloudflare/unenv-preset@2.6.0(unenv@2.0.0-rc.19)(workerd@1.20250803.0)': + '@cloudflare/unenv-preset@2.6.1(unenv@2.0.0-rc.19)(workerd@1.20250813.0)': dependencies: unenv: 2.0.0-rc.19 optionalDependencies: - workerd: 1.20250803.0 + workerd: 1.20250813.0 - '@cloudflare/vite-plugin@1.7.5(rollup@4.44.1)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(workerd@1.20250803.0)(wrangler@4.28.1(@cloudflare/workers-types@4.20250628.0))': + '@cloudflare/vite-plugin@1.7.5(rollup@4.44.1)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(workerd@1.20250813.0)(wrangler@4.30.0(@cloudflare/workers-types@4.20250628.0))': dependencies: - '@cloudflare/unenv-preset': 2.3.3(unenv@2.0.0-rc.17)(workerd@1.20250803.0) + '@cloudflare/unenv-preset': 2.3.3(unenv@2.0.0-rc.17)(workerd@1.20250813.0) '@mjackson/node-fetch-server': 0.6.1 '@rollup/plugin-replace': 6.0.2(rollup@4.44.1) get-port: 7.1.0 @@ -10579,7 +10602,7 @@ snapshots: tinyglobby: 0.2.14 unenv: 2.0.0-rc.17 vite: 6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) - wrangler: 4.28.1(@cloudflare/workers-types@4.20250628.0) + wrangler: 4.30.0(@cloudflare/workers-types@4.20250628.0) ws: 8.18.0 transitivePeerDependencies: - bufferutil @@ -10590,31 +10613,31 @@ snapshots: '@cloudflare/workerd-darwin-64@1.20250617.0': optional: true - '@cloudflare/workerd-darwin-64@1.20250803.0': + '@cloudflare/workerd-darwin-64@1.20250813.0': optional: true '@cloudflare/workerd-darwin-arm64@1.20250617.0': optional: true - '@cloudflare/workerd-darwin-arm64@1.20250803.0': + '@cloudflare/workerd-darwin-arm64@1.20250813.0': optional: true '@cloudflare/workerd-linux-64@1.20250617.0': optional: true - '@cloudflare/workerd-linux-64@1.20250803.0': + '@cloudflare/workerd-linux-64@1.20250813.0': optional: true '@cloudflare/workerd-linux-arm64@1.20250617.0': optional: true - '@cloudflare/workerd-linux-arm64@1.20250803.0': + '@cloudflare/workerd-linux-arm64@1.20250813.0': optional: true '@cloudflare/workerd-windows-64@1.20250617.0': optional: true - '@cloudflare/workerd-windows-64@1.20250803.0': + '@cloudflare/workerd-windows-64@1.20250813.0': optional: true '@cloudflare/workers-types@4.20250628.0': {} @@ -10647,6 +10670,20 @@ snapshots: '@csstools/css-tokenizer@3.0.4': {} + '@datadog/datadog-api-client@1.40.0': + dependencies: + '@types/buffer-from': 1.1.3 + '@types/node': 22.15.29 + '@types/pako': 1.0.7 + buffer-from: 1.1.2 + cross-fetch: 3.2.0 + es6-promise: 4.2.8 + form-data: 4.0.3 + loglevel: 1.9.2 + pako: 2.1.0 + transitivePeerDependencies: + - encoding + '@daybrush/utils@1.13.0': {} '@dnd-kit/accessibility@3.1.1(react@19.1.0)': @@ -11341,7 +11378,7 @@ snapshots: dependencies: '@jridgewell/set-array': 1.2.1 '@jridgewell/sourcemap-codec': 1.5.0 - '@jridgewell/trace-mapping': 0.3.29 + '@jridgewell/trace-mapping': 0.3.25 '@jridgewell/resolve-uri@3.1.2': {} @@ -11758,7 +11795,7 @@ snapshots: '@opentelemetry/sdk-logs': 0.200.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 2.0.0(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 2.0.0(@opentelemetry/api@1.9.0) - protobufjs: 7.5.3 + protobufjs: 7.5.4 '@opentelemetry/redis-common@0.36.2': {} @@ -11899,7 +11936,7 @@ snapshots: dependencies: '@poppinss/colors': 4.1.5 '@sindresorhus/is': 7.0.2 - supports-color: 10.1.0 + supports-color: 10.2.0 '@poppinss/exception@1.2.2': {} @@ -13313,7 +13350,7 @@ snapshots: dependencies: react: 19.1.0 - '@react-router/dev@7.6.3(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(react-router@7.6.3(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(tsx@4.19.4)(typescript@5.8.3)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(wrangler@4.28.1(@cloudflare/workers-types@4.20250628.0))(yaml@2.8.0)': + '@react-router/dev@7.6.3(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(react-router@7.6.3(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(tsx@4.19.4)(typescript@5.8.3)(vite@6.3.5(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0))(wrangler@4.30.0(@cloudflare/workers-types@4.20250628.0))(yaml@2.8.0)': dependencies: '@babel/core': 7.27.7 '@babel/generator': 7.27.5 @@ -13346,7 +13383,7 @@ snapshots: vite-node: 3.2.4(@types/node@22.13.8)(jiti@2.4.2)(lightningcss@1.30.1)(tsx@4.19.4)(yaml@2.8.0) optionalDependencies: typescript: 5.8.3 - wrangler: 4.28.1(@cloudflare/workers-types@4.20250628.0) + wrangler: 4.30.0(@cloudflare/workers-types@4.20250628.0) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -13672,9 +13709,9 @@ snapshots: fflate: 0.7.4 string.prototype.codepointat: 0.2.1 - '@simplewebauthn/browser@13.1.0': {} + '@simplewebauthn/browser@13.1.2': {} - '@simplewebauthn/server@13.1.1': + '@simplewebauthn/server@13.1.2': dependencies: '@hexagon/base64': 1.1.28 '@levischuck/tiny-cbor': 0.2.11 @@ -14126,6 +14163,10 @@ snapshots: dependencies: '@babel/types': 7.28.2 + '@types/buffer-from@1.1.3': + dependencies: + '@types/node': 22.15.29 + '@types/canvas-confetti@1.9.0': {} '@types/chai@5.2.2': @@ -14239,6 +14280,8 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/pako@1.0.7': {} + '@types/pg-pool@2.0.6': dependencies: '@types/pg': 8.6.1 @@ -14815,20 +14858,20 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.3.4(react-dom@19.1.0(react@19.1.0))(react@19.1.0): + better-auth@1.3.7(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(zod@3.25.67): dependencies: - '@better-auth/utils': 0.2.5 + '@better-auth/utils': 0.2.6 '@better-fetch/fetch': 1.1.18 '@noble/ciphers': 0.6.0 '@noble/hashes': 1.8.0 - '@simplewebauthn/browser': 13.1.0 - '@simplewebauthn/server': 13.1.1 + '@simplewebauthn/browser': 13.1.2 + '@simplewebauthn/server': 13.1.2 better-call: 1.0.13 defu: 6.1.4 jose: 5.10.0 - kysely: 0.28.2 + kysely: 0.28.5 nanostores: 0.11.4 - zod: 4.0.15 + zod: 3.25.67 optionalDependencies: react: 19.1.0 react-dom: 19.1.0(react@19.1.0) @@ -15216,6 +15259,12 @@ snapshots: cron-schedule@5.0.4: {} + cross-fetch@3.2.0: + dependencies: + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -15502,13 +15551,13 @@ snapshots: transitivePeerDependencies: - supports-color - drizzle-orm@0.43.1(@cloudflare/workers-types@4.20250628.0)(@opentelemetry/api@1.9.0)(@types/pg@8.6.1)(better-sqlite3@11.10.0)(kysely@0.28.2)(postgres@3.4.5): + drizzle-orm@0.43.1(@cloudflare/workers-types@4.20250628.0)(@opentelemetry/api@1.9.0)(@types/pg@8.6.1)(better-sqlite3@11.10.0)(kysely@0.28.5)(postgres@3.4.5): optionalDependencies: '@cloudflare/workers-types': 4.20250628.0 '@opentelemetry/api': 1.9.0 '@types/pg': 8.6.1 better-sqlite3: 11.10.0 - kysely: 0.28.2 + kysely: 0.28.5 postgres: 3.4.5 dub@0.64.2(@modelcontextprotocol/sdk@1.15.1)(zod@3.25.67): @@ -15741,6 +15790,8 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 + es6-promise@4.2.8: {} + esbuild-register@3.6.0(esbuild@0.25.5): dependencies: debug: 4.4.1 @@ -17116,7 +17167,7 @@ snapshots: kysely@0.27.6: {} - kysely@0.28.2: {} + kysely@0.28.5: {} leac@0.6.0: {} @@ -17231,6 +17282,8 @@ snapshots: lodash@4.17.21: {} + loglevel@1.9.2: {} + long@5.3.2: {} longest-streak@3.1.0: {} @@ -17595,7 +17648,7 @@ snapshots: - bufferutil - utf-8-validate - miniflare@4.20250803.0: + miniflare@4.20250813.1: dependencies: '@cspotcode/source-map-support': 0.8.1 acorn: 8.14.0 @@ -17605,7 +17658,7 @@ snapshots: sharp: 0.33.5 stoppable: 1.1.0 undici: 7.11.0 - workerd: 1.20250803.0 + workerd: 1.20250813.0 ws: 8.18.0 youch: 4.1.0-beta.10 zod: 3.22.3 @@ -18015,6 +18068,8 @@ snapshots: pako@1.0.11: {} + pako@2.1.0: {} + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -18384,7 +18439,7 @@ snapshots: proto-list@1.2.4: {} - protobufjs@7.5.3: + protobufjs@7.5.4: dependencies: '@protobufjs/aspromise': 1.1.2 '@protobufjs/base64': 1.1.2 @@ -19431,7 +19486,7 @@ snapshots: dependencies: copy-anything: 3.0.5 - supports-color@10.1.0: {} + supports-color@10.2.0: {} supports-color@7.2.0: dependencies: @@ -20254,13 +20309,13 @@ snapshots: '@cloudflare/workerd-linux-arm64': 1.20250617.0 '@cloudflare/workerd-windows-64': 1.20250617.0 - workerd@1.20250803.0: + workerd@1.20250813.0: optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20250803.0 - '@cloudflare/workerd-darwin-arm64': 1.20250803.0 - '@cloudflare/workerd-linux-64': 1.20250803.0 - '@cloudflare/workerd-linux-arm64': 1.20250803.0 - '@cloudflare/workerd-windows-64': 1.20250803.0 + '@cloudflare/workerd-darwin-64': 1.20250813.0 + '@cloudflare/workerd-darwin-arm64': 1.20250813.0 + '@cloudflare/workerd-linux-64': 1.20250813.0 + '@cloudflare/workerd-linux-arm64': 1.20250813.0 + '@cloudflare/workerd-windows-64': 1.20250813.0 workers-og@0.0.25: dependencies: @@ -20269,16 +20324,16 @@ snapshots: satori: 0.10.14 yoga-wasm-web: 0.3.3 - wrangler@4.28.1(@cloudflare/workers-types@4.20250628.0): + wrangler@4.30.0(@cloudflare/workers-types@4.20250628.0): dependencies: '@cloudflare/kv-asset-handler': 0.4.0 - '@cloudflare/unenv-preset': 2.6.0(unenv@2.0.0-rc.19)(workerd@1.20250803.0) + '@cloudflare/unenv-preset': 2.6.1(unenv@2.0.0-rc.19)(workerd@1.20250813.0) blake3-wasm: 2.1.5 esbuild: 0.25.4 - miniflare: 4.20250803.0 + miniflare: 4.20250813.1 path-to-regexp: 6.3.0 unenv: 2.0.0-rc.19 - workerd: 1.20250803.0 + workerd: 1.20250813.0 optionalDependencies: '@cloudflare/workers-types': 4.20250628.0 fsevents: 2.3.3 @@ -20387,8 +20442,6 @@ snapshots: zod@3.25.67: {} - zod@4.0.15: {} - zustand@4.5.7(@types/react@19.0.10)(react@19.1.0): dependencies: use-sync-external-store: 1.5.0(react@19.1.0) From 191f550fc97d8a2cb3b1731b1b034c455eaf35c1 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Fri, 8 Aug 2025 17:43:18 +0400 Subject: [PATCH 04/13] removed whitespace --- apps/server/src/lib/logging-durable-object.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts index d348d2bc0e..d208948161 100644 --- a/apps/server/src/lib/logging-durable-object.ts +++ b/apps/server/src/lib/logging-durable-object.ts @@ -60,7 +60,7 @@ export class LoggingDurableObject extends DurableObject { if (timeSinceLastActivity > this.SESSION_TIMEOUT && currentState.calls.length > 0) { // Export expired session to Datadog await this.exportSessionToDatadog(currentState); - + // Start new session await this.clearSession(); const newState = await this.getState(); @@ -158,7 +158,7 @@ export class LoggingDurableObject extends DurableObject { async exportSessionToDatadog(state: LoggingState): Promise { if (state.calls.length === 0) return; - + try { await this.datadogService.exportSessionLogs( state.sessionId, From 61027383b0b22026bda39f4ce90e050b3aa466ba Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Mon, 11 Aug 2025 20:34:35 +0400 Subject: [PATCH 05/13] datadog fixes --- apps/server/src/lib/datadog-service.ts | 228 +++++++++++++++--- apps/server/src/lib/logging-durable-object.ts | 9 + apps/server/src/trpc/trpc.ts | 22 ++ 3 files changed, 221 insertions(+), 38 deletions(-) diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts index 6662f5c183..34f95ce032 100644 --- a/apps/server/src/lib/datadog-service.ts +++ b/apps/server/src/lib/datadog-service.ts @@ -4,6 +4,9 @@ import type { ZeroEnv } from '../env'; export class DatadogService { private apiInstance: v2.LogsApi; + private apiKey: string; + private appKey: string; + private site: string; constructor(env?: ZeroEnv) { const configuration = client.createConfiguration({ @@ -19,48 +22,132 @@ export class DatadogService { } this.apiInstance = new v2.LogsApi(configuration); + this.apiKey = env?.DD_API_KEY || ''; + this.appKey = env?.DD_APP_KEY || ''; + this.site = env?.DD_SITE || 'datadoghq.com'; + } + + // Simple hash function for generating consistent trace and span IDs + // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces + // Trace IDs must be 32-character lowercase hexadecimal strings + // Span IDs must be 16-character lowercase hexadecimal strings + private simpleHash(str: string): string { + let hash = 0; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32-bit integer + } + // Convert to positive hex string and pad to 32 characters + const positiveHash = Math.abs(hash).toString(16); + return positiveHash.padEnd(32, '0').slice(0, 32).toLowerCase(); } async exportSessionLogs(sessionId: string, userId: string, logs: TRPCCallLog[]): Promise { if (logs.length === 0) return; try { - const logEntries = logs.map(log => ({ - message: `TRPC call: ${log.procedure}`, - service: 'zero-mail-app', - ddsource: 'trpc-logging', - ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure}`, - hostname: 'cloudflare-worker', - timestamp: Date.now(), - // Custom fields for TRPC data - trpc: { + const logEntries = logs.map((log, index) => { + // Generate consistent trace and span IDs for distributed tracing + // Use session-based trace ID for correlation across all calls in a session + // Must be 32-character lowercase hexadecimal strings for trace_id + // Must be 16-character lowercase hexadecimal strings for span_id + + // Create a consistent trace ID based on session and user + // Using a simple hash function that doesn't require async operations + // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces + const sessionString = `${sessionId}-${userId}`; + const traceId = this.simpleHash(sessionString).slice(0, 32); + + // Create a span ID based on the specific call + const callString = `${log.id}-${log.procedure}-${index}`; + const spanId = this.simpleHash(callString).slice(0, 16); + + // Include ALL possible information in the logs for maximum visibility + return { + message: `TRPC call: ${log.procedure} (${log.duration}ms)`, + service: 'zero-mail-app', + ddsource: 'trpc-logging', + ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${!!log.error}`, + hostname: 'cloudflare-worker', + timestamp: log.timestamp, + // Trace correlation fields - must be nested in 'dd' object // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces + dd: { + trace_id: traceId, + span_id: spanId, + }, + // Environment and version for better correlation + env: 'development', + version: '1.0.0', + // Rich context at root level for better searchability and correlation + // Include ALL possible information for maximum visibility procedure: log.procedure, - input: log.input, - output: log.output, - error: log.error, duration: log.duration, - sessionId, - userId, - }, - })); + session_id: sessionId, + user_id: userId, + call_id: log.id, + performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + has_error: !!log.error, + error_message: log.error || null, + error_type: log.error ? 'trpc_error' : null, + // Additional context for better debugging + call_sequence: index + 1, + total_calls_in_session: logs.length, + session_start_time: logs[0]?.timestamp || log.timestamp, + session_duration: logs.length > 1 ? (logs[logs.length - 1]?.timestamp || log.timestamp) - (logs[0]?.timestamp || log.timestamp) : 0, + // HTTP context + http_method: 'POST', + http_url: `/api/trpc/${log.procedure}`, + // Request context (if available) + request_id: log.id, + request_timestamp: log.timestamp, + // Performance metrics + response_time_ms: log.duration, + response_time_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + // Error details + error_details: log.error ? { + message: log.error, + type: 'trpc_error', + procedure: log.procedure, + timestamp: log.timestamp + } : null, + // Input/Output data (truncated for safety) + input_data: log.input ? (typeof log.input === 'string' ? log.input.slice(0, 1000) : JSON.stringify(log.input).slice(0, 1000)) : null, + output_data: log.output ? (typeof log.output === 'string' ? log.output.slice(0, 1000) : JSON.stringify(log.output).slice(0, 1000)) : null, + // Metadata + metadata: { + session_id: sessionId, + user_id: userId, + call_id: log.id, + procedure: log.procedure, + duration: log.duration, + timestamp: log.timestamp, + has_error: !!log.error, + error_message: log.error || null, + call_sequence: index + 1, + total_calls_in_session: logs.length, + performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + } + }; + }); console.log('📊 Sending to Datadog:', { sessionId, userId, logCount: logs.length, - // logEntries: logEntries + sampleLogEntry: logEntries[0], // Show the first log entry structure }); - const params = { + // Send logs + const logParams = { body: logEntries, }; - - await this.apiInstance.submitLog(params); + await this.apiInstance.submitLog(logParams); console.log('✅ Successfully exported to Datadog:', { sessionId, userId, - logCount: logs.length + logCount: logs.length, }); } catch (error) { console.error('❌ Failed to export session logs to Datadog:', error); @@ -74,23 +161,88 @@ export class DatadogService { const allLogEntries: any[] = []; for (const { sessionId, userId, logs: sessionLogs } of logs) { - const logEntries = sessionLogs.map(log => ({ - message: `TRPC call: ${log.procedure}`, - service: 'zero-mail-app', - ddsource: 'trpc-logging', - ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure}`, - hostname: 'cloudflare-worker', - timestamp: Date.now(), - trpc: { + const logEntries = sessionLogs.map((log, index) => { + // Generate consistent trace and span IDs for distributed tracing + // Use session-based trace ID for correlation across all calls in a session + // Must be 32-character lowercase hexadecimal strings for trace_id + // Must be 16-character lowercase hexadecimal strings for span_id + + // Create a consistent trace ID based on session and user + const sessionString = `${sessionId}-${userId}`; + const traceId = this.simpleHash(sessionString).slice(0, 32); + + // Create a span ID based on the specific call + const callString = `${log.id}-${log.procedure}-${index}`; + const spanId = this.simpleHash(callString).slice(0, 16); + + // Include ALL possible information in the logs for maximum visibility + return { + message: `TRPC call: ${log.procedure} (${log.duration}ms)`, + service: 'zero-mail-app', + ddsource: 'trpc-logging', + ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${!!log.error}`, + hostname: 'cloudflare-worker', + timestamp: log.timestamp, + // Trace correlation fields - must be nested in 'dd' object + // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces + dd: { + trace_id: traceId, + span_id: spanId, + }, + // Environment and version for better correlation + env: 'development', + version: '1.0.0', + // Rich context at root level for better searchability and correlation + // Include ALL possible information for maximum visibility procedure: log.procedure, - input: log.input, - output: log.output, - error: log.error, duration: log.duration, - sessionId, - userId, - }, - })); + session_id: sessionId, + user_id: userId, + call_id: log.id, + performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + has_error: !!log.error, + error_message: log.error || null, + error_type: log.error ? 'trpc_error' : null, + // Additional context for better debugging + call_sequence: index + 1, + total_calls_in_session: sessionLogs.length, + session_start_time: sessionLogs[0]?.timestamp || log.timestamp, + session_duration: sessionLogs.length > 1 ? (sessionLogs[sessionLogs.length - 1]?.timestamp || log.timestamp) - (sessionLogs[0]?.timestamp || log.timestamp) : 0, + // HTTP context + http_method: 'POST', + http_url: `/api/trpc/${log.procedure}`, + // Request context (if available) + request_id: log.id, + request_timestamp: log.timestamp, + // Performance metrics + response_time_ms: log.duration, + response_time_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + // Error details + error_details: log.error ? { + message: log.error, + type: 'trpc_error', + procedure: log.procedure, + timestamp: log.timestamp + } : null, + // Input/Output data (truncated for safety) + input_data: log.input ? (typeof log.input === 'string' ? log.input.slice(0, 1000) : JSON.stringify(log.input).slice(0, 1000)) : null, + output_data: log.output ? (typeof log.output === 'string' ? log.output.slice(0, 1000) : JSON.stringify(log.output).slice(0, 1000)) : null, + // Metadata + metadata: { + session_id: sessionId, + user_id: userId, + call_id: log.id, + procedure: log.procedure, + duration: log.duration, + timestamp: log.timestamp, + has_error: !!log.error, + error_message: log.error || null, + call_sequence: index + 1, + total_calls_in_session: sessionLogs.length, + performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + } + }; + }); allLogEntries.push(...logEntries); } @@ -102,11 +254,11 @@ export class DatadogService { }); if (allLogEntries.length > 0) { - const params = { + const logParams: v2.LogsApiSubmitLogRequest = { body: allLogEntries, }; - await this.apiInstance.submitLog(params); + await this.apiInstance.submitLog(logParams); } } catch (error) { console.error('Failed to export batch logs to Datadog:', error); diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts index d208948161..2f351c4e61 100644 --- a/apps/server/src/lib/logging-durable-object.ts +++ b/apps/server/src/lib/logging-durable-object.ts @@ -17,6 +17,15 @@ export interface TRPCCallLog { userAgent?: string; ip?: string; method: 'query' | 'mutation' | 'subscription'; + // Additional metadata + referer?: string; + origin?: string; + acceptLanguage?: string; + acceptEncoding?: string; + requestId?: string; + timestamp?: string; + startTime?: number; + endTime?: number; }; } diff --git a/apps/server/src/trpc/trpc.ts b/apps/server/src/trpc/trpc.ts index baf6c12ee2..f9aab9f821 100644 --- a/apps/server/src/trpc/trpc.ts +++ b/apps/server/src/trpc/trpc.ts @@ -38,6 +38,17 @@ const createLoggingMiddleware = () => { userAgent: ctx.c.req.header('user-agent'), ip: getConnInfo(ctx.c).remote.address, method: type, + // Additional metadata + referer: ctx.c.req.header('referer'), + origin: ctx.c.req.header('origin'), + acceptLanguage: ctx.c.req.header('accept-language'), + acceptEncoding: ctx.c.req.header('accept-encoding'), + // Request context + requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), + timestamp: new Date().toISOString(), + // Performance context + startTime, + endTime: Date.now(), }, }).catch(err => console.error('Failed to log TRPC call:', err)); @@ -55,6 +66,17 @@ const createLoggingMiddleware = () => { userAgent: ctx.c.req.header('user-agent'), ip: getConnInfo(ctx.c).remote.address, method: type, + // Additional metadata + referer: ctx.c.req.header('referer'), + origin: ctx.c.req.header('origin'), + acceptLanguage: ctx.c.req.header('accept-language'), + acceptEncoding: ctx.c.req.header('accept-encoding'), + // Request context + requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), + timestamp: new Date().toISOString(), + // Performance context + startTime, + endTime: Date.now(), }, }).catch(err => console.error('Failed to log TRPC call:', err)); From efebf19dce7dd8fe8383b841b8618575af8fccba Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Tue, 12 Aug 2025 01:38:56 +0400 Subject: [PATCH 06/13] FIXES JSON ON ROOT!!!!! --- apps/server/src/lib/datadog-service.ts | 327 +++++++++++-------------- 1 file changed, 146 insertions(+), 181 deletions(-) diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts index 34f95ce032..cde5eca0b2 100644 --- a/apps/server/src/lib/datadog-service.ts +++ b/apps/server/src/lib/datadog-service.ts @@ -43,90 +43,169 @@ export class DatadogService { return positiveHash.padEnd(32, '0').slice(0, 32).toLowerCase(); } + // Check if a procedure is logging-related to avoid recursive logging + private isLoggingProcedure(procedure: string): boolean { + const loggingProcedures = [ + 'logging.getSessionStats', + 'logging.getCallHistory', + 'logging.clearSession', + 'logging.getSessionState', + 'logging.exportToDatadog', + 'logging.endSession' + ]; + return loggingProcedures.includes(procedure); + } + async exportSessionLogs(sessionId: string, userId: string, logs: TRPCCallLog[]): Promise { if (logs.length === 0) return; + // Filter out logging-related procedures to avoid recursive logging + const filteredLogs = logs.filter(log => !this.isLoggingProcedure(log.procedure)); + + if (filteredLogs.length === 0) return; + try { - const logEntries = logs.map((log, index) => { + const logEntries = filteredLogs.map((log, index) => { // Generate consistent trace and span IDs for distributed tracing - // Use session-based trace ID for correlation across all calls in a session - // Must be 32-character lowercase hexadecimal strings for trace_id - // Must be 16-character lowercase hexadecimal strings for span_id - - // Create a consistent trace ID based on session and user - // Using a simple hash function that doesn't require async operations - // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces const sessionString = `${sessionId}-${userId}`; const traceId = this.simpleHash(sessionString).slice(0, 32); - - // Create a span ID based on the specific call const callString = `${log.id}-${log.procedure}-${index}`; const spanId = this.simpleHash(callString).slice(0, 16); - // Include ALL possible information in the logs for maximum visibility + // Calculate performance category once + const performanceCategory = log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow'; + const hasError = !!log.error; + + // Determine log level based on error status and performance + const logLevel = hasError ? 'ERROR' : performanceCategory === 'slow' ? 'WARNING' : 'INFO'; + + // Parse user agent for device/browser info + const parseUserAgent = (userAgent?: string) => { + if (!userAgent) return {}; + + const browsers = { + chrome: /Chrome\/([0-9.]+)/i, + firefox: /Firefox\/([0-9.]+)/i, + safari: /Safari\/([0-9.]+)/i, + edge: /Edg\/([0-9.]+)/i, + }; + + const os = { + windows: /Windows NT ([0-9.]+)/i, + macos: /Mac OS X ([0-9_.]+)/i, + linux: /Linux/i, + android: /Android ([0-9.]+)/i, + ios: /OS ([0-9_]+)/i, + }; + + const devices = { + mobile: /Mobile|Android|iPhone/i, + tablet: /iPad|Tablet/i, + desktop: /Windows|Mac|Linux/i, + }; + + let browser = 'unknown', browserVersion = '', operatingSystem = 'unknown', osVersion = '', deviceType = 'unknown'; + + // Detect browser + for (const [name, regex] of Object.entries(browsers)) { + const match = userAgent.match(regex); + if (match) { + browser = name; + browserVersion = match[1]; + break; + } + } + + // Detect OS + for (const [name, regex] of Object.entries(os)) { + const match = userAgent.match(regex); + if (match) { + operatingSystem = name; + osVersion = match[1]?.replace(/_/g, '.') || ''; + break; + } + } + + // Detect device type + for (const [type, regex] of Object.entries(devices)) { + if (regex.test(userAgent)) { + deviceType = type; + break; + } + } + + return { + browser, + browser_version: browserVersion, + operating_system: operatingSystem, + os_version: osVersion, + device_type: deviceType, + user_agent: userAgent, + }; + }; + + const deviceInfo = parseUserAgent(log.metadata?.userAgent); + return { - message: `TRPC call: ${log.procedure} (${log.duration}ms)`, + message: `${logLevel}: TRPC call: [${log.procedure}] (${log.duration}ms)`, service: 'zero-mail-app', ddsource: 'trpc-logging', - ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${!!log.error}`, + ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${hasError},performance:${performanceCategory},browser:${deviceInfo.browser},device:${deviceInfo.device_type}`, hostname: 'cloudflare-worker', - timestamp: log.timestamp, - // Trace correlation fields - must be nested in 'dd' object // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces + // Trace correlation fields dd: { trace_id: traceId, span_id: spanId, }, - // Environment and version for better correlation - env: 'development', - version: '1.0.0', - // Rich context at root level for better searchability and correlation - // Include ALL possible information for maximum visibility - procedure: log.procedure, - duration: log.duration, - session_id: sessionId, - user_id: userId, - call_id: log.id, - performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', - has_error: !!log.error, - error_message: log.error || null, - error_type: log.error ? 'trpc_error' : null, - // Additional context for better debugging - call_sequence: index + 1, - total_calls_in_session: logs.length, - session_start_time: logs[0]?.timestamp || log.timestamp, - session_duration: logs.length > 1 ? (logs[logs.length - 1]?.timestamp || log.timestamp) - (logs[0]?.timestamp || log.timestamp) : 0, - // HTTP context - http_method: 'POST', - http_url: `/api/trpc/${log.procedure}`, - // Request context (if available) - request_id: log.id, - request_timestamp: log.timestamp, - // Performance metrics - response_time_ms: log.duration, - response_time_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', - // Error details - error_details: log.error ? { - message: log.error, - type: 'trpc_error', - procedure: log.procedure, - timestamp: log.timestamp - } : null, - // Input/Output data (truncated for safety) - input_data: log.input ? (typeof log.input === 'string' ? log.input.slice(0, 1000) : JSON.stringify(log.input).slice(0, 1000)) : null, - output_data: log.output ? (typeof log.output === 'string' ? log.output.slice(0, 1000) : JSON.stringify(log.output).slice(0, 1000)) : null, - // Metadata - metadata: { - session_id: sessionId, - user_id: userId, + + additionalProperties: { + // Core call data call_id: log.id, procedure: log.procedure, duration: log.duration, - timestamp: log.timestamp, - has_error: !!log.error, - error_message: log.error || null, + performance_category: performanceCategory, + trpc_method: log.metadata?.method || 'unknown', + + // Session context + session_id: sessionId, + user_id: userId, call_sequence: index + 1, - total_calls_in_session: logs.length, - performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', + total_calls_in_session: filteredLogs.length, + + // HTTP context + http_method: 'POST', + http_url: `/api/trpc/${log.procedure}`, + client_ip: log.metadata?.ip, + referer: log.metadata?.referer, + origin: log.metadata?.origin, + accept_language: log.metadata?.acceptLanguage, + accept_encoding: log.metadata?.acceptEncoding, + request_id: log.metadata?.requestId, + + // Device and browser information + ...deviceInfo, + + // Error handling + has_error: hasError, + log_level: logLevel, + ...(log.error && { + error_message: log.error, + error_type: 'trpc_error', + }), + + // Full request/response data + request_payload: log.input, + ...(log.output && { + response_payload: log.output, + }), + + // Performance metrics + timing: { + start_time: log.metadata?.startTime || log.timestamp, + end_time: log.metadata?.endTime || (log.timestamp + log.duration), + duration_ms: log.duration, + performance_category: performanceCategory, + }, } }; }); @@ -134,134 +213,20 @@ export class DatadogService { console.log('📊 Sending to Datadog:', { sessionId, userId, - logCount: logs.length, - sampleLogEntry: logEntries[0], // Show the first log entry structure + totalLogs: logs.length, + filteredLogs: filteredLogs.length, + excludedLoggingCalls: logs.length - filteredLogs.length, }); - // Send logs - const logParams = { - body: logEntries, - }; - await this.apiInstance.submitLog(logParams); + await this.apiInstance.submitLog({ body: logEntries }); console.log('✅ Successfully exported to Datadog:', { sessionId, userId, - logCount: logs.length, + exportedLogs: filteredLogs.length, }); } catch (error) { console.error('❌ Failed to export session logs to Datadog:', error); } } - - async exportBatchLogs(logs: Array<{ sessionId: string; userId: string; logs: TRPCCallLog[] }>): Promise { - if (logs.length === 0) return; - - try { - const allLogEntries: any[] = []; - - for (const { sessionId, userId, logs: sessionLogs } of logs) { - const logEntries = sessionLogs.map((log, index) => { - // Generate consistent trace and span IDs for distributed tracing - // Use session-based trace ID for correlation across all calls in a session - // Must be 32-character lowercase hexadecimal strings for trace_id - // Must be 16-character lowercase hexadecimal strings for span_id - - // Create a consistent trace ID based on session and user - const sessionString = `${sessionId}-${userId}`; - const traceId = this.simpleHash(sessionString).slice(0, 32); - - // Create a span ID based on the specific call - const callString = `${log.id}-${log.procedure}-${index}`; - const spanId = this.simpleHash(callString).slice(0, 16); - - // Include ALL possible information in the logs for maximum visibility - return { - message: `TRPC call: ${log.procedure} (${log.duration}ms)`, - service: 'zero-mail-app', - ddsource: 'trpc-logging', - ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${!!log.error}`, - hostname: 'cloudflare-worker', - timestamp: log.timestamp, - // Trace correlation fields - must be nested in 'dd' object - // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces - dd: { - trace_id: traceId, - span_id: spanId, - }, - // Environment and version for better correlation - env: 'development', - version: '1.0.0', - // Rich context at root level for better searchability and correlation - // Include ALL possible information for maximum visibility - procedure: log.procedure, - duration: log.duration, - session_id: sessionId, - user_id: userId, - call_id: log.id, - performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', - has_error: !!log.error, - error_message: log.error || null, - error_type: log.error ? 'trpc_error' : null, - // Additional context for better debugging - call_sequence: index + 1, - total_calls_in_session: sessionLogs.length, - session_start_time: sessionLogs[0]?.timestamp || log.timestamp, - session_duration: sessionLogs.length > 1 ? (sessionLogs[sessionLogs.length - 1]?.timestamp || log.timestamp) - (sessionLogs[0]?.timestamp || log.timestamp) : 0, - // HTTP context - http_method: 'POST', - http_url: `/api/trpc/${log.procedure}`, - // Request context (if available) - request_id: log.id, - request_timestamp: log.timestamp, - // Performance metrics - response_time_ms: log.duration, - response_time_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', - // Error details - error_details: log.error ? { - message: log.error, - type: 'trpc_error', - procedure: log.procedure, - timestamp: log.timestamp - } : null, - // Input/Output data (truncated for safety) - input_data: log.input ? (typeof log.input === 'string' ? log.input.slice(0, 1000) : JSON.stringify(log.input).slice(0, 1000)) : null, - output_data: log.output ? (typeof log.output === 'string' ? log.output.slice(0, 1000) : JSON.stringify(log.output).slice(0, 1000)) : null, - // Metadata - metadata: { - session_id: sessionId, - user_id: userId, - call_id: log.id, - procedure: log.procedure, - duration: log.duration, - timestamp: log.timestamp, - has_error: !!log.error, - error_message: log.error || null, - call_sequence: index + 1, - total_calls_in_session: sessionLogs.length, - performance_category: log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow', - } - }; - }); - - allLogEntries.push(...logEntries); - } - - console.log('📊 Sending batch to Datadog:', { - sessionCount: logs.length, - totalLogCount: allLogEntries.length, - sessions: logs.map(l => ({ sessionId: l.sessionId, userId: l.userId, logCount: l.logs.length })) - }); - - if (allLogEntries.length > 0) { - const logParams: v2.LogsApiSubmitLogRequest = { - body: allLogEntries, - }; - - await this.apiInstance.submitLog(logParams); - } - } catch (error) { - console.error('Failed to export batch logs to Datadog:', error); - } - } } \ No newline at end of file From 84ffed83e140a0ef407e66619d7a38e1235410e8 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Mon, 18 Aug 2025 16:46:59 +0400 Subject: [PATCH 07/13] Remove demo routes and pages; enhance logging with request tracing - Deleted demo-related routes and pages from the mail application. - Updated logging functionality to include request tracing with trace IDs and request IDs for better tracking in Datadog. - Improved error handling and logging structure in the server context and TRPC middleware. --- apps/mail/app/(routes)/demo/logging/page.tsx | 135 -------- apps/mail/app/(routes)/demo/page.tsx | 19 -- apps/mail/app/routes.ts | 2 - apps/server/src/ctx.ts | 2 + apps/server/src/lib/datadog-service.ts | 296 ++++++++---------- apps/server/src/lib/logging-durable-object.ts | 92 +++--- apps/server/src/lib/trace-context.ts | 146 +++++++++ apps/server/src/lib/trpc-logging.ts | 159 +++++++++- apps/server/src/main.ts | 109 ++++++- apps/server/src/trpc/trpc.ts | 192 +++++++----- 10 files changed, 701 insertions(+), 451 deletions(-) delete mode 100644 apps/mail/app/(routes)/demo/logging/page.tsx delete mode 100644 apps/mail/app/(routes)/demo/page.tsx create mode 100644 apps/server/src/lib/trace-context.ts diff --git a/apps/mail/app/(routes)/demo/logging/page.tsx b/apps/mail/app/(routes)/demo/logging/page.tsx deleted file mode 100644 index 0db0ce8618..0000000000 --- a/apps/mail/app/(routes)/demo/logging/page.tsx +++ /dev/null @@ -1,135 +0,0 @@ -import { useEffect, useState } from 'react'; - -export default function LoggingDemo() { - const [sessionStats, setSessionStats] = useState(null); - const [callHistory, setCallHistory] = useState(null); - const [loading, setLoading] = useState(false); - - const fetchData = async () => { - setLoading(true); - try { - // Fetch session stats - const statsResponse = await fetch('http://localhost:8787/api/trpc/logging.getSessionStats', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Cookie': document.cookie // Include auth cookies - }, - credentials: 'include', - body: JSON.stringify({ json: {} }), - }); - const statsData = await statsResponse.json(); - setSessionStats(statsData); - - // Fetch call history - const historyResponse = await fetch('http://localhost:8787/api/trpc/logging.getCallHistory', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Cookie': document.cookie // Include auth cookies - }, - credentials: 'include', - body: JSON.stringify({ json: { limit: 100 } }), - }); - const historyData = await historyResponse.json(); - setCallHistory(historyData); - } catch (error) { - console.error('Error fetching data:', error); - } finally { - setLoading(false); - } - }; - - const exportToDatadog = async () => { - try { - const response = await fetch('http://localhost:8787/api/trpc/logging.exportToDatadog', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Cookie': document.cookie - }, - credentials: 'include', - body: JSON.stringify({ json: {} }), - }); - const result = await response.json(); - console.log('Datadog export result:', result); - console.log('Session exported to Datadog!'); - } catch (error) { - console.error('Error exporting to Datadog:', error); - console.error('Failed to export to Datadog'); - } - }; - - const endSession = async () => { - try { - const response = await fetch('http://localhost:8787/api/trpc/logging.endSession', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Cookie': document.cookie - }, - credentials: 'include', - body: JSON.stringify({ json: {} }), - }); - const result = await response.json(); - console.log('Session end result:', result); - console.log('Session ended and exported to Datadog!'); - fetchData(); // Refresh data - } catch (error) { - console.error('Error ending session:', error); - console.error('Failed to end session'); - } - }; - - useEffect(() => { - fetchData(); - }, []); - - return ( -
-

TRPC Logging Demo

- -
- - - -
- -
-
-

Session Stats

-
- 
-                            {sessionStats ? JSON.stringify(sessionStats, null, 2) : 'Loading...'}
-
-
-
- -
-

Call History

-
- 
-                            {callHistory ? JSON.stringify(callHistory, null, 2) : 'Loading...'}
-
-
-
-
-
- ); -} \ No newline at end of file diff --git a/apps/mail/app/(routes)/demo/page.tsx b/apps/mail/app/(routes)/demo/page.tsx deleted file mode 100644 index c5b1e1d637..0000000000 --- a/apps/mail/app/(routes)/demo/page.tsx +++ /dev/null @@ -1,19 +0,0 @@ -import { Link } from 'react-router'; - -export default function DemoPage() { - return ( -
-

TRPC Call Logging

-

- Simple JSON output of all TRPC calls with routes, inputs, and outputs. -

- -
-

Session Logging

- - /demo/logging - View all TRPC calls JSON - -
-
- ); -} \ No newline at end of file diff --git a/apps/mail/app/routes.ts b/apps/mail/app/routes.ts index 82a5cb3e40..57d9fbe23f 100644 --- a/apps/mail/app/routes.ts +++ b/apps/mail/app/routes.ts @@ -23,8 +23,6 @@ export default [ layout('(routes)/layout.tsx', [ route('/developer', '(routes)/developer/page.tsx'), - route('/demo', '(routes)/demo/page.tsx'), - route('/demo/logging', '(routes)/demo/logging/page.tsx'), layout( '(routes)/mail/layout.tsx', prefix('/mail', [ diff --git a/apps/server/src/ctx.ts b/apps/server/src/ctx.ts index 681f3b8f67..83ec8e2950 100644 --- a/apps/server/src/ctx.ts +++ b/apps/server/src/ctx.ts @@ -8,6 +8,8 @@ export type HonoVariables = { auth: Auth; sessionUser?: SessionUser; autumn?: Autumn; + traceId?: string; + requestId?: string; }; export type HonoContext = { Variables: HonoVariables; Bindings: ZeroEnv }; diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts index cde5eca0b2..aa7944f1ad 100644 --- a/apps/server/src/lib/datadog-service.ts +++ b/apps/server/src/lib/datadog-service.ts @@ -56,177 +56,157 @@ export class DatadogService { return loggingProcedures.includes(procedure); } - async exportSessionLogs(sessionId: string, userId: string, logs: TRPCCallLog[]): Promise { - if (logs.length === 0) return; + async logSingleCall(sessionId: string, userId: string, log: TRPCCallLog): Promise { + // Skip logging-related procedures to avoid recursive logging + if (this.isLoggingProcedure(log.procedure)) { + return; + } - // Filter out logging-related procedures to avoid recursive logging - const filteredLogs = logs.filter(log => !this.isLoggingProcedure(log.procedure)); + try { + const traceId = this.simpleHash(`${sessionId}-${userId}`).slice(0, 32); + const spanId = this.simpleHash(`${log.id}-${log.procedure}-${log.timestamp}`).slice(0, 16); + + const performanceCategory = log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow'; + const hasError = !!log.error; + const logLevel = hasError ? 'ERROR' : performanceCategory === 'slow' ? 'WARNING' : 'INFO'; + + // Parse user agent for device/browser info + const parseUserAgent = (userAgent?: string) => { + if (!userAgent) return {}; + + const browsers = { + chrome: /Chrome\/([0-9.]+)/i, + firefox: /Firefox\/([0-9.]+)/i, + safari: /Safari\/([0-9.]+)/i, + edge: /Edg\/([0-9.]+)/i, + }; - if (filteredLogs.length === 0) return; + const os = { + windows: /Windows NT ([0-9.]+)/i, + macos: /Mac OS X ([0-9_.]+)/i, + linux: /Linux/i, + android: /Android ([0-9.]+)/i, + ios: /OS ([0-9_]+)/i, + }; - try { - const logEntries = filteredLogs.map((log, index) => { - // Generate consistent trace and span IDs for distributed tracing - const sessionString = `${sessionId}-${userId}`; - const traceId = this.simpleHash(sessionString).slice(0, 32); - const callString = `${log.id}-${log.procedure}-${index}`; - const spanId = this.simpleHash(callString).slice(0, 16); - - // Calculate performance category once - const performanceCategory = log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow'; - const hasError = !!log.error; - - // Determine log level based on error status and performance - const logLevel = hasError ? 'ERROR' : performanceCategory === 'slow' ? 'WARNING' : 'INFO'; - - // Parse user agent for device/browser info - const parseUserAgent = (userAgent?: string) => { - if (!userAgent) return {}; - - const browsers = { - chrome: /Chrome\/([0-9.]+)/i, - firefox: /Firefox\/([0-9.]+)/i, - safari: /Safari\/([0-9.]+)/i, - edge: /Edg\/([0-9.]+)/i, - }; - - const os = { - windows: /Windows NT ([0-9.]+)/i, - macos: /Mac OS X ([0-9_.]+)/i, - linux: /Linux/i, - android: /Android ([0-9.]+)/i, - ios: /OS ([0-9_]+)/i, - }; - - const devices = { - mobile: /Mobile|Android|iPhone/i, - tablet: /iPad|Tablet/i, - desktop: /Windows|Mac|Linux/i, - }; - - let browser = 'unknown', browserVersion = '', operatingSystem = 'unknown', osVersion = '', deviceType = 'unknown'; - - // Detect browser - for (const [name, regex] of Object.entries(browsers)) { - const match = userAgent.match(regex); - if (match) { - browser = name; - browserVersion = match[1]; - break; - } - } + const devices = { + mobile: /Mobile|Android|iPhone/i, + tablet: /iPad|Tablet/i, + desktop: /Windows|Mac|Linux/i, + }; - // Detect OS - for (const [name, regex] of Object.entries(os)) { - const match = userAgent.match(regex); - if (match) { - operatingSystem = name; - osVersion = match[1]?.replace(/_/g, '.') || ''; - break; - } + let browser = 'unknown', browserVersion = '', operatingSystem = 'unknown', osVersion = '', deviceType = 'unknown'; + + // Detect browser + for (const [name, regex] of Object.entries(browsers)) { + const match = userAgent.match(regex); + if (match) { + browser = name; + browserVersion = match[1]; + break; + } + } + + // Detect OS + for (const [name, regex] of Object.entries(os)) { + const match = userAgent.match(regex); + if (match) { + operatingSystem = name; + osVersion = match[1]?.replace(/_/g, '.') || ''; + break; } + } - // Detect device type - for (const [type, regex] of Object.entries(devices)) { - if (regex.test(userAgent)) { - deviceType = type; - break; - } + // Detect device type + for (const [type, regex] of Object.entries(devices)) { + if (regex.test(userAgent)) { + deviceType = type; + break; } + } - return { - browser, - browser_version: browserVersion, - operating_system: operatingSystem, - os_version: osVersion, - device_type: deviceType, - user_agent: userAgent, - }; + return { + browser, + browser_version: browserVersion, + operating_system: operatingSystem, + os_version: osVersion, + device_type: deviceType, + user_agent: userAgent, }; + }; + + const deviceInfo = parseUserAgent(log.metadata?.userAgent); + + const logEntry = { + message: `${logLevel}: TRPC call: [${log.procedure}] (${log.duration}ms)`, + service: 'zero-mail-app', + ddsource: 'trpc-logging', + ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${hasError},performance:${performanceCategory},browser:${deviceInfo.browser},device:${deviceInfo.device_type}`, + hostname: 'cloudflare-worker', + timestamp: log.timestamp, + + // Trace correlation fields + dd: { + trace_id: traceId, + span_id: spanId, + }, + + additionalProperties: { + // Core call data + call_id: log.id, + procedure: log.procedure, + duration: log.duration, + performance_category: performanceCategory, + trpc_method: log.metadata?.method || 'unknown', + + // Session context + session_id: sessionId, + user_id: userId, + + // HTTP context + http_method: 'POST', + http_url: `/api/trpc/${log.procedure}`, + client_ip: log.metadata?.ip, + referer: log.metadata?.referer, + origin: log.metadata?.origin, + accept_language: log.metadata?.acceptLanguage, + accept_encoding: log.metadata?.acceptEncoding, + request_id: log.metadata?.requestId, + + // Device and browser information + ...deviceInfo, + + // Error handling + has_error: hasError, + log_level: logLevel, + ...(log.error && { + error_message: log.error, + error_type: 'trpc_error', + }), + + // Full request/response data + request_payload: log.input, + ...(log.output && { + response_payload: log.output, + }), + + // Performance metrics + timing: { + start_time: log.metadata?.startTime || log.timestamp, + end_time: log.metadata?.endTime || (log.timestamp + log.duration), + duration_ms: log.duration, + performance_category: performanceCategory, + }, - const deviceInfo = parseUserAgent(log.metadata?.userAgent); + // Complete request trace with all spans (from log.trace) + trace: log.trace, + } + }; - return { - message: `${logLevel}: TRPC call: [${log.procedure}] (${log.duration}ms)`, - service: 'zero-mail-app', - ddsource: 'trpc-logging', - ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${hasError},performance:${performanceCategory},browser:${deviceInfo.browser},device:${deviceInfo.device_type}`, - hostname: 'cloudflare-worker', - // Trace correlation fields - dd: { - trace_id: traceId, - span_id: spanId, - }, + await this.apiInstance.submitLog({ body: [logEntry] }); - additionalProperties: { - // Core call data - call_id: log.id, - procedure: log.procedure, - duration: log.duration, - performance_category: performanceCategory, - trpc_method: log.metadata?.method || 'unknown', - - // Session context - session_id: sessionId, - user_id: userId, - call_sequence: index + 1, - total_calls_in_session: filteredLogs.length, - - // HTTP context - http_method: 'POST', - http_url: `/api/trpc/${log.procedure}`, - client_ip: log.metadata?.ip, - referer: log.metadata?.referer, - origin: log.metadata?.origin, - accept_language: log.metadata?.acceptLanguage, - accept_encoding: log.metadata?.acceptEncoding, - request_id: log.metadata?.requestId, - - // Device and browser information - ...deviceInfo, - - // Error handling - has_error: hasError, - log_level: logLevel, - ...(log.error && { - error_message: log.error, - error_type: 'trpc_error', - }), - - // Full request/response data - request_payload: log.input, - ...(log.output && { - response_payload: log.output, - }), - - // Performance metrics - timing: { - start_time: log.metadata?.startTime || log.timestamp, - end_time: log.metadata?.endTime || (log.timestamp + log.duration), - duration_ms: log.duration, - performance_category: performanceCategory, - }, - } - }; - }); - - console.log('📊 Sending to Datadog:', { - sessionId, - userId, - totalLogs: logs.length, - filteredLogs: filteredLogs.length, - excludedLoggingCalls: logs.length - filteredLogs.length, - }); - - await this.apiInstance.submitLog({ body: logEntries }); - - console.log('✅ Successfully exported to Datadog:', { - sessionId, - userId, - exportedLogs: filteredLogs.length, - }); } catch (error) { - console.error('❌ Failed to export session logs to Datadog:', error); + console.error('❌ Failed to log TRPC call to Datadog:', error); } } } \ No newline at end of file diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts index 2f351c4e61..f43edf0a4b 100644 --- a/apps/server/src/lib/logging-durable-object.ts +++ b/apps/server/src/lib/logging-durable-object.ts @@ -3,6 +3,18 @@ import { Queryable } from 'dormroom'; import type { ZeroEnv } from '../env'; import { DatadogService } from './datadog-service'; +export interface TraceSpan { + id: string; + name: string; + startTime: number; + endTime?: number; + duration?: number; + status: 'started' | 'completed' | 'error'; + metadata?: Record; + error?: string; + tags?: Record; +} + export interface TRPCCallLog { id: string; timestamp: number; @@ -26,13 +38,26 @@ export interface TRPCCallLog { timestamp?: string; startTime?: number; endTime?: number; + // Trace information + traceId?: string; + requestDuration?: number; + }; + // Complete trace spans for this request + trace?: { + traceId: string; + requestStartTime: number; + requestEndTime?: number; + requestDuration?: number; + spans: TraceSpan[]; + totalSpans: number; + completedSpans: number; + errorSpans: number; }; } export interface LoggingState { sessionId: string; userId: string; - calls: TRPCCallLog[]; startedAt: number; lastActivity: number; totalCalls: number; @@ -61,24 +86,19 @@ export class LoggingDurableObject extends DurableObject { timestamp: Date.now(), }; - // Get current state - const currentState = await this.getState(); - - // Check if session has expired - const timeSinceLastActivity = Date.now() - currentState.lastActivity; - if (timeSinceLastActivity > this.SESSION_TIMEOUT && currentState.calls.length > 0) { - // Export expired session to Datadog - await this.exportSessionToDatadog(currentState); - - // Start new session - await this.clearSession(); - const newState = await this.getState(); - newState.userId = currentState.userId; // Preserve user ID - await this.state.storage.put('state', newState); + // Immediately export to Datadog (no session storage) + try { + await this.datadogService.logSingleCall( + callData.sessionId, + callData.userId, + log + ); + } catch (error) { + console.error('❌ Failed to log TRPC call to Datadog:', error); } - // Add the new call - currentState.calls.push(log); + // Optional: Keep minimal stats for dashboard (no call storage) + const currentState = await this.getState(); currentState.lastActivity = log.timestamp; currentState.totalCalls++; currentState.totalDuration += log.duration; @@ -87,12 +107,7 @@ export class LoggingDurableObject extends DurableObject { currentState.totalErrors++; } - // Keep only last 1000 calls to prevent memory issues - if (currentState.calls.length > 1000) { - currentState.calls = currentState.calls.slice(-1000); - } - - // Store updated state + // Save updated stats only (no call arrays) await this.state.storage.put('state', currentState); } @@ -103,7 +118,6 @@ export class LoggingDurableObject extends DurableObject { const newState: LoggingState = { sessionId: crypto.randomUUID(), userId: '', - calls: [], startedAt: Date.now(), lastActivity: Date.now(), totalCalls: 0, @@ -125,9 +139,9 @@ export class LoggingDurableObject extends DurableObject { await this.state.storage.put('state', state); } - async getCallHistory(limit: number = 100): Promise { - const state = await this.getState(); - return state.calls.slice(-limit); + async getCallHistory(_limit: number = 100): Promise { + // No longer storing call history - all logs go directly to Datadog + return []; } async getSessionStats(): Promise<{ @@ -155,7 +169,6 @@ export class LoggingDurableObject extends DurableObject { const newState: LoggingState = { sessionId: crypto.randomUUID(), userId: '', - calls: [], startedAt: Date.now(), lastActivity: Date.now(), totalCalls: 0, @@ -165,30 +178,13 @@ export class LoggingDurableObject extends DurableObject { await this.state.storage.put('state', newState); } - async exportSessionToDatadog(state: LoggingState): Promise { - if (state.calls.length === 0) return; - - try { - await this.datadogService.exportSessionLogs( - state.sessionId, - state.userId, - state.calls - ); - } catch (error) { - console.error('Failed to export session to Datadog:', error); - } - } - async exportCurrentSessionToDatadog(): Promise { - const state = await this.getState(); - await this.exportSessionToDatadog(state); + // No longer needed - all logs go directly to Datadog in real-time + console.log('✅ All logs are already in Datadog (real-time logging)'); } async endSession(): Promise { - const state = await this.getState(); - if (state.calls.length > 0) { - await this.exportSessionToDatadog(state); - } + // Just clear stats - no export needed since logs go directly to Datadog await this.clearSession(); } } \ No newline at end of file diff --git a/apps/server/src/lib/trace-context.ts b/apps/server/src/lib/trace-context.ts new file mode 100644 index 0000000000..b5c4198ce4 --- /dev/null +++ b/apps/server/src/lib/trace-context.ts @@ -0,0 +1,146 @@ +export interface TraceSpan { + id: string; + name: string; + startTime: number; + endTime?: number; + duration?: number; + status: 'started' | 'completed' | 'error'; + metadata?: Record; + error?: string; + tags?: Record; +} + +export interface RequestTrace { + traceId: string; + startTime: number; + endTime?: number; + duration?: number; + spans: TraceSpan[]; + metadata: { + procedure?: string; + userId?: string; + sessionId?: string; + ip?: string; + userAgent?: string; + requestId?: string; + }; +} + +class TraceContextClass { + private traces = new Map(); + + createTrace(traceId: string, metadata: RequestTrace['metadata']): RequestTrace { + const trace: RequestTrace = { + traceId, + startTime: Date.now(), + spans: [], + metadata, + }; + this.traces.set(traceId, trace); + return trace; + } + + getTrace(traceId: string): RequestTrace | undefined { + return this.traces.get(traceId); + } + + addSpan(traceId: string, span: Omit): TraceSpan { + const trace = this.traces.get(traceId); + if (!trace) { + throw new Error(`Trace not found: ${traceId}`); + } + + const fullSpan: TraceSpan = { + id: crypto.randomUUID(), + startTime: Date.now(), + status: 'started', + ...span, + }; + + trace.spans.push(fullSpan); + return fullSpan; + } + + completeSpan(traceId: string, spanId: string, metadata?: Record, error?: string): void { + const trace = this.traces.get(traceId); + if (!trace) return; + + const span = trace.spans.find(s => s.id === spanId); + if (!span) return; + + span.endTime = Date.now(); + span.duration = span.endTime - span.startTime; + span.status = error ? 'error' : 'completed'; + if (error) span.error = error; + if (metadata) span.metadata = { ...span.metadata, ...metadata }; + } + + completeTrace(traceId: string): RequestTrace | undefined { + const trace = this.traces.get(traceId); + if (!trace) return; + + trace.endTime = Date.now(); + trace.duration = trace.endTime - trace.startTime; + + // Clean up completed trace after a delay to allow for any async operations + setTimeout(() => { + this.traces.delete(traceId); + }, 30000); // 30 seconds + + return trace; + } + + // Helper to create and immediately start a span + startSpan(traceId: string, name: string, metadata?: Record, tags?: Record): TraceSpan { + return this.addSpan(traceId, { + name, + metadata, + tags, + }); + } +} + +export const TraceContext = new TraceContextClass(); + +// Helper function to safely get trace from request context using context variables +export function getRequestTrace(c: any): RequestTrace | undefined { + // Try to get trace ID from context variables (set in main.ts) + const traceId = c?.var?.traceId || c?.get?.('traceId'); + + // Fallback to headers if context variables aren't available + if (!traceId) { + const headerTraceId = c.req?.header?.('X-Trace-ID') || + c.req?.header?.('x-trace-id') || + c.req?.headers?.get?.('X-Trace-ID') || + c.req?.headers?.get?.('x-trace-id'); + + if (!headerTraceId) { + return undefined; + } + + return TraceContext.getTrace(headerTraceId); + } + + return TraceContext.getTrace(traceId); +} + +// Helper function to get trace ID from context variables or headers +export function getTraceId(c: any): string | undefined { + return c?.var?.traceId || c?.get?.('traceId') || c.req?.header?.('X-Trace-ID') || c.req?.header?.('x-trace-id'); +} + +// Helper function to safely add span to current request +export function addRequestSpan(c: any, name: string, metadata?: Record, tags?: Record): TraceSpan | undefined { + const traceId = getTraceId(c); + if (!traceId) return undefined; + + return TraceContext.startSpan(traceId, name, metadata, tags); +} + +// Helper function to complete span in current request +export function completeRequestSpan(c: any, spanId: string, metadata?: Record, error?: string): void { + const traceId = getTraceId(c); + if (!traceId) return; + + TraceContext.completeSpan(traceId, spanId, metadata, error); +} \ No newline at end of file diff --git a/apps/server/src/lib/trpc-logging.ts b/apps/server/src/lib/trpc-logging.ts index 5f0611439f..01b67a401c 100644 --- a/apps/server/src/lib/trpc-logging.ts +++ b/apps/server/src/lib/trpc-logging.ts @@ -16,12 +16,14 @@ export const createLoggingMiddleware = () => { input: any; ctx: any; }) => { + const startTime = Date.now(); const c = getContext(); const sessionId = c.var.sessionUser?.id || 'anonymous'; const userId = c.var.sessionUser?.id; // Initialize session if this is the first call + if (userId) { try { await initializeLoggingSession(sessionId, userId); @@ -33,35 +35,132 @@ export const createLoggingMiddleware = () => { let output: any; let error: string | undefined; + // Start TRPC procedure execution span + const { addRequestSpan, completeRequestSpan } = await import('./trace-context'); + const procedureSpan = addRequestSpan(c, 'trpc_procedure_execution', { + procedure: opts.path, + type: opts.type, + hasInput: !!opts.input, + inputSize: opts.input ? JSON.stringify(opts.input).length : 0, + }, { + 'trpc.procedure': opts.path, + 'trpc.type': opts.type, + }); + try { // Execute the TRPC call output = await opts.next(); + // Complete procedure span + if (procedureSpan) { + completeRequestSpan(c, procedureSpan.id, { + success: true, + hasOutput: !!output, + outputSize: output ? JSON.stringify(output).length : 0, + }); + } + + // Sanitize output to remove non-serializable objects + const sanitizeOutput = (obj: any): any => { + if (obj === null || obj === undefined) return obj; + if (typeof obj !== 'object') return obj; + if (Array.isArray(obj)) return obj.map(sanitizeOutput); + + const sanitized: any = {}; + for (const [key, value] of Object.entries(obj)) { + // Skip known non-serializable fields + if (key === 'ctx' && value && typeof value === 'object') { + // Skip the ctx field entirely as it contains Fetcher/Context objects + continue; + } + + try { + // Test if the value can be serialized + structuredClone(value); + sanitized[key] = sanitizeOutput(value); + } catch (err) { + // If it can't be serialized, replace with a description + sanitized[key] = `[Non-serializable: ${value?.constructor?.name || typeof value}]`; + } + } + return sanitized; + }; + // Log successful call - const callData: Omit = { + const callData: TRPCCallLog = { + id: crypto.randomUUID(), + timestamp: startTime, userId: userId || 'anonymous', sessionId, procedure: opts.path, input: opts.input, - output: output, + output: sanitizeOutput(output), duration: Date.now() - startTime, metadata: { method: opts.type, userAgent: c.req.header('User-Agent'), ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + referer: c.req.header('Referer'), + origin: c.req.header('Origin'), + acceptLanguage: c.req.header('Accept-Language'), + acceptEncoding: c.req.header('Accept-Encoding'), + requestId: c.req.header('X-Request-Id') || crypto.randomUUID(), + timestamp: new Date().toISOString(), + startTime, + endTime: Date.now(), }, }; - // Log asynchronously to avoid blocking the response - logTRPCCall(sessionId, callData).catch((err) => { - console.error('Failed to log TRPC call:', err); - }); + // Log to Durable Object which will immediately export to Datadog + if (c.env && userId) { + const { getRequestTrace } = await import('./trace-context'); + + // Get the complete trace for this request + const trace = getRequestTrace(c); + + // Add trace to call data + if (trace) { + callData.trace = { + traceId: trace.traceId, + requestStartTime: trace.startTime, + requestEndTime: trace.endTime, + requestDuration: trace.duration, + spans: trace.spans, + totalSpans: trace.spans.length, + completedSpans: trace.spans.filter(s => s.status === 'completed').length, + errorSpans: trace.spans.filter(s => s.status === 'error').length, + }; + callData.metadata.traceId = trace.traceId; + callData.metadata.requestDuration = trace.duration; + } + + // Send to DO which will immediately log to Datadog + logTRPCCall(sessionId, callData).catch((err) => { + console.error('Failed to log TRPC call:', err); + }); + + // Complete the trace after logging + if (trace) { + const { TraceContext } = await import('./trace-context'); + TraceContext.completeTrace(trace.traceId); + } + } } catch (err) { error = err instanceof Error ? err.message : 'Unknown error'; + // Complete procedure span with error + if (procedureSpan) { + completeRequestSpan(c, procedureSpan.id, { + success: false, + errorType: err instanceof Error ? err.constructor.name : 'UnknownError', + }, error); + } + // Log failed call - const callData: Omit = { + const callData: TRPCCallLog = { + id: crypto.randomUUID(), + timestamp: startTime, userId: userId || 'anonymous', sessionId, procedure: opts.path, @@ -72,13 +171,51 @@ export const createLoggingMiddleware = () => { method: opts.type, userAgent: c.req.header('User-Agent'), ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + referer: c.req.header('Referer'), + origin: c.req.header('Origin'), + acceptLanguage: c.req.header('Accept-Language'), + acceptEncoding: c.req.header('Accept-Encoding'), + requestId: c.req.header('X-Request-Id') || crypto.randomUUID(), + timestamp: new Date().toISOString(), + startTime, + endTime: Date.now(), }, }; - // Log asynchronously to avoid blocking the response - logTRPCCall(sessionId, callData).catch((logErr) => { - console.error('Failed to log TRPC error:', logErr); - }); + // Log error to Durable Object which will immediately export to Datadog + if (c.env && userId) { + const { getRequestTrace } = await import('./trace-context'); + + // Get the complete trace for this request + const trace = getRequestTrace(c); + + // Add trace to call data + if (trace) { + callData.trace = { + traceId: trace.traceId, + requestStartTime: trace.startTime, + requestEndTime: trace.endTime, + requestDuration: trace.duration, + spans: trace.spans, + totalSpans: trace.spans.length, + completedSpans: trace.spans.filter(s => s.status === 'completed').length, + errorSpans: trace.spans.filter(s => s.status === 'error').length, + }; + callData.metadata.traceId = trace.traceId; + callData.metadata.requestDuration = trace.duration; + } + + // Send to DO which will immediately log to Datadog + logTRPCCall(sessionId, callData).catch((logErr) => { + console.error('Failed to log TRPC error:', logErr); + }); + + // Complete the trace after logging error + if (trace) { + const { TraceContext } = await import('./trace-context'); + TraceContext.completeTrace(trace.traceId); + } + } throw err; } diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index fd98a8e06d..af0f0cf361 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -568,29 +568,122 @@ class ZeroDB extends DurableObject { const api = new Hono() .use(contextStorage()) .use('*', async (c, next) => { + // Initialize request tracing using headers (no context pollution) + const traceId = c.req.header('X-Trace-ID') || crypto.randomUUID(); + const requestId = c.req.header('X-Request-Id') || crypto.randomUUID(); + + // Set trace ID in response headers for client correlation + c.header('X-Trace-ID', traceId); + c.header('X-Request-ID', requestId); + + // Store trace ID in context variables for TRPC access + c.set('traceId', traceId); + c.set('requestId', requestId); + + const { TraceContext } = await import('./lib/trace-context'); + + // Create trace for this request + const trace = TraceContext.createTrace(traceId, { + requestId, + ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + userAgent: c.req.header('User-Agent'), + }); + + // Start authentication span + const authSpan = TraceContext.startSpan(traceId, 'authentication', { + method: c.req.method, + url: c.req.url, + hasAuthHeader: !!c.req.header('Authorization'), + }, { + 'auth.method': c.req.header('Authorization') ? 'bearer_token' : 'session_cookie' + }); + const auth = createAuth(); c.set('auth', auth); const session = await auth.api.getSession({ headers: c.req.raw.headers }); c.set('sessionUser', session?.user); if (c.req.header('Authorization') && !session?.user) { + // Start token verification span + const tokenSpan = TraceContext.startSpan(traceId, 'token_verification', { + tokenPresent: true, + }, { + 'auth.token_type': 'jwt' + }); + const token = c.req.header('Authorization')?.split(' ')[1]; if (token) { - const localJwks = await auth.api.getJwks(); - const jwks = createLocalJWKSet(localJwks); + try { + const localJwks = await auth.api.getJwks(); + const jwks = createLocalJWKSet(localJwks); + + const { payload } = await jwtVerify(token, jwks); + const userId = payload.sub; - const { payload } = await jwtVerify(token, jwks); - const userId = payload.sub; + if (userId) { + const db = await getZeroDB(userId); + const user = await db.findUser(); + c.set('sessionUser', user); - if (userId) { - const db = await getZeroDB(userId); - c.set('sessionUser', await db.findUser()); + TraceContext.completeSpan(traceId, tokenSpan.id, { + success: true, + userId, + userEmail: user?.email, + }); + } else { + TraceContext.completeSpan(traceId, tokenSpan.id, { + success: false, + reason: 'no_user_id_in_token', + }); + } + } catch (error) { + TraceContext.completeSpan(traceId, tokenSpan.id, { + success: false, + reason: 'token_verification_failed', + }, error instanceof Error ? error.message : 'Unknown token error'); } + } else { + TraceContext.completeSpan(traceId, tokenSpan.id, { + success: false, + reason: 'no_token_provided', + }); } } - await next(); + // Complete auth span + TraceContext.completeSpan(traceId, authSpan.id, { + authenticated: !!c.var.sessionUser, + userId: c.var.sessionUser?.id, + userEmail: c.var.sessionUser?.email, + authMethod: session?.user ? 'session' : (c.req.header('Authorization') ? 'token' : 'none'), + }); + + // Update trace metadata with user info + trace.metadata.userId = c.var.sessionUser?.id; + trace.metadata.sessionId = c.var.sessionUser?.id || 'anonymous'; + + const autumn = new Autumn({ secretKey: env.AUTUMN_SECRET_KEY }); + c.set('autumn', autumn); + + // Start request processing span + const requestSpan = TraceContext.startSpan(traceId, 'request_processing', { + authenticated: !!c.var.sessionUser, + path: new URL(c.req.url).pathname, + }); + + try { + await next(); + // Don't complete the request span here - let TRPC middleware handle it + } catch (error) { + TraceContext.completeSpan(traceId, requestSpan.id, { + success: false, + + statusCode: c.res.status, + }, error instanceof Error ? error.message : 'Unknown request error'); + throw error; + } + // Note: Trace will be completed by TRPC middleware after logging c.set('sessionUser', undefined); c.set('auth', undefined as any); diff --git a/apps/server/src/trpc/trpc.ts b/apps/server/src/trpc/trpc.ts index f9aab9f821..1cd4e01e75 100644 --- a/apps/server/src/trpc/trpc.ts +++ b/apps/server/src/trpc/trpc.ts @@ -1,8 +1,9 @@ -import { getActiveConnection, getZeroDB, logTRPCCall, initializeLoggingSession } from '../lib/server-utils'; +import { getActiveConnection, getZeroDB } from '../lib/server-utils'; import { Ratelimit, type RatelimitConfig } from '@upstash/ratelimit'; import type { HonoContext, HonoVariables } from '../ctx'; import { getConnInfo } from 'hono/cloudflare-workers'; import { initTRPC, TRPCError } from '@trpc/server'; +import { createLoggingMiddleware } from '../lib/trpc-logging'; import { redis } from '../lib/services'; import type { Context } from 'hono'; @@ -15,75 +16,75 @@ type TrpcContext = { const t = initTRPC.context().create({ transformer: superjson }); // Logging middleware -const createLoggingMiddleware = () => { - return t.middleware(async ({ ctx, next, path, type, input }) => { - const startTime = Date.now(); - const sessionId = ctx.sessionUser?.id || 'anonymous'; - - try { - // Initialize session if needed - await initializeLoggingSession(sessionId, sessionId); - - const result = await next(); - const duration = Date.now() - startTime; - - // Log the call asynchronously (don't block the response) - logTRPCCall(sessionId, { - procedure: path, - input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size - output: result.ok ? JSON.stringify(result.data).slice(0, 1000) : undefined, // Limit output size - error: !result.ok ? result.error?.message : undefined, - duration, - metadata: { - userAgent: ctx.c.req.header('user-agent'), - ip: getConnInfo(ctx.c).remote.address, - method: type, - // Additional metadata - referer: ctx.c.req.header('referer'), - origin: ctx.c.req.header('origin'), - acceptLanguage: ctx.c.req.header('accept-language'), - acceptEncoding: ctx.c.req.header('accept-encoding'), - // Request context - requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), - timestamp: new Date().toISOString(), - // Performance context - startTime, - endTime: Date.now(), - }, - }).catch(err => console.error('Failed to log TRPC call:', err)); - - return result; - } catch (error) { - const duration = Date.now() - startTime; - - // Log the error asynchronously - logTRPCCall(sessionId, { - procedure: path, - input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size - error: error instanceof Error ? error.message : 'Unknown error', - duration, - metadata: { - userAgent: ctx.c.req.header('user-agent'), - ip: getConnInfo(ctx.c).remote.address, - method: type, - // Additional metadata - referer: ctx.c.req.header('referer'), - origin: ctx.c.req.header('origin'), - acceptLanguage: ctx.c.req.header('accept-language'), - acceptEncoding: ctx.c.req.header('accept-encoding'), - // Request context - requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), - timestamp: new Date().toISOString(), - // Performance context - startTime, - endTime: Date.now(), - }, - }).catch(err => console.error('Failed to log TRPC call:', err)); - - throw error; - } - }); -}; +// const createLoggingMiddleware = () => { +// return t.middleware(async ({ ctx, next, path, type, input }) => { +// const startTime = Date.now(); +// const sessionId = ctx.sessionUser?.id || 'anonymous'; + +// try { +// // Initialize session if needed +// await initializeLoggingSession(sessionId, sessionId); + +// const result = await next(); +// const duration = Date.now() - startTime; + +// // Log the call asynchronously (don't block the response) +// logTRPCCall(sessionId, { +// procedure: path, +// input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size +// output: result.ok ? JSON.stringify(result.data).slice(0, 1000) : undefined, // Limit output size +// error: !result.ok ? result.error?.message : undefined, +// duration, +// metadata: { +// userAgent: ctx.c.req.header('user-agent'), +// ip: getConnInfo(ctx.c).remote.address, +// method: type, +// // Additional metadata +// referer: ctx.c.req.header('referer'), +// origin: ctx.c.req.header('origin'), +// acceptLanguage: ctx.c.req.header('accept-language'), +// acceptEncoding: ctx.c.req.header('accept-encoding'), +// // Request context +// requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), +// timestamp: new Date().toISOString(), +// // Performance context +// startTime, +// endTime: Date.now(), +// }, +// }).catch(err => console.error('Failed to log TRPC call:', err)); + +// return result; +// } catch (error) { +// const duration = Date.now() - startTime; + +// // Log the error asynchronously +// logTRPCCall(sessionId, { +// procedure: path, +// input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size +// error: error instanceof Error ? error.message : 'Unknown error', +// duration, +// metadata: { +// userAgent: ctx.c.req.header('user-agent'), +// ip: getConnInfo(ctx.c).remote.address, +// method: type, +// // Additional metadata +// referer: ctx.c.req.header('referer'), +// origin: ctx.c.req.header('origin'), +// acceptLanguage: ctx.c.req.header('accept-language'), +// acceptEncoding: ctx.c.req.header('accept-encoding'), +// // Request context +// requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), +// timestamp: new Date().toISOString(), +// // Performance context +// startTime, +// endTime: Date.now(), +// }, +// }).catch(err => console.error('Failed to log TRPC call:', err)); + +// throw error; +// } +// }); +// }; const loggingMiddleware = createLoggingMiddleware(); @@ -91,20 +92,71 @@ export const router = t.router; export const publicProcedure = t.procedure.use(loggingMiddleware); export const privateProcedure = publicProcedure.use(async ({ ctx, next }) => { + const { addRequestSpan, completeRequestSpan } = await import('../lib/trace-context'); + + // Start auth validation span + const authSpan = addRequestSpan(ctx.c, 'trpc_auth_validation', { + hasSessionUser: !!ctx.sessionUser, + procedure: 'private', + }, { + 'trpc.auth_required': 'true' + }); + if (!ctx.sessionUser) { + if (authSpan) { + completeRequestSpan(ctx.c, authSpan.id, { + success: false, + reason: 'no_session_user', + }, 'UNAUTHORIZED: No session user found'); + } + throw new TRPCError({ code: 'UNAUTHORIZED', }); } + if (authSpan) { + completeRequestSpan(ctx.c, authSpan.id, { + success: true, + userId: ctx.sessionUser.id, + userEmail: ctx.sessionUser.email, + }); + } + return next({ ctx: { ...ctx, sessionUser: ctx.sessionUser } }); }); export const activeConnectionProcedure = privateProcedure.use(async ({ ctx, next }) => { + const { addRequestSpan, completeRequestSpan } = await import('../lib/trace-context'); + + // Start connection validation span + const connectionSpan = addRequestSpan(ctx.c, 'trpc_connection_validation', { + userId: ctx.sessionUser.id, + }, { + 'trpc.connection_required': 'true' + }); + try { const activeConnection = await getActiveConnection(); + + if (connectionSpan) { + completeRequestSpan(ctx.c, connectionSpan.id, { + success: true, + connectionId: activeConnection.id, + connectionType: activeConnection.providerId, + connectionEmail: activeConnection.email, + }); + } + return next({ ctx: { ...ctx, activeConnection } }); } catch (err) { + if (connectionSpan) { + completeRequestSpan(ctx.c, connectionSpan.id, { + success: false, + reason: 'connection_not_found', + }, err instanceof Error ? err.message : 'Failed to get active connection'); + } + await ctx.c.var.auth.api.signOut({ headers: ctx.c.req.raw.headers }); throw new TRPCError({ code: 'BAD_REQUEST', From 79dc812bf0300351778eda9794f2655e6cee1d46 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Mon, 18 Aug 2025 16:47:22 +0400 Subject: [PATCH 08/13] fixes lint --- apps/server/src/lib/trpc-logging.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/lib/trpc-logging.ts b/apps/server/src/lib/trpc-logging.ts index 01b67a401c..c44a114f45 100644 --- a/apps/server/src/lib/trpc-logging.ts +++ b/apps/server/src/lib/trpc-logging.ts @@ -80,6 +80,7 @@ export const createLoggingMiddleware = () => { sanitized[key] = sanitizeOutput(value); } catch (err) { // If it can't be serialized, replace with a description + console.log('🔍 [TRACE DEBUG ERROR] Non-serializable value:', err); sanitized[key] = `[Non-serializable: ${value?.constructor?.name || typeof value}]`; } } From c9689d3c281d8c950ea37c75915217feed7b290f Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Mon, 18 Aug 2025 17:13:06 +0400 Subject: [PATCH 09/13] fix autumn import --- apps/server/src/main.ts | 2 +- apps/server/src/trpc/index.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index af0f0cf361..adc7c29a86 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -24,6 +24,7 @@ import { SyncThreadsCoordinatorWorkflow } from './workflows/sync-threads-coordin import { WorkerEntrypoint, DurableObject, RpcTarget } from 'cloudflare:workers'; // import { instrument, type ResolveConfigFn } from '@microlabs/otel-cf-workers'; import { getZeroAgent, getZeroDB, verifyToken } from './lib/server-utils'; +import { Autumn } from 'autumn-js'; import { SyncThreadsWorkflow } from './workflows/sync-threads-workflow'; import { ShardRegistry, ZeroAgent, ZeroDriver } from './routes/agent'; import { ThreadSyncWorker } from './routes/agent/sync-worker'; @@ -31,7 +32,6 @@ import { oAuthDiscoveryMetadata } from 'better-auth/plugins'; import { EProviders, type IEmailSendBatch } from './types'; import { eq, and, desc, asc, inArray } from 'drizzle-orm'; import { ThinkingMCP } from './lib/sequential-thinking'; -import { ZeroAgent, ZeroDriver } from './routes/agent'; import { LoggingDurableObject } from './lib/logging-durable-object'; import { contextStorage } from 'hono/context-storage'; import { defaultUserSettings } from './lib/schemas'; diff --git a/apps/server/src/trpc/index.ts b/apps/server/src/trpc/index.ts index 3b473c6543..2df9cf237a 100644 --- a/apps/server/src/trpc/index.ts +++ b/apps/server/src/trpc/index.ts @@ -17,8 +17,6 @@ import { bimiRouter } from './routes/bimi'; import type { HonoContext } from '../ctx'; import { aiRouter } from './routes/ai'; import { router } from './trpc'; -import { categoriesRouter } from './routes/categories'; -import { templatesRouter } from './routes/templates'; import { loggingRouter } from './routes/logging'; export const appRouter = router({ From 16e8da627fcb9256ecb866194747231b97faa642 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Tue, 19 Aug 2025 23:05:09 +0300 Subject: [PATCH 10/13] address AI review --- apps/mail/components/logging-dashboard.tsx | 198 ------------------ apps/mail/lib/trpc.ts | 2 +- apps/server/src/lib/datadog-service.ts | 22 +- apps/server/src/lib/logging-durable-object.ts | 16 -- apps/server/src/lib/server-utils.ts | 24 +-- apps/server/src/lib/trace-context.ts | 2 + apps/server/src/lib/trpc-logging.ts | 4 +- apps/server/src/main.ts | 2 - apps/server/src/trpc/routes/logging.ts | 54 ++--- apps/server/src/trpc/trpc.ts | 73 ------- 10 files changed, 42 insertions(+), 355 deletions(-) delete mode 100644 apps/mail/components/logging-dashboard.tsx diff --git a/apps/mail/components/logging-dashboard.tsx b/apps/mail/components/logging-dashboard.tsx deleted file mode 100644 index efda45094e..0000000000 --- a/apps/mail/components/logging-dashboard.tsx +++ /dev/null @@ -1,198 +0,0 @@ -'use client'; - -import { api } from '@/lib/trpc'; -import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; -import { Button } from '@/components/ui/button'; -import { Badge } from '@/components/ui/badge'; -import { ScrollArea } from '@/components/ui/scroll-area'; -import { formatDistanceToNow } from 'date-fns'; -import { RefreshCw, Trash2, Activity } from 'lucide-react'; -import { useState } from 'react'; -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, - AlertDialogTrigger, -} from '@/components/ui/alert-dialog'; - -export function LoggingDashboard() { - const [limit, setLimit] = useState(50); - - const { data: stats, refetch: refetchStats } = api.logging.getSessionStats.useQuery(); - const { data: history, refetch: refetchHistory } = api.logging.getCallHistory.useQuery({ limit }); - const clearSessionMutation = api.logging.clearSession.useMutation({ - onSuccess: () => { - refetchStats(); - refetchHistory(); - }, - }); - - const handleClearSession = () => { - clearSessionMutation.mutate(); - }; - - if (!stats || !history) { - return
Loading...
; - } - - return ( -
-
-

Session Logging Dashboard

-
- - - - - - - - Clear Session Logs - - Are you sure you want to clear all session logs? This action cannot be undone. - - - - Cancel - - Clear Logs - - - - -
-
- - {/* Stats Cards */} -
- - - Total Calls - - - -
{stats.totalCalls}
- - - - - - Errors - 5 ? 'destructive' : 'secondary'}> - {stats.totalErrors} - - - -
{stats.errorRate.toFixed(1)}%
- - - - - - Avg Duration - - -
{stats.averageDuration.toFixed(0)}ms
- - - - - - Session Duration - - -
- {formatDistanceToNow(Date.now() - stats.sessionDuration, { addSuffix: true })} -
- - -
- - {/* Call History */} - - - Recent Calls - - -
- - - -
- - -
- {history.map((call) => ( -
-
-
- - {call.metadata.method} - - {call.procedure} -
-
- {call.duration}ms -
-
- -
- {new Date(call.timestamp).toLocaleString()} -
- - {call.error && ( -
- Error: {call.error} -
- )} - - {call.input && Object.keys(call.input).length > 0 && ( -
- Input - 
-                                                {JSON.stringify(call.input, null, 2)}
-
-
- )} -
- ))} -
- - - -
- ); -} \ No newline at end of file diff --git a/apps/mail/lib/trpc.ts b/apps/mail/lib/trpc.ts index aeb1b81394..abe366138a 100644 --- a/apps/mail/lib/trpc.ts +++ b/apps/mail/lib/trpc.ts @@ -2,7 +2,7 @@ import { createTRPCClient, httpBatchLink } from '@trpc/client'; import type { AppRouter } from '@zero/server/trpc'; import superjson from 'superjson'; -const getUrl = () => 'http://localhost:8787/api/trpc'; +const getUrl = () => import.meta.env.VITE_PUBLIC_BACKEND_URL + '/api/trpc'; export const api = createTRPCClient({ links: [ diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts index aa7944f1ad..153e06d46c 100644 --- a/apps/server/src/lib/datadog-service.ts +++ b/apps/server/src/lib/datadog-service.ts @@ -27,31 +27,17 @@ export class DatadogService { this.site = env?.DD_SITE || 'datadoghq.com'; } - // Simple hash function for generating consistent trace and span IDs - // According to Datadog docs: https://docs.datadoghq.com/tracing/connect_logs_and_traces - // Trace IDs must be 32-character lowercase hexadecimal strings - // Span IDs must be 16-character lowercase hexadecimal strings - private simpleHash(str: string): string { - let hash = 0; - for (let i = 0; i < str.length; i++) { - const char = str.charCodeAt(i); - hash = ((hash << 5) - hash) + char; - hash = hash & hash; // Convert to 32-bit integer - } - // Convert to positive hex string and pad to 32 characters - const positiveHash = Math.abs(hash).toString(16); - return positiveHash.padEnd(32, '0').slice(0, 32).toLowerCase(); + private generateId(): string { + return crypto.randomUUID().replace(/-/g, ''); } // Check if a procedure is logging-related to avoid recursive logging private isLoggingProcedure(procedure: string): boolean { const loggingProcedures = [ 'logging.getSessionStats', - 'logging.getCallHistory', 'logging.clearSession', 'logging.getSessionState', 'logging.exportToDatadog', - 'logging.endSession' ]; return loggingProcedures.includes(procedure); } @@ -63,8 +49,8 @@ export class DatadogService { } try { - const traceId = this.simpleHash(`${sessionId}-${userId}`).slice(0, 32); - const spanId = this.simpleHash(`${log.id}-${log.procedure}-${log.timestamp}`).slice(0, 16); + const traceId = this.generateId(); + const spanId = this.generateId(); const performanceCategory = log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow'; const hasError = !!log.error; diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts index f43edf0a4b..1aa5ea3213 100644 --- a/apps/server/src/lib/logging-durable-object.ts +++ b/apps/server/src/lib/logging-durable-object.ts @@ -70,7 +70,6 @@ export class LoggingDurableObject extends DurableObject { private state: DurableObjectState; protected env: ZeroEnv; private datadogService: DatadogService; - private readonly SESSION_TIMEOUT = 30 * 60 * 1000; // 30 minutes constructor(state: DurableObjectState, env: ZeroEnv) { super(state, env); @@ -139,11 +138,6 @@ export class LoggingDurableObject extends DurableObject { await this.state.storage.put('state', state); } - async getCallHistory(_limit: number = 100): Promise { - // No longer storing call history - all logs go directly to Datadog - return []; - } - async getSessionStats(): Promise<{ totalCalls: number; totalErrors: number; @@ -177,14 +171,4 @@ export class LoggingDurableObject extends DurableObject { }; await this.state.storage.put('state', newState); } - - async exportCurrentSessionToDatadog(): Promise { - // No longer needed - all logs go directly to Datadog in real-time - console.log('✅ All logs are already in Datadog (real-time logging)'); - } - - async endSession(): Promise { - // Just clear stats - no export needed since logs go directly to Datadog - await this.clearSession(); - } } \ No newline at end of file diff --git a/apps/server/src/lib/server-utils.ts b/apps/server/src/lib/server-utils.ts index 54106d962a..f6c6423fb8 100644 --- a/apps/server/src/lib/server-utils.ts +++ b/apps/server/src/lib/server-utils.ts @@ -286,15 +286,15 @@ export const getThread: ( connectionId: string, threadId: string, ) => { - const result = await Effect.runPromise(getThreadEffect(connectionId, threadId)); - if (!result.result) { - throw new Error(`Thread ${threadId} not found`); - } - if (!result.shardId) { - throw new Error(`Thread ${threadId} not found in any shard`); - } - return { result: result.result, shardId: result.shardId }; -}; + const result = await Effect.runPromise(getThreadEffect(connectionId, threadId)); + if (!result.result) { + throw new Error(`Thread ${threadId} not found`); + } + if (!result.shardId) { + throw new Error(`Thread ${threadId} not found in any shard`); + } + return { result: result.result, shardId: result.shardId }; + }; export const modifyThreadLabelsInDB = async ( connectionId: string, @@ -501,7 +501,7 @@ const getCounts = async (connectionId: string): Promise => { export const sendDoState = async (connectionId: string) => { try { const agent = await getZeroSocketAgent(connectionId); - + const cached = await agent.getCachedDoState(); if (cached) { console.log(`[sendDoState] Using cached data for connection ${connectionId}`); @@ -522,9 +522,9 @@ export const sendDoState = async (connectionId: string) => { getCounts(connectionId), ]); const shards = await listShards(registry); - + await agent.setCachedDoState(size, counts, shards.length); - + return agent.broadcastChatMessage({ type: OutgoingMessageType.Do_State, isSyncing: false, diff --git a/apps/server/src/lib/trace-context.ts b/apps/server/src/lib/trace-context.ts index b5c4198ce4..960bb2014f 100644 --- a/apps/server/src/lib/trace-context.ts +++ b/apps/server/src/lib/trace-context.ts @@ -30,6 +30,8 @@ class TraceContextClass { private traces = new Map(); createTrace(traceId: string, metadata: RequestTrace['metadata']): RequestTrace { + const existing = this.traces.get(traceId); + if (existing) return existing; const trace: RequestTrace = { traceId, startTime: Date.now(), diff --git a/apps/server/src/lib/trpc-logging.ts b/apps/server/src/lib/trpc-logging.ts index c44a114f45..26b852cde9 100644 --- a/apps/server/src/lib/trpc-logging.ts +++ b/apps/server/src/lib/trpc-logging.ts @@ -70,17 +70,15 @@ export const createLoggingMiddleware = () => { for (const [key, value] of Object.entries(obj)) { // Skip known non-serializable fields if (key === 'ctx' && value && typeof value === 'object') { - // Skip the ctx field entirely as it contains Fetcher/Context objects continue; } try { - // Test if the value can be serialized structuredClone(value); sanitized[key] = sanitizeOutput(value); } catch (err) { // If it can't be serialized, replace with a description - console.log('🔍 [TRACE DEBUG ERROR] Non-serializable value:', err); + console.log('[TRACE DEBUG] Non-serializable value:', err); sanitized[key] = `[Non-serializable: ${value?.constructor?.name || typeof value}]`; } } diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index adc7c29a86..911c850238 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -629,7 +629,6 @@ const api = new Hono() TraceContext.completeSpan(traceId, tokenSpan.id, { success: true, userId, - userEmail: user?.email, }); } else { TraceContext.completeSpan(traceId, tokenSpan.id, { @@ -655,7 +654,6 @@ const api = new Hono() TraceContext.completeSpan(traceId, authSpan.id, { authenticated: !!c.var.sessionUser, userId: c.var.sessionUser?.id, - userEmail: c.var.sessionUser?.email, authMethod: session?.user ? 'session' : (c.req.header('Authorization') ? 'token' : 'none'), }); diff --git a/apps/server/src/trpc/routes/logging.ts b/apps/server/src/trpc/routes/logging.ts index e5629e9935..60cc9e1646 100644 --- a/apps/server/src/trpc/routes/logging.ts +++ b/apps/server/src/trpc/routes/logging.ts @@ -1,30 +1,30 @@ import { privateProcedure, router } from '../trpc'; import { getLoggingDO } from '../../lib/server-utils'; -import { z } from 'zod'; +import { TRPCError } from '@trpc/server'; export const loggingRouter = router({ getSessionStats: privateProcedure .query(async ({ ctx }) => { - const sessionId = ctx.sessionUser?.id || 'anonymous'; + if (!ctx.sessionUser) { + throw new TRPCError({ + code: 'UNAUTHORIZED', + message: 'Authentication required', + }); + } + const sessionId = ctx.sessionUser.id; const loggingDO = await getLoggingDO(sessionId); return await loggingDO.getSessionStats(); }), - getCallHistory: privateProcedure - .input( - z.object({ - limit: z.number().min(1).max(1000).default(100), - }) - ) - .query(async ({ ctx, input }) => { - const sessionId = ctx.sessionUser?.id || 'anonymous'; - const loggingDO = await getLoggingDO(sessionId); - return await loggingDO.getCallHistory(input.limit); - }), - clearSession: privateProcedure .mutation(async ({ ctx }) => { - const sessionId = ctx.sessionUser?.id || 'anonymous'; + if (!ctx.sessionUser) { + throw new TRPCError({ + code: 'UNAUTHORIZED', + message: 'Authentication required', + }); + } + const sessionId = ctx.sessionUser.id; const loggingDO = await getLoggingDO(sessionId); await loggingDO.clearSession(); return { success: true }; @@ -32,24 +32,14 @@ export const loggingRouter = router({ getSessionState: privateProcedure .query(async ({ ctx }) => { - const sessionId = ctx.sessionUser?.id || 'anonymous'; + if (!ctx.sessionUser) { + throw new TRPCError({ + code: 'UNAUTHORIZED', + message: 'Authentication required', + }); + } + const sessionId = ctx.sessionUser.id; const loggingDO = await getLoggingDO(sessionId); return await loggingDO.getState(); }), - - exportToDatadog: privateProcedure - .mutation(async ({ ctx }) => { - const sessionId = ctx.sessionUser?.id || 'anonymous'; - const loggingDO = await getLoggingDO(sessionId); - await loggingDO.exportCurrentSessionToDatadog(); - return { success: true }; - }), - - endSession: privateProcedure - .mutation(async ({ ctx }) => { - const sessionId = ctx.sessionUser?.id || 'anonymous'; - const loggingDO = await getLoggingDO(sessionId); - await loggingDO.endSession(); - return { success: true }; - }), }); \ No newline at end of file diff --git a/apps/server/src/trpc/trpc.ts b/apps/server/src/trpc/trpc.ts index 1cd4e01e75..6edf75e1fb 100644 --- a/apps/server/src/trpc/trpc.ts +++ b/apps/server/src/trpc/trpc.ts @@ -15,77 +15,6 @@ type TrpcContext = { const t = initTRPC.context().create({ transformer: superjson }); -// Logging middleware -// const createLoggingMiddleware = () => { -// return t.middleware(async ({ ctx, next, path, type, input }) => { -// const startTime = Date.now(); -// const sessionId = ctx.sessionUser?.id || 'anonymous'; - -// try { -// // Initialize session if needed -// await initializeLoggingSession(sessionId, sessionId); - -// const result = await next(); -// const duration = Date.now() - startTime; - -// // Log the call asynchronously (don't block the response) -// logTRPCCall(sessionId, { -// procedure: path, -// input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size -// output: result.ok ? JSON.stringify(result.data).slice(0, 1000) : undefined, // Limit output size -// error: !result.ok ? result.error?.message : undefined, -// duration, -// metadata: { -// userAgent: ctx.c.req.header('user-agent'), -// ip: getConnInfo(ctx.c).remote.address, -// method: type, -// // Additional metadata -// referer: ctx.c.req.header('referer'), -// origin: ctx.c.req.header('origin'), -// acceptLanguage: ctx.c.req.header('accept-language'), -// acceptEncoding: ctx.c.req.header('accept-encoding'), -// // Request context -// requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), -// timestamp: new Date().toISOString(), -// // Performance context -// startTime, -// endTime: Date.now(), -// }, -// }).catch(err => console.error('Failed to log TRPC call:', err)); - -// return result; -// } catch (error) { -// const duration = Date.now() - startTime; - -// // Log the error asynchronously -// logTRPCCall(sessionId, { -// procedure: path, -// input: input ? JSON.stringify(input).slice(0, 1000) : undefined, // Limit input size -// error: error instanceof Error ? error.message : 'Unknown error', -// duration, -// metadata: { -// userAgent: ctx.c.req.header('user-agent'), -// ip: getConnInfo(ctx.c).remote.address, -// method: type, -// // Additional metadata -// referer: ctx.c.req.header('referer'), -// origin: ctx.c.req.header('origin'), -// acceptLanguage: ctx.c.req.header('accept-language'), -// acceptEncoding: ctx.c.req.header('accept-encoding'), -// // Request context -// requestId: ctx.c.req.header('x-request-id') || crypto.randomUUID(), -// timestamp: new Date().toISOString(), -// // Performance context -// startTime, -// endTime: Date.now(), -// }, -// }).catch(err => console.error('Failed to log TRPC call:', err)); - -// throw error; -// } -// }); -// }; - const loggingMiddleware = createLoggingMiddleware(); export const router = t.router; @@ -119,7 +48,6 @@ export const privateProcedure = publicProcedure.use(async ({ ctx, next }) => { completeRequestSpan(ctx.c, authSpan.id, { success: true, userId: ctx.sessionUser.id, - userEmail: ctx.sessionUser.email, }); } @@ -144,7 +72,6 @@ export const activeConnectionProcedure = privateProcedure.use(async ({ ctx, next success: true, connectionId: activeConnection.id, connectionType: activeConnection.providerId, - connectionEmail: activeConnection.email, }); } From 2cc510c72b92f4194ece79e016560e5af6b04700 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Tue, 19 Aug 2025 23:10:15 +0300 Subject: [PATCH 11/13] fix wrangler migrations --- apps/server/wrangler.jsonc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apps/server/wrangler.jsonc b/apps/server/wrangler.jsonc index a8bc512e67..04aeb827da 100644 --- a/apps/server/wrangler.jsonc +++ b/apps/server/wrangler.jsonc @@ -147,6 +147,9 @@ { "tag": "v9", "new_sqlite_classes": ["ShardRegistry"], + }, + { + "tag": "v10", "new_classes": ["LoggingDurableObject"], }, ], @@ -290,6 +293,10 @@ "name": "SHARD_REGISTRY", "class_name": "ShardRegistry", }, + { + "name": "LOGGING", + "class_name": "LoggingDurableObject", + }, ], }, "workflows": [ @@ -378,6 +385,10 @@ "tag": "v10", "new_sqlite_classes": ["ShardRegistry"], }, + { + "tag": "v11", + "new_classes": ["LoggingDurableObject"], + }, ], "observability": { "enabled": true, @@ -519,6 +530,10 @@ "name": "SHARD_REGISTRY", "class_name": "ShardRegistry", }, + { + "name": "LOGGING", + "class_name": "LoggingDurableObject", + }, ], }, "workflows": [ @@ -601,6 +616,10 @@ "tag": "v10", "new_sqlite_classes": ["ShardRegistry"], }, + { + "tag": "v11", + "new_classes": ["LoggingDurableObject"], + }, ], "vars": { "NODE_ENV": "production", From a546e163d0c045e98cfd1814d943ef9f1edd144b Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Tue, 19 Aug 2025 23:39:02 +0300 Subject: [PATCH 12/13] address more ai comments --- apps/server/src/lib/datadog-service.ts | 32 +++--- apps/server/src/lib/logging-durable-object.ts | 22 ++--- apps/server/src/lib/trace-context.ts | 98 ++++++++++++++++++- apps/server/src/lib/trpc-logging.ts | 24 ++++- apps/server/src/main.ts | 27 ++++- apps/server/src/trpc/routes/logging.ts | 2 +- apps/server/wrangler.jsonc | 6 ++ 7 files changed, 176 insertions(+), 35 deletions(-) diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts index 153e06d46c..47681429bd 100644 --- a/apps/server/src/lib/datadog-service.ts +++ b/apps/server/src/lib/datadog-service.ts @@ -9,22 +9,30 @@ export class DatadogService { private site: string; constructor(env?: ZeroEnv) { + // Runtime validation for required Datadog credentials + if (!env?.DD_API_KEY || env.DD_API_KEY.trim() === '') { + throw new Error('DD_API_KEY environment variable is required and cannot be empty for Datadog service'); + } + + if (!env?.DD_APP_KEY || env.DD_APP_KEY.trim() === '') { + throw new Error('DD_APP_KEY environment variable is required and cannot be empty for Datadog service'); + } + const configuration = client.createConfiguration({ authMethods: { - apiKeyAuth: env?.DD_API_KEY || '', - appKeyAuth: env?.DD_APP_KEY || '', + apiKeyAuth: env.DD_API_KEY, + appKeyAuth: env.DD_APP_KEY, }, }); - // Set the site for the configuration - if (env?.DD_SITE) { - configuration.setServerVariables({ site: env.DD_SITE }); - } + // Set the site for the configuration (defaults to datadoghq.com if not provided) + const ddSite = env?.DD_SITE || 'datadoghq.com'; + configuration.setServerVariables({ site: ddSite }); this.apiInstance = new v2.LogsApi(configuration); - this.apiKey = env?.DD_API_KEY || ''; - this.appKey = env?.DD_APP_KEY || ''; - this.site = env?.DD_SITE || 'datadoghq.com'; + this.apiKey = env.DD_API_KEY; + this.appKey = env.DD_APP_KEY; + this.site = ddSite; } private generateId(): string { @@ -54,7 +62,7 @@ export class DatadogService { const performanceCategory = log.duration < 100 ? 'fast' : log.duration < 500 ? 'normal' : 'slow'; const hasError = !!log.error; - const logLevel = hasError ? 'ERROR' : performanceCategory === 'slow' ? 'WARNING' : 'INFO'; + const logLevel = hasError ? 'error' : performanceCategory === 'slow' ? 'warn' : 'info'; // Parse user agent for device/browser info const parseUserAgent = (userAgent?: string) => { @@ -124,7 +132,8 @@ export class DatadogService { const deviceInfo = parseUserAgent(log.metadata?.userAgent); const logEntry = { - message: `${logLevel}: TRPC call: [${log.procedure}] (${log.duration}ms)`, + message: `${logLevel.toUpperCase()}: TRPC call: [${log.procedure}] (${log.duration}ms)`, + status: logLevel, service: 'zero-mail-app', ddsource: 'trpc-logging', ddtags: `session:${sessionId},user:${userId},procedure:${log.procedure},duration:${log.duration}ms,has_error:${hasError},performance:${performanceCategory},browser:${deviceInfo.browser},device:${deviceInfo.device_type}`, @@ -164,7 +173,6 @@ export class DatadogService { // Error handling has_error: hasError, - log_level: logLevel, ...(log.error && { error_message: log.error, error_type: 'trpc_error', diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts index 1aa5ea3213..db5e984141 100644 --- a/apps/server/src/lib/logging-durable-object.ts +++ b/apps/server/src/lib/logging-durable-object.ts @@ -67,15 +67,13 @@ export interface LoggingState { @Queryable() export class LoggingDurableObject extends DurableObject { - private state: DurableObjectState; - protected env: ZeroEnv; private datadogService: DatadogService; + private storage: DurableObjectStorage; - constructor(state: DurableObjectState, env: ZeroEnv) { - super(state, env); - this.state = state; - this.env = env; - this.datadogService = new DatadogService(env); + constructor(ctx: DurableObjectState, env: ZeroEnv) { + super(ctx, env); + this.storage = ctx.storage; + this.datadogService = new DatadogService(this.env); } async logCall(callData: Omit): Promise { @@ -107,11 +105,11 @@ export class LoggingDurableObject extends DurableObject { } // Save updated stats only (no call arrays) - await this.state.storage.put('state', currentState); + await this.storage.put('state', currentState); } async getState(): Promise { - const state = await this.state.storage.get('state'); + const state = await this.storage.get('state'); if (!state) { // Initialize new state const newState: LoggingState = { @@ -123,7 +121,7 @@ export class LoggingDurableObject extends DurableObject { totalErrors: 0, totalDuration: 0, }; - await this.state.storage.put('state', newState); + await this.storage.put('state', newState); return newState; } return state; @@ -135,7 +133,7 @@ export class LoggingDurableObject extends DurableObject { state.sessionId = crypto.randomUUID(); state.startedAt = Date.now(); state.lastActivity = Date.now(); - await this.state.storage.put('state', state); + await this.storage.put('state', state); } async getSessionStats(): Promise<{ @@ -169,6 +167,6 @@ export class LoggingDurableObject extends DurableObject { totalErrors: 0, totalDuration: 0, }; - await this.state.storage.put('state', newState); + await this.storage.put('state', newState); } } \ No newline at end of file diff --git a/apps/server/src/lib/trace-context.ts b/apps/server/src/lib/trace-context.ts index 960bb2014f..8b63ffc581 100644 --- a/apps/server/src/lib/trace-context.ts +++ b/apps/server/src/lib/trace-context.ts @@ -28,10 +28,84 @@ export interface RequestTrace { class TraceContextClass { private traces = new Map(); + private readonly MAX_TRACES = 10000; // Maximum number of traces to keep in memory + private readonly TRACE_TTL_MS = 5 * 60 * 1000; // 5 minutes TTL for uncompleted traces + private cleanupInterval: NodeJS.Timeout; + + constructor() { + // Start periodic cleanup every 2 minutes + this.cleanupInterval = setInterval(() => { + this.performCleanup(); + }, 2 * 60 * 1000); + } + + /** + * Performs cleanup of stale traces based on TTL and enforces max size + */ + private performCleanup(): void { + const now = Date.now(); + const tracesToDelete: string[] = []; + + // Find traces that have exceeded TTL + for (const [traceId, trace] of this.traces) { + const age = now - trace.startTime; + if (age > this.TRACE_TTL_MS) { + tracesToDelete.push(traceId); + } + } + + // Remove stale traces + for (const traceId of tracesToDelete) { + this.traces.delete(traceId); + } + + // If still over max size, remove oldest traces (LRU-style eviction) + if (this.traces.size > this.MAX_TRACES) { + const sortedTraces = Array.from(this.traces.entries()) + .sort(([, a], [, b]) => a.startTime - b.startTime); + + const excessCount = this.traces.size - this.MAX_TRACES; + for (let i = 0; i < excessCount; i++) { + this.traces.delete(sortedTraces[i][0]); + } + } + + // Log cleanup statistics in development + if (tracesToDelete.length > 0 || this.traces.size > this.MAX_TRACES * 0.8) { + console.debug(`Trace cleanup: removed ${tracesToDelete.length} stale traces, ${this.traces.size} traces remaining`); + } + } + + /** + * Get current trace statistics for monitoring + */ + getStats(): { totalTraces: number; oldestTraceAge: number } { + if (this.traces.size === 0) { + return { totalTraces: 0, oldestTraceAge: 0 }; + } + + const now = Date.now(); + let oldestAge = 0; + for (const trace of this.traces.values()) { + const age = now - trace.startTime; + oldestAge = Math.max(oldestAge, age); + } + + return { + totalTraces: this.traces.size, + oldestTraceAge: oldestAge, + }; + } createTrace(traceId: string, metadata: RequestTrace['metadata']): RequestTrace { const existing = this.traces.get(traceId); if (existing) return existing; + + // Trigger cleanup if we're approaching max capacity + if (this.traces.size >= this.MAX_TRACES * 0.9) { + this.performCleanup(); + } + const trace: RequestTrace = { traceId, startTime: Date.now(), @@ -74,7 +148,9 @@ class TraceContextClass { span.duration = span.endTime - span.startTime; span.status = error ? 'error' : 'completed'; if (error) span.error = error; - if (metadata) span.metadata = { ...span.metadata, ...metadata }; + if (metadata) { + span.metadata = span.metadata ? { ...span.metadata, ...metadata } : metadata; + } } completeTrace(traceId: string): RequestTrace | undefined { @@ -84,14 +160,20 @@ class TraceContextClass { trace.endTime = Date.now(); trace.duration = trace.endTime - trace.startTime; - // Clean up completed trace after a delay to allow for any async operations setTimeout(() => { this.traces.delete(traceId); - }, 30000); // 30 seconds + }, 10000); return trace; } + destroy(): void { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + } + this.traces.clear(); + } + // Helper to create and immediately start a span startSpan(traceId: string, name: string, metadata?: Record, tags?: Record): TraceSpan { return this.addSpan(traceId, { @@ -145,4 +227,14 @@ export function completeRequestSpan(c: any, spanId: string, metadata?: Record { metadata: { method: opts.type, userAgent: c.req.header('User-Agent'), - ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + ip: hashIpAddress(c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For')), referer: c.req.header('Referer'), origin: c.req.header('Origin'), acceptLanguage: c.req.header('Accept-Language'), @@ -169,7 +189,7 @@ export const createLoggingMiddleware = () => { metadata: { method: opts.type, userAgent: c.req.header('User-Agent'), - ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + ip: hashIpAddress(c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For')), referer: c.req.header('Referer'), origin: c.req.header('Origin'), acceptLanguage: c.req.header('Accept-Language'), diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 911c850238..be5b0f7a1a 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -24,7 +24,6 @@ import { SyncThreadsCoordinatorWorkflow } from './workflows/sync-threads-coordin import { WorkerEntrypoint, DurableObject, RpcTarget } from 'cloudflare:workers'; // import { instrument, type ResolveConfigFn } from '@microlabs/otel-cf-workers'; import { getZeroAgent, getZeroDB, verifyToken } from './lib/server-utils'; -import { Autumn } from 'autumn-js'; import { SyncThreadsWorkflow } from './workflows/sync-threads-workflow'; import { ShardRegistry, ZeroAgent, ZeroDriver } from './routes/agent'; import { ThreadSyncWorker } from './routes/agent/sync-worker'; @@ -565,6 +564,26 @@ class ZeroDB extends DurableObject { } } +// Utility function to hash IP addresses for PII protection +function hashIpAddress(ip: string | undefined): string | undefined { + if (!ip) return undefined; + + // Simple but effective hash for IP addresses + // This preserves uniqueness while protecting PII + const salt = 'zero-mail-ip-salt-2024'; // Consider using env variable for production + let hash = 0; + const str = ip + salt; + + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) - hash) + char; + hash = hash & hash; // Convert to 32bit integer + } + + // Return a prefixed hex representation + return `ip_${Math.abs(hash).toString(16).padStart(8, '0')}`; +} + const api = new Hono() .use(contextStorage()) .use('*', async (c, next) => { @@ -583,9 +602,10 @@ const api = new Hono() const { TraceContext } = await import('./lib/trace-context'); // Create trace for this request + const rawIp = c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'); const trace = TraceContext.createTrace(traceId, { requestId, - ip: c.req.header('CF-Connecting-IP') || c.req.header('X-Forwarded-For'), + ip: hashIpAddress(rawIp), // Hash IP address to protect PII userAgent: c.req.header('User-Agent'), }); @@ -661,9 +681,6 @@ const api = new Hono() trace.metadata.userId = c.var.sessionUser?.id; trace.metadata.sessionId = c.var.sessionUser?.id || 'anonymous'; - const autumn = new Autumn({ secretKey: env.AUTUMN_SECRET_KEY }); - c.set('autumn', autumn); - // Start request processing span const requestSpan = TraceContext.startSpan(traceId, 'request_processing', { authenticated: !!c.var.sessionUser, diff --git a/apps/server/src/trpc/routes/logging.ts b/apps/server/src/trpc/routes/logging.ts index 60cc9e1646..d7328d336c 100644 --- a/apps/server/src/trpc/routes/logging.ts +++ b/apps/server/src/trpc/routes/logging.ts @@ -42,4 +42,4 @@ export const loggingRouter = router({ const loggingDO = await getLoggingDO(sessionId); return await loggingDO.getState(); }), -}); \ No newline at end of file +}); diff --git a/apps/server/wrangler.jsonc b/apps/server/wrangler.jsonc index 04aeb827da..088917e99e 100644 --- a/apps/server/wrangler.jsonc +++ b/apps/server/wrangler.jsonc @@ -412,6 +412,9 @@ "DISABLE_WORKFLOWS": "false", "OTEL_EXPORTER_OTLP_ENDPOINT": "https://api.axiom.co/v1/traces", "OTEL_SERVICE_NAME": "zero-email-server-staging", + "DD_API_KEY": "", + "DD_APP_KEY": "", + "DD_SITE": "datadoghq.com", }, "kv_namespaces": [ { @@ -633,6 +636,9 @@ "DISABLE_WORKFLOWS": "true", "OTEL_EXPORTER_OTLP_ENDPOINT": "https://api.axiom.co/v1/traces", "OTEL_SERVICE_NAME": "zero-email-server-production", + "DD_API_KEY": "", + "DD_APP_KEY": "", + "DD_SITE": "datadoghq.com", }, "kv_namespaces": [ { From 8d592e2dc74ab303696483c8e8ed94ca37cf1e60 Mon Sep 17 00:00:00 2001 From: Adam Abu Ghaida <44762129+AdamGhaida@users.noreply.github.com> Date: Wed, 20 Aug 2025 18:58:45 +0300 Subject: [PATCH 13/13] Migrates away from Durable Objects for logging --- apps/server/src/env.ts | 4 +- apps/server/src/lib/datadog-service.ts | 2 +- apps/server/src/lib/logging-durable-object.ts | 172 ------------------ apps/server/src/lib/logging-service.ts | 117 ++++++++++++ apps/server/src/lib/server-utils.ts | 14 -- apps/server/src/lib/trpc-logging.ts | 31 ++-- apps/server/src/main.ts | 3 +- apps/server/src/trpc/routes/logging.ts | 14 +- apps/server/src/types/logging.ts | 70 +++++++ apps/server/wrangler.jsonc | 30 +-- 10 files changed, 220 insertions(+), 237 deletions(-) delete mode 100644 apps/server/src/lib/logging-durable-object.ts create mode 100644 apps/server/src/lib/logging-service.ts create mode 100644 apps/server/src/types/logging.ts diff --git a/apps/server/src/env.ts b/apps/server/src/env.ts index 25cbc5c56d..4a7b37125e 100644 --- a/apps/server/src/env.ts +++ b/apps/server/src/env.ts @@ -1,6 +1,6 @@ import type { ThinkingMCP, ThreadSyncWorker, WorkflowRunner, ZeroDB, ZeroMCP } from './main'; import type { ShardRegistry, ZeroAgent, ZeroDriver } from './routes/agent'; -import type { LoggingDurableObject } from './lib/logging-durable-object'; + import { env as _env } from 'cloudflare:workers'; import type { QueryableHandler } from 'dormroom'; @@ -12,7 +12,7 @@ export type ZeroEnv = { ZERO_MCP: DurableObjectNamespace; THINKING_MCP: DurableObjectNamespace; WORKFLOW_RUNNER: DurableObjectNamespace; - LOGGING: DurableObjectNamespace; + THREAD_SYNC_WORKER: DurableObjectNamespace; SYNC_THREADS_WORKFLOW: Workflow; SYNC_THREADS_COORDINATOR_WORKFLOW: Workflow; diff --git a/apps/server/src/lib/datadog-service.ts b/apps/server/src/lib/datadog-service.ts index 47681429bd..c7737eeec1 100644 --- a/apps/server/src/lib/datadog-service.ts +++ b/apps/server/src/lib/datadog-service.ts @@ -1,5 +1,5 @@ import { client, v2 } from '@datadog/datadog-api-client'; -import type { TRPCCallLog } from './logging-durable-object'; +import type { TRPCCallLog } from '../types/logging'; import type { ZeroEnv } from '../env'; export class DatadogService { diff --git a/apps/server/src/lib/logging-durable-object.ts b/apps/server/src/lib/logging-durable-object.ts deleted file mode 100644 index db5e984141..0000000000 --- a/apps/server/src/lib/logging-durable-object.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { DurableObject } from 'cloudflare:workers'; -import { Queryable } from 'dormroom'; -import type { ZeroEnv } from '../env'; -import { DatadogService } from './datadog-service'; - -export interface TraceSpan { - id: string; - name: string; - startTime: number; - endTime?: number; - duration?: number; - status: 'started' | 'completed' | 'error'; - metadata?: Record; - error?: string; - tags?: Record; -} - -export interface TRPCCallLog { - id: string; - timestamp: number; - userId: string; - sessionId: string; - procedure: string; - input: any; - output?: any; - error?: string; - duration: number; - metadata: { - userAgent?: string; - ip?: string; - method: 'query' | 'mutation' | 'subscription'; - // Additional metadata - referer?: string; - origin?: string; - acceptLanguage?: string; - acceptEncoding?: string; - requestId?: string; - timestamp?: string; - startTime?: number; - endTime?: number; - // Trace information - traceId?: string; - requestDuration?: number; - }; - // Complete trace spans for this request - trace?: { - traceId: string; - requestStartTime: number; - requestEndTime?: number; - requestDuration?: number; - spans: TraceSpan[]; - totalSpans: number; - completedSpans: number; - errorSpans: number; - }; -} - -export interface LoggingState { - sessionId: string; - userId: string; - startedAt: number; - lastActivity: number; - totalCalls: number; - totalErrors: number; - totalDuration: number; -} - -@Queryable() -export class LoggingDurableObject extends DurableObject { - private datadogService: DatadogService; - private storage: DurableObjectStorage; - - constructor(ctx: DurableObjectState, env: ZeroEnv) { - super(ctx, env); - this.storage = ctx.storage; - this.datadogService = new DatadogService(this.env); - } - - async logCall(callData: Omit): Promise { - const log: TRPCCallLog = { - ...callData, - id: crypto.randomUUID(), - timestamp: Date.now(), - }; - - // Immediately export to Datadog (no session storage) - try { - await this.datadogService.logSingleCall( - callData.sessionId, - callData.userId, - log - ); - } catch (error) { - console.error('❌ Failed to log TRPC call to Datadog:', error); - } - - // Optional: Keep minimal stats for dashboard (no call storage) - const currentState = await this.getState(); - currentState.lastActivity = log.timestamp; - currentState.totalCalls++; - currentState.totalDuration += log.duration; - - if (log.error) { - currentState.totalErrors++; - } - - // Save updated stats only (no call arrays) - await this.storage.put('state', currentState); - } - - async getState(): Promise { - const state = await this.storage.get('state'); - if (!state) { - // Initialize new state - const newState: LoggingState = { - sessionId: crypto.randomUUID(), - userId: '', - startedAt: Date.now(), - lastActivity: Date.now(), - totalCalls: 0, - totalErrors: 0, - totalDuration: 0, - }; - await this.storage.put('state', newState); - return newState; - } - return state; - } - - async initializeSession(userId: string): Promise { - const state = await this.getState(); - state.userId = userId; - state.sessionId = crypto.randomUUID(); - state.startedAt = Date.now(); - state.lastActivity = Date.now(); - await this.storage.put('state', state); - } - - async getSessionStats(): Promise<{ - totalCalls: number; - totalErrors: number; - totalDuration: number; - averageDuration: number; - errorRate: number; - sessionDuration: number; - }> { - const state = await this.getState(); - const sessionDuration = Date.now() - state.startedAt; - - return { - totalCalls: state.totalCalls, - totalErrors: state.totalErrors, - totalDuration: state.totalDuration, - averageDuration: state.totalCalls > 0 ? state.totalDuration / state.totalCalls : 0, - errorRate: state.totalCalls > 0 ? (state.totalErrors / state.totalCalls) * 100 : 0, - sessionDuration, - }; - } - - async clearSession(): Promise { - const newState: LoggingState = { - sessionId: crypto.randomUUID(), - userId: '', - startedAt: Date.now(), - lastActivity: Date.now(), - totalCalls: 0, - totalErrors: 0, - totalDuration: 0, - }; - await this.storage.put('state', newState); - } -} \ No newline at end of file diff --git a/apps/server/src/lib/logging-service.ts b/apps/server/src/lib/logging-service.ts new file mode 100644 index 0000000000..b18187be07 --- /dev/null +++ b/apps/server/src/lib/logging-service.ts @@ -0,0 +1,117 @@ +import type { TRPCCallLog, LoggingState, SessionStats } from '../types/logging'; +import { DatadogService } from './datadog-service'; +import type { ZeroEnv } from '../env'; + +// In-memory session storage for stats +// In a production environment, you might want to use a distributed cache like Redis +const sessionStats = new Map(); + +export class LoggingService { + private datadogService: DatadogService; + + constructor(env: ZeroEnv) { + this.datadogService = new DatadogService(env); + } + + async logCall(callData: Omit): Promise { + const log: TRPCCallLog = { + ...callData, + id: crypto.randomUUID(), + timestamp: Date.now(), + }; + + // Immediately export to Datadog + try { + await this.datadogService.logSingleCall( + callData.sessionId, + callData.userId, + log + ); + } catch (error) { + console.error('❌ Failed to log TRPC call to Datadog:', error); + } + + // Update in-memory session stats + this.updateSessionStats(callData.sessionId, callData.userId, log); + } + + private updateSessionStats(sessionId: string, userId: string, log: TRPCCallLog): void { + let currentState = sessionStats.get(sessionId); + + if (!currentState) { + currentState = { + sessionId, + userId, + startedAt: Date.now(), + lastActivity: Date.now(), + totalCalls: 0, + totalErrors: 0, + totalDuration: 0, + }; + } + + currentState.lastActivity = log.timestamp; + currentState.totalCalls++; + currentState.totalDuration += log.duration; + + if (log.error) { + currentState.totalErrors++; + } + + sessionStats.set(sessionId, currentState); + } + + getState(sessionId: string): LoggingState { + let state = sessionStats.get(sessionId); + if (!state) { + // Initialize new state + state = { + sessionId, + userId: '', + startedAt: Date.now(), + lastActivity: Date.now(), + totalCalls: 0, + totalErrors: 0, + totalDuration: 0, + }; + sessionStats.set(sessionId, state); + } + return state; + } + + initializeSession(sessionId: string, userId: string): void { + const state = this.getState(sessionId); + state.userId = userId; + state.sessionId = sessionId; + state.startedAt = Date.now(); + state.lastActivity = Date.now(); + sessionStats.set(sessionId, state); + } + + getSessionStats(sessionId: string): SessionStats { + const state = this.getState(sessionId); + const sessionDuration = Date.now() - state.startedAt; + + return { + totalCalls: state.totalCalls, + totalErrors: state.totalErrors, + totalDuration: state.totalDuration, + averageDuration: state.totalCalls > 0 ? state.totalDuration / state.totalCalls : 0, + errorRate: state.totalCalls > 0 ? (state.totalErrors / state.totalCalls) * 100 : 0, + sessionDuration, + }; + } + + clearSession(sessionId: string): void { + const newState: LoggingState = { + sessionId, + userId: '', + startedAt: Date.now(), + lastActivity: Date.now(), + totalCalls: 0, + totalErrors: 0, + totalDuration: 0, + }; + sessionStats.set(sessionId, newState); + } +} diff --git a/apps/server/src/lib/server-utils.ts b/apps/server/src/lib/server-utils.ts index f6c6423fb8..c88991263a 100644 --- a/apps/server/src/lib/server-utils.ts +++ b/apps/server/src/lib/server-utils.ts @@ -604,21 +604,7 @@ export const verifyToken = async (token: string) => { return !!data; }; -// Logging utility functions -export const getLoggingDO = async (sessionId: string) => { - const stub = env.LOGGING.get(env.LOGGING.idFromName(sessionId)); - return stub; -}; -export const logTRPCCall = async (sessionId: string, callData: any) => { - const loggingDO = await getLoggingDO(sessionId); - await loggingDO.logCall(callData); -}; - -export const initializeLoggingSession = async (sessionId: string, userId: string) => { - const loggingDO = await getLoggingDO(sessionId); - await loggingDO.initializeSession(userId); -}; export const resetConnection = async (connectionId: string) => { const { db, conn } = createDb(env.HYPERDRIVE.connectionString); diff --git a/apps/server/src/lib/trpc-logging.ts b/apps/server/src/lib/trpc-logging.ts index 52afdc11ed..3dfd7db28d 100644 --- a/apps/server/src/lib/trpc-logging.ts +++ b/apps/server/src/lib/trpc-logging.ts @@ -1,5 +1,5 @@ -import type { TRPCCallLog } from './logging-durable-object'; -import { logTRPCCall, initializeLoggingSession } from './server-utils'; +import type { TRPCCallLog } from '../types/logging'; +import { LoggingService } from './logging-service'; import { getContext } from 'hono/context-storage'; import type { HonoContext } from '../ctx'; @@ -42,13 +42,14 @@ export const createLoggingMiddleware = () => { const sessionId = c.var.sessionUser?.id || 'anonymous'; const userId = c.var.sessionUser?.id; - // Initialize session if this is the first call - - if (userId) { + // Initialize logging service + let loggingService: LoggingService | undefined; + if (userId && c.env) { try { - await initializeLoggingSession(sessionId, userId); + loggingService = new LoggingService(c.env); + loggingService.initializeSession(sessionId, userId); } catch (error) { - console.error('Failed to initialize logging session:', error); + console.error('Failed to initialize logging service:', error); } } @@ -130,8 +131,8 @@ export const createLoggingMiddleware = () => { }, }; - // Log to Durable Object which will immediately export to Datadog - if (c.env && userId) { + // Log using the new logging service + if (loggingService) { const { getRequestTrace } = await import('./trace-context'); // Get the complete trace for this request @@ -153,8 +154,8 @@ export const createLoggingMiddleware = () => { callData.metadata.requestDuration = trace.duration; } - // Send to DO which will immediately log to Datadog - logTRPCCall(sessionId, callData).catch((err) => { + // Log using the new service which will immediately log to Datadog + loggingService.logCall(callData).catch((err) => { console.error('Failed to log TRPC call:', err); }); @@ -201,8 +202,8 @@ export const createLoggingMiddleware = () => { }, }; - // Log error to Durable Object which will immediately export to Datadog - if (c.env && userId) { + // Log error using the new logging service + if (loggingService) { const { getRequestTrace } = await import('./trace-context'); // Get the complete trace for this request @@ -224,8 +225,8 @@ export const createLoggingMiddleware = () => { callData.metadata.requestDuration = trace.duration; } - // Send to DO which will immediately log to Datadog - logTRPCCall(sessionId, callData).catch((logErr) => { + // Log using the new service which will immediately log to Datadog + loggingService.logCall(callData).catch((logErr) => { console.error('Failed to log TRPC error:', logErr); }); diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index be5b0f7a1a..54d7d2f0d5 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -31,7 +31,7 @@ import { oAuthDiscoveryMetadata } from 'better-auth/plugins'; import { EProviders, type IEmailSendBatch } from './types'; import { eq, and, desc, asc, inArray } from 'drizzle-orm'; import { ThinkingMCP } from './lib/sequential-thinking'; -import { LoggingDurableObject } from './lib/logging-durable-object'; + import { contextStorage } from 'hono/context-storage'; import { defaultUserSettings } from './lib/schemas'; import { createLocalJWKSet, jwtVerify } from 'jose'; @@ -1258,5 +1258,4 @@ export { SyncThreadsWorkflow, SyncThreadsCoordinatorWorkflow, ShardRegistry, - LoggingDurableObject, }; diff --git a/apps/server/src/trpc/routes/logging.ts b/apps/server/src/trpc/routes/logging.ts index d7328d336c..417d3a970d 100644 --- a/apps/server/src/trpc/routes/logging.ts +++ b/apps/server/src/trpc/routes/logging.ts @@ -1,5 +1,5 @@ import { privateProcedure, router } from '../trpc'; -import { getLoggingDO } from '../../lib/server-utils'; +import { LoggingService } from '../../lib/logging-service'; import { TRPCError } from '@trpc/server'; export const loggingRouter = router({ @@ -12,8 +12,8 @@ export const loggingRouter = router({ }); } const sessionId = ctx.sessionUser.id; - const loggingDO = await getLoggingDO(sessionId); - return await loggingDO.getSessionStats(); + const loggingService = new LoggingService(ctx.c.env); + return loggingService.getSessionStats(sessionId); }), clearSession: privateProcedure @@ -25,8 +25,8 @@ export const loggingRouter = router({ }); } const sessionId = ctx.sessionUser.id; - const loggingDO = await getLoggingDO(sessionId); - await loggingDO.clearSession(); + const loggingService = new LoggingService(ctx.c.env); + loggingService.clearSession(sessionId); return { success: true }; }), @@ -39,7 +39,7 @@ export const loggingRouter = router({ }); } const sessionId = ctx.sessionUser.id; - const loggingDO = await getLoggingDO(sessionId); - return await loggingDO.getState(); + const loggingService = new LoggingService(ctx.c.env); + return loggingService.getState(sessionId); }), }); diff --git a/apps/server/src/types/logging.ts b/apps/server/src/types/logging.ts new file mode 100644 index 0000000000..d177ced454 --- /dev/null +++ b/apps/server/src/types/logging.ts @@ -0,0 +1,70 @@ +export interface TraceSpan { + id: string; + name: string; + startTime: number; + endTime?: number; + duration?: number; + status: 'started' | 'completed' | 'error'; + metadata?: Record; + error?: string; + tags?: Record; +} + +export interface TRPCCallLog { + id: string; + timestamp: number; + userId: string; + sessionId: string; + procedure: string; + input: any; + output?: any; + error?: string; + duration: number; + metadata: { + userAgent?: string; + ip?: string; + method: 'query' | 'mutation' | 'subscription'; + // Additional metadata + referer?: string; + origin?: string; + acceptLanguage?: string; + acceptEncoding?: string; + requestId?: string; + timestamp?: string; + startTime?: number; + endTime?: number; + // Trace information + traceId?: string; + requestDuration?: number; + }; + // Complete trace spans for this request + trace?: { + traceId: string; + requestStartTime: number; + requestEndTime?: number; + requestDuration?: number; + spans: TraceSpan[]; + totalSpans: number; + completedSpans: number; + errorSpans: number; + }; +} + +export interface LoggingState { + sessionId: string; + userId: string; + startedAt: number; + lastActivity: number; + totalCalls: number; + totalErrors: number; + totalDuration: number; +} + +export interface SessionStats { + totalCalls: number; + totalErrors: number; + totalDuration: number; + averageDuration: number; + errorRate: number; + sessionDuration: number; +} diff --git a/apps/server/wrangler.jsonc b/apps/server/wrangler.jsonc index 088917e99e..f26210292a 100644 --- a/apps/server/wrangler.jsonc +++ b/apps/server/wrangler.jsonc @@ -58,10 +58,7 @@ "name": "WORKFLOW_RUNNER", "class_name": "WorkflowRunner", }, - { - "name": "LOGGING", - "class_name": "LoggingDurableObject", - }, + { "name": "THREAD_SYNC_WORKER", "class_name": "ThreadSyncWorker", @@ -148,10 +145,7 @@ "tag": "v9", "new_sqlite_classes": ["ShardRegistry"], }, - { - "tag": "v10", - "new_classes": ["LoggingDurableObject"], - }, + ], "observability": { @@ -293,10 +287,7 @@ "name": "SHARD_REGISTRY", "class_name": "ShardRegistry", }, - { - "name": "LOGGING", - "class_name": "LoggingDurableObject", - }, + ], }, "workflows": [ @@ -385,10 +376,7 @@ "tag": "v10", "new_sqlite_classes": ["ShardRegistry"], }, - { - "tag": "v11", - "new_classes": ["LoggingDurableObject"], - }, + ], "observability": { "enabled": true, @@ -533,10 +521,7 @@ "name": "SHARD_REGISTRY", "class_name": "ShardRegistry", }, - { - "name": "LOGGING", - "class_name": "LoggingDurableObject", - }, + ], }, "workflows": [ @@ -619,10 +604,7 @@ "tag": "v10", "new_sqlite_classes": ["ShardRegistry"], }, - { - "tag": "v11", - "new_classes": ["LoggingDurableObject"], - }, + ], "vars": { "NODE_ENV": "production",