Skip to content

Streamline email assistant prompt and add server-side tool execution #1887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MrgSub merged 1 commit into from
Aug 1, 2025
Merged

Streamline email assistant prompt and add server-side tool execution #1887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
587 changes: 153 additions & 434 deletions apps/mail/lib/prompts.ts
View file
Open in desktop

Large diffs are not rendered by default.

25 changes: 25 additions & 0 deletions apps/mail/lib/server-tool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
export async function callServerTool(action: string, payload: unknown, caller: string) {
const base = import.meta.env.VITE_PUBLIC_SERVER_URL;
const voiceSecret = import.meta.env.VITE_PUBLIC_VOICE_SECRET;

const res = await fetch(`${base}/api/ai/do/${action}`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Voice-Secret': voiceSecret,
'X-Caller': caller,
},
body: JSON.stringify(payload ?? {}),
});

// network / non-200 safety
if (!res.ok) {
const txt = await res.text();
throw new Error(`Server error (${res.status}): ${txt}`);
}

const data = await res.json<{ success: boolean; result?: unknown; error?: string }>(); // { success, result?, error? }
if (!data.success) throw new Error(data.error ?? 'Unknown error');

return data.result; // ⇦ what ElevenLabs expects
}
1 change: 1 addition & 0 deletions apps/mail/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
"@dnd-kit/modifiers": "9.0.0",
"@dnd-kit/sortable": "10.0.0",
"@dnd-kit/utilities": "3.2.2",
"@dub/analytics": "0.0.29",
"@elevenlabs/react": "0.1.5",
"@fontsource-variable/geist": "5.2.6",
"@fontsource-variable/geist-mono": "5.2.6",
Expand Down
67 changes: 45 additions & 22 deletions apps/mail/providers/voice-provider.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createContext, useContext, useState } from 'react';
import { toolExecutors } from '@/lib/elevenlabs-tools';
import { useConversation } from '@elevenlabs/react';
// import { callServerTool } from '@/lib/server-tool';
import { useSession } from '@/lib/auth-client';
import type { ReactNode } from 'react';
import { toast } from 'sonner';
Expand All @@ -19,6 +19,23 @@ interface VoiceContextType {
sendContext: (context: any) => void;
}

const toolNames = [
'listEmails',
'getEmail',
'sendEmail',
'markAsRead',
'markAsUnread',
'archiveEmails',
'deleteEmails',
'deleteEmail',
'createLabel',
'applyLabel',
'removeLabel',
'searchEmails',
'webSearch',
'summarizeEmail',
] as const;

const VoiceContext = createContext<VoiceContextType | undefined>(undefined);

export function VoiceProvider({ children }: { children: ReactNode }) {
Expand All @@ -27,7 +44,7 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
const [isInitializing, setIsInitializing] = useState(false);
const [lastToolCall, setLastToolCall] = useState<string | null>(null);
const [isOpen, setOpen] = useState(false);
const [currentContext, setCurrentContext] = useState<any>(null);
const [, setCurrentContext] = useState<any>(null);
Copy link
Contributor

@cubic-dev-ai cubic-dev-ai bot Aug 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

State getter is ignored; prefer useRef when only the setter-equivalent is required to avoid extra re-renders.

Prompt for AI agents 
Address the following comment on apps/mail/providers/voice-provider.tsx at line 47:

<comment>State getter is ignored; prefer `useRef` when only the setter-equivalent is required to avoid extra re-renders.</comment>

<file context>
@@ -27,7 +44,7 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
   const [isInitializing, setIsInitializing] = useState(false);
   const [lastToolCall, setLastToolCall] = useState&lt;string | null&gt;(null);
   const [isOpen, setOpen] = useState(false);
-  const [currentContext, setCurrentContext] = useState&lt;any&gt;(null);
+  const [, setCurrentContext] = useState&lt;any&gt;(null);
 
   const conversation = useConversation({
</file context>


const conversation = useConversation({
onConnect: () => {
Expand All @@ -42,26 +59,32 @@ export function VoiceProvider({ children }: { children: ReactNode }) {
toast.error(typeof error === 'string' ? error : error.message);
setIsInitializing(false);
},
clientTools: Object.entries(toolExecutors).reduce(
(acc: Record<string, any>, [name, executor]) => {
acc[name] = async (params: any) => {
console.log(`[Voice Tool] ${name} called with params:`, params);
setLastToolCall(`Executing: ${name}`);

const paramsWithContext = {
...params,
_context: currentContext,
};

const result = await executor(paramsWithContext);
console.log(`[Voice Tool] ${name} result:`, result);
setLastToolCall(null);
return result;
};
return acc;
},
{},
),
// clientTools: toolNames.reduce(
// (acc, name) => {
// acc[name] = async (params: any) => {
// console.log(`[Voice Tool] ${name} called with params:`, params);
// setLastToolCall(`Executing: ${name}`);

// try {
// const result = await callServerTool(
// name,
// { ...params, _context: currentContext },
// session?.user.phoneNumber ?? session?.user.email ?? '',
// );

// console.log(`[Voice Tool] ${name} result:`, result);
// setLastToolCall(null);
// return result;
// } catch (err) {
// setLastToolCall(null);
// toast.error(`Tool "${name}" failed: ${(err as Error).message}`);
// throw err;
// }
// };
// return acc;
// },
// {} as Record<string, (params: any) => Promise<any>>,
// ),
});

const { status, isSpeaking } = conversation;
Expand Down
1 change: 1 addition & 0 deletions apps/mail/types/tools.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export enum Tools {
GetThreadSummary = 'getThreadSummary',
GetThread = 'getThread',
ComposeEmail = 'composeEmail',
ListThreads = 'listThreads',
Expand Down
2 changes: 2 additions & 0 deletions apps/server/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"@arcadeai/arcadejs": "1.8.1",
"@barkleapp/css-sanitizer": "1.0.0",
"@coinbase/cookie-manager": "1.1.8",
"@dub/better-auth": "0.0.3",
"@googleapis/gmail": "12.0.0",
"@googleapis/people": "3.0.9",
"@hono/trpc-server": "^0.3.4",
Expand All @@ -55,6 +56,7 @@
"dedent": "^1.6.0",
"dormroom": "1.0.1",
"drizzle-orm": "catalog:",
"dub": "0.64.2",
"effect": "3.16.12",
"elevenlabs": "1.59.0",
"email-addresses": "^5.0.0",
Expand Down
30 changes: 0 additions & 30 deletions apps/server/src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -668,36 +668,6 @@ const app = new Hono<HonoContext>()
},
{ replaceRequest: false },
)
.mount(
'/vsse',
async (request, env, ctx) => {
const authBearer = request.headers.get('Authorization');
if (!authBearer) {
console.log('No auth provided');
return new Response('Unauthorized', { status: 401 });
}
if (authBearer !== env.VOICE_SECRET) {
return new Response('Unauthorized', { status: 401 });
}
const callerId = request.headers.get('X-Caller');
if (!callerId || callerId === 'system__caller_id') {
return ZeroMCP.serveSSE('/vsse', { binding: 'ZERO_MCP' }).fetch(request, env, ctx);
}
const { db, conn } = createDb(env.HYPERDRIVE.connectionString);
const foundUser = await db.query.user.findFirst({
where: and(eq(user.phoneNumber, callerId), eq(user.phoneNumberVerified, true)),
});
await conn.end();
if (!foundUser) {
return new Response('Unauthorized', { status: 401 });
}
ctx.props = {
userId: foundUser.id,
};
return ZeroMCP.serveSSE('/vsse', { binding: 'ZERO_MCP' }).fetch(request, env, ctx);
},
{ replaceRequest: false },
)
.mount(
'/mcp/thinking/sse',
async (request, env, ctx) => {
Expand Down
58 changes: 49 additions & 9 deletions apps/server/src/routes/agent/tools.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { getCurrentDateContext, GmailSearchAssistantSystemPrompt } from '../../lib/prompts';
import { composeEmail } from '../../trpc/routes/ai/compose';
import { perplexity } from '@ai-sdk/perplexity';
import { generateText, tool } from 'ai';

import { getZeroAgent } from '../../lib/server-utils';
import { perplexity } from '@ai-sdk/perplexity';
import { colors } from '../../lib/prompts';
import { openai } from '@ai-sdk/openai';
import { generateText, tool } from 'ai';
import { Tools } from '../../types';
import { env } from '../../env';
import { z } from 'zod';
Expand Down Expand Up @@ -377,6 +378,49 @@ const deleteLabel = (connectionId: string) =>
},
});

const buildGmailSearchQuery = () =>
tool({
description: 'Build a Gmail search query',
parameters: z.object({
query: z.string().describe('The search query to build, provided in natural language'),
}),
execute: async (params) => {
console.log('[DEBUG] buildGmailSearchQuery', params);

const result = await generateText({
model: openai(env.OPENAI_MODEL || 'gpt-4o'),
system: GmailSearchAssistantSystemPrompt(),
prompt: params.query,
});
return {
content: [
{
type: 'text',
text: result.text,
},
],
};
},
});

const getCurrentDate = () =>
tool({
description: 'Get the current date',
parameters: z.object({}).default({}),
execute: async () => {
console.log('[DEBUG] getCurrentDate');

return {
content: [
{
type: 'text',
text: getCurrentDateContext(),
},
],
};
},
});

export const webSearch = () =>
tool({
description: 'Search the web for information using Perplexity AI',
Expand Down Expand Up @@ -418,12 +462,8 @@ export const tools = async (connectionId: string) => {
[Tools.BulkDelete]: bulkDelete(connectionId),
[Tools.BulkArchive]: bulkArchive(connectionId),
[Tools.DeleteLabel]: deleteLabel(connectionId),
[Tools.WebSearch]: tool({
description: 'Search the web for information using Perplexity AI',
parameters: z.object({
query: z.string().describe('The query to search the web for'),
}),
}),
[Tools.BuildGmailSearchQuery]: buildGmailSearchQuery(),
[Tools.GetCurrentDate]: getCurrentDate(),
[Tools.InboxRag]: tool({
description:
'Search the inbox for emails using natural language. Returns only an array of threadIds.',
Expand Down
Loading
Loading