(EAI-1187) Instruct chatbot on handling fetch_page fallback (#897)

* fetch_page fallback disclaimer instruction

* PR review - importantNote shouldn't be in the systemPromptContent

* 1 prompt adherence eval case for fallback instruction

* Tool instructions in tool descriptions

* Clarify fetch_page vs. search_content usage

Author: hschawe

packages/chatbot-server-mongodb-public/evalCases/system_prompt_conversations.yml

@@ -1,3 +1,43 @@
+- name: default_system_prompt_fetch_fallback
+  messages: 
+    - role: user
+      content: use this https://www.geeksforgeeks.org/mongodb/search-text-in-mongodb/ how do I do a vector search query
+  expectation: The response should let the user know if the fallback search was used.
+  expectedPromptAdherence:
+    - The response provides a disclaimer that the provided URL did not contain relevant information and that the knowledge base was searched instead.
+  tags:
+    - fetch_page
+    - systemPromptAdherence
+- name: default_system_prompt_no_fetch_fallback
+  messages: 
+    - role: user
+      content: What is the Atlas Architecture Center?
+  expectation: The response should not say a search was performed.
+  expectedPromptAdherence:
+    - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar.
+  tags:
+    - fetch_page
+    - systemPromptAdherence
+- name: default_system_prompt_no_fetch_fallback_2
+  messages: 
+    - role: user
+      content: How does voyage3.5 compare to voyage-3?
+  expectation: The response should not say a search was performed.
+  expectedPromptAdherence:
+    - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar.
+  tags:
+    - fetch_page
+    - systemPromptAdherence
+- name: default_system_prompt_no_fetch_fallback_3
+  messages: 
+    - role: user
+      content: What are the core values of MongoDB?
+  expectation: The response should not say a search was performed.
+  expectedPromptAdherence:
+    - The response does not say that a search was performed or mention that they used a "knowledge base". It is OK if the response is "I don't know" or similar.
+  tags:
+    - fetch_page
+    - systemPromptAdherence
 - name: custom_personality_override
   messages:
     - role: user

packages/chatbot-server-mongodb-public/src/systemPrompt.ts

@@ -3,6 +3,10 @@ import {
   mongoDbProducts,
   mongoDbProgrammingLanguages,
 } from "mongodb-rag-core/mongoDbMetadata";
+import {
+  makeMarkdownUnorderedList,
+  makeMarkdownNumberedList,
+} from "mongodb-rag-core/dataSources";
 import { SEARCH_TOOL_NAME } from "./tools/search";
 import {
   FETCH_PAGE_TOOL_NAME,
@@ -60,20 +64,14 @@ const searchRequiresRephraseNotes = [
   "You should also transform the user query into a fully formed question, if relevant.",
 ];
 
-const searchContentToolNotes = [
-  "Search all of the available MongoDB reference documents for a given user input.",
-  "You must generate an appropriate search query for a given user input.",
-  "You are doing this for MongoDB, and all queries relate to MongoDB products.",
-  `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`,
+const coordinateToolNotes = [
+  `When deciding whether to call ${FETCH_PAGE_TOOL_NAME}, DO NOT ASSUME that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the pageUrl in the Front Matter.`,
+  `The ${FETCH_PAGE_TOOL_NAME} tool should only be used if the user asks you to reference the page they are on (e.g. "Use the page I'm on..."), or if the user explicitly provides a URL in their question.`,
+  `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`,
 ];
 
-const fetchPageToolNotes = [
-  "Fetch the entire page content for a given URL.",
-  `Do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the URL in the Front Matter. The ${FETCH_PAGE_TOOL_NAME} should ONLY be used if the user implies you should look on the page or if the user explicitly provides a URL in their question.`,
-  `If the user provides URLs in their query, ONLY call the ${FETCH_PAGE_TOOL_NAME} for those URLs, and do NOT call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`,
-  "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.",
-  "Do not include URLs in the search query.",
-  `If the ${FETCH_PAGE_TOOL_NAME} tool returns the string "${SEARCH_ALL_FALLBACK_TEXT}", you MUST immediately call the ${SEARCH_TOOL_NAME} tool.`,
+const toolUseDisclaimers = [
+  `If you called the ${FETCH_PAGE_TOOL_NAME} tool and it returned the string "${SEARCH_ALL_FALLBACK_TEXT}", you must tell the user in your final answer: "I couldn't use that page to answer your question, so I searched my knowledge base to find an answer."`,
 ];
 
 const importantNote = `<important>
@@ -138,28 +136,27 @@ ${makeMarkdownNumberedList(searchRequiresRephraseNotes)}
 
 <tools>
 
-<tool name="${SEARCH_TOOL_NAME}">
+You have access to the ${SEARCH_TOOL_NAME} and ${FETCH_PAGE_TOOL_NAME} tools.
+
+<coordinate_tools>
 
-You have access to the ${SEARCH_TOOL_NAME} tool. Use the ${SEARCH_TOOL_NAME} tool as follows:
-${makeMarkdownNumberedList(searchContentToolNotes)}
+Follow these guidelines when using the tools:
 
-When you search, include metadata about the relevant MongoDB programming language and product.
-</tool>
+${makeMarkdownUnorderedList(coordinateToolNotes)}
 
-<tool name=${FETCH_PAGE_TOOL_NAME}>
+</coordinate_tools>
 
-You have access to the ${FETCH_PAGE_TOOL_NAME} tool. Use the ${FETCH_PAGE_TOOL_NAME} tool as follows:
-${makeMarkdownNumberedList(fetchPageToolNotes)}
+<tool_disclaimers>
 
-</tool>
+When writing your final answer, provide any necessary disclaimers:
+
+${makeMarkdownUnorderedList(toolUseDisclaimers)}
+
+</tool_disclaimers>
 
 </tools>`,
 } satisfies SystemMessage;
 
-function makeMarkdownNumberedList(items: string[]) {
-  return items.map((item, i) => `${i + 1}. ${item}`).join("\n");
-}
-
 export const makeMongoDbAssistantSystemPrompt: MakeSystemPrompt = (
   customSystemPrompt,
   customToolDefinitions

packages/chatbot-server-mongodb-public/src/tools/fetchPage.ts

@@ -6,7 +6,10 @@ import {
   Reference,
 } from "mongodb-rag-core";
 import { MakeReferenceLinksFunc } from "mongodb-chatbot-server";
-import { normalizeUrl } from "mongodb-rag-core/dataSources";
+import {
+  normalizeUrl,
+  makeMarkdownNumberedList,
+} from "mongodb-rag-core/dataSources";
 import { wrapTraced } from "mongodb-rag-core/braintrust";
 import { Tool, tool } from "mongodb-rag-core/aiSdk";
 
@@ -38,6 +41,13 @@ export interface MakeFetchPageToolParams {
   pageLengthCutoff?: number;
 }
 
+const fetchPageToolNotes = [
+  `Remember - do not assume that the user wants to use the ${FETCH_PAGE_TOOL_NAME} based on the pageUrl in the Front Matter. Only call this tool when specifically instructed to use a certain page.`,
+  `If the user provides multiple URLs in their query, call the ${FETCH_PAGE_TOOL_NAME} once for each URL, and do not call the ${FETCH_PAGE_TOOL_NAME} for the URL in the Front Matter.`,
+  "Sometimes, when a page is very long, a search will be performed over the page. Therefore, you must also provide a search query to the tool.",
+  "Do not include URLs in the search query.",
+];
+
 export function makeFetchPageTool({
   loadPage,
   findContent,
@@ -47,7 +57,9 @@ export function makeFetchPageTool({
 }: MakeFetchPageToolParams): FetchPageTool {
   return tool({
     inputSchema: MongoDbFetchPageToolArgsSchema,
-    description: "Fetch all content for a specific URL",
+    description: `Fetches the entire page contents for a specific URL. Use this tool as follows:
+
+${makeMarkdownNumberedList(fetchPageToolNotes)}`,
     toModelOutput(output) {
       return {
         type: "content",

packages/chatbot-server-mongodb-public/src/tools/search.ts

@@ -11,6 +11,7 @@ import {
   mongoDbProducts,
   mongoDbProgrammingLanguageIds,
 } from "mongodb-rag-core/mongoDbMetadata";
+import { makeMarkdownNumberedList } from "mongodb-rag-core/dataSources";
 import { wrapTraced } from "mongodb-rag-core/braintrust";
 import { MakeReferenceLinksFunc } from "mongodb-chatbot-server";
 
@@ -56,13 +57,24 @@ export interface MakeSearchToolParams {
   makeReferences: MakeReferenceLinksFunc;
 }
 
+const searchContentToolNotes = [
+  "Search all of the available MongoDB reference documents for a given user input.",
+  "You must generate an appropriate search query for a given user input.",
+  "You are doing this for MongoDB, and all queries relate to MongoDB products.",
+  `Only generate ONE ${SEARCH_TOOL_NAME} tool call per user message unless there are clearly multiple distinct queries needed to answer the user query.`,
+];
+
 export function makeSearchTool({
   findContent,
   makeReferences,
 }: MakeSearchToolParams): SearchTool {
   const searchTool: SearchTool = tool({
     inputSchema: MongoDbSearchToolArgsSchema,
-    description: "Search MongoDB content",
+    description: `Search MongoDB content. Use the ${SEARCH_TOOL_NAME} tool as follows:
+
+${makeMarkdownNumberedList(searchContentToolNotes)}
+    
+When you search, include metadata about the relevant MongoDB programming language and product.`,
 
     toModelOutput(result) {
       return {

packages/mongodb-rag-core/src/dataSources/index.ts

@@ -11,3 +11,4 @@ export * from "./MarkdownUrlDataSource";
 export * from "./LangchainDocumentLoaderDataSource";
 export * from "./CodeOnGithubTextDataSource";
 export * from "./normalizeUrl";
+export * from "./makeMarkdownLists";

packages/mongodb-rag-core/src/dataSources/makeMarkdownLists.ts

@@ -0,0 +1,7 @@
+export function makeMarkdownNumberedList(items: string[]) {
+  return items.map((item, i) => `${i + 1}. ${item}`).join("\n");
+}
+
+export function makeMarkdownUnorderedList(items: string[]) {
+  return items.map((item) => `- ${item}`).join("\n");
+}