(DOCSP-32253): Add more semantically relevant product names to metadata

chat-core/src/DatabaseConnection.test.ts

@@ -51,7 +51,9 @@ describe("DatabaseConnection", () => {
       body: "foo",
       format: "md",
       sourceName: "source1",
-      tags: [],
+      metadata: {
+        tags: [],
+      },
       updated: new Date(),
       url: "/x/y/z",
     };
@@ -113,7 +115,9 @@ describe("DatabaseConnection", () => {
       body: "foo",
       format: "md",
       sourceName: "source1",
-      tags: [],
+      metadata: {
+        tags: [],
+      },
       updated: new Date(),
       url: "/x/y/z",
     };
@@ -153,7 +157,9 @@ describe("DatabaseConnection", () => {
         body: "The Matrix (1999) comes out",
         format: "md",
         sourceName: "",
-        tags: [],
+        metadata: {
+          tags: [],
+        },
         updated: new Date("1999-03-31"),
         url: "matrix1",
       },
@@ -162,7 +168,9 @@ describe("DatabaseConnection", () => {
         body: "The Matrix: Reloaded (2003) comes out",
         format: "md",
         sourceName: "",
-        tags: [],
+        metadata: {
+          tags: [],
+        },
         updated: new Date("2003-05-15"),
         url: "matrix2",
       },

chat-core/src/EmbeddedContent.ts

@@ -12,7 +12,7 @@ export interface EmbeddedContent {
   sourceName: string;
 
   /**
-    The original text.
+    The text represented by the vector embedding.
    */
   text: string;
 

chat-core/src/Page.ts

@@ -22,9 +22,15 @@ export type Page = {
   sourceName: string;
 
   /**
-    Arbitrary tags.
+     Arbitrary metadata for page.
    */
-  tags: string[];
+  metadata?: {
+    /**
+      Arbitrary tags.
+     */
+    tags?: string[];
+    [k: string]: unknown;
+  };
 };
 
 export type PageAction = "created" | "updated" | "deleted";

chat-server/src/processors/MongoDbUserQueryPreprocessorResponse.d.ts

@@ -10,8 +10,8 @@ export interface MongoDbUserQueryPreprocessorResponse {
   programmingLanguages: string[];
   /** One or more MongoDB products present in the content.
     Which MongoDB products is the user interested in? Ordered by relevancy.
-    Include driver if the user is asking about a programming language with a MongoDB driver.
-    @example ["atlas", "charts", "server", "compass", "bi-connector", "realm", "driver", ...other MongoDB products]
+    Include "Driver" if the user is asking about a programming language with a MongoDB driver.
+    @example ["MongoDB Atlas", "Atlas Charts", "Atlas Search", "Aggregation Framework", "MongoDB Server", "Compass", "MongoDB Connector for BI", "Realm SDK", "Driver", "Atlas App Services", ...other MongoDB products]
    */
   mongoDbProducts: string[];
   /** Using your knowledge of MongoDB and the conversational context,

chat-server/src/processors/makePreprocessMongoDbUserQuery.test.ts

@@ -60,7 +60,7 @@ describe("makePreprocessMongoDbUserQuery()", () => {
     } = response;
     expect(outputQuery).toContain("MongoDB");
     expect(outputQuery).toContain("code example");
-    expect(outputQuery).toContain("aggregation");
+    expect(outputQuery.toLowerCase()).toContain("aggregation");
     expect(outputQuery).toContain("?");
     expect(programmingLanguages).toStrictEqual(["shell"]);
     expect(mongoDbProducts[0]).toBeDefined();
@@ -83,7 +83,7 @@ describe("makePreprocessMongoDbUserQuery()", () => {
       messages,
     });
     const { mongoDbProducts } = response;
-    expect(mongoDbProducts[0]).toBe("charts");
+    expect(mongoDbProducts[0]).toBe("Atlas Charts");
   });
   test("should be aware of MongoDB", async () => {
     const query = "ruby lookup example";

ingest/src/DevCenterDataSource.ts

@@ -52,7 +52,9 @@ export const makeDevCenterDataSource = async ({
             }),
             format: "md",
             sourceName: name,
-            tags: [], // TODO
+            metadata: {
+              tags: [], // TODO
+            },
             url: /^https?:\/\//.test(document.calculated_slug)
               ? document.calculated_slug
               : new URL(

ingest/src/ProjectBase.d.ts

@@ -17,4 +17,10 @@ export interface ProjectBase {
     @example ["kotlin", "docs", "driver"]
    */
   tags?: string[];
+
+  /**
+    Name of the product.
+    @example "MongoDB Atlas"
+  */
+  productName?: string;
 }

ingest/src/SnootyDataSource.test.ts

@@ -57,7 +57,9 @@ describe("SnootyDataSource", () => {
       expect(pages[1]).toMatchObject({
         format: "md",
         sourceName: "snooty-test",
-        tags: ["docs", "manual"],
+        metadata: {
+          tags: ["docs", "manual"],
+        },
         url: "https://mongodb.com/docs/v6.0/administration/",
         body: firstPageText,
       });
@@ -73,7 +75,9 @@ describe("SnootyDataSource", () => {
       expect(pages[0]).toMatchObject({
         format: "md",
         sourceName: "snooty-docs",
-        tags: ["docs", "manual"],
+        metadata: {
+          tags: ["docs", "manual"],
+        },
         url: "https://mongodb.com/docs/v6.0/",
       });
 
@@ -83,23 +87,29 @@ describe("SnootyDataSource", () => {
       expect(pages[2]).toMatchObject({
         format: "md",
         sourceName: "snooty-docs",
-        tags: ["docs", "manual"],
+        metadata: {
+          tags: ["docs", "manual"],
+        },
         url: "https://mongodb.com/docs/v6.0/administration/analyzing-mongodb-performance/index/",
       });
 
       // This has index in the middle of the page_id that should not be stripped
       expect(pages[3]).toMatchObject({
         format: "md",
         sourceName: "snooty-docs",
-        tags: ["docs", "manual"],
+        metadata: {
+          tags: ["docs", "manual"],
+        },
         url: "https://mongodb.com/docs/v6.0/administration/index/backup-sharded-clusters/",
       });
 
       // This has index but part of a wider phrase so should not be stripped
       expect(pages[4]).toMatchObject({
         format: "md",
         sourceName: "snooty-docs",
-        tags: ["docs", "manual"],
+        metadata: {
+          tags: ["docs", "manual"],
+        },
         url: "https://mongodb.com/docs/v6.0/administration/change-streams-production-recommendations/how-to-index/",
       });
     });

ingest/src/SnootyDataSource.ts

@@ -97,7 +97,13 @@ export const makeSnootyDataSource = async ({
     _snootyProjectName: string;
   }
 > => {
-  const { baseUrl, currentBranch, name: snootyProjectName, tags } = project;
+  const {
+    baseUrl,
+    currentBranch,
+    name: snootyProjectName,
+    tags,
+    productName,
+  } = project;
   return {
     // Additional members for testing purposes
     _baseUrl: baseUrl,
@@ -128,7 +134,7 @@ export const makeSnootyDataSource = async ({
                 (async () => {
                   const page = await handlePage(
                     (entry as SnootyPageEntry).data,
-                    { sourceName, baseUrl, tags: tags ?? [] }
+                    { sourceName, baseUrl, tags: tags ?? [], productName }
                   );
                   pages.push(page);
                 })()
@@ -214,10 +220,12 @@ const handlePage = async (
     sourceName,
     baseUrl,
     tags = [],
+    productName,
   }: {
     sourceName: string;
     baseUrl: string;
     tags: string[];
+    productName?: string;
   }
 ): Promise<Page> => {
   // Strip first three path segments - according to Snooty team, they'll always
@@ -242,6 +250,9 @@ const handlePage = async (
     title: getTitleFromSnootyAst(page.ast),
     body: snootyAstToMd(page.ast, { baseUrl }),
     format: "md",
-    tags,
+    metadata: {
+      tags,
+      productName,
+    },
   };
 };