From dfce0ed47b5bad4475d4996a412de66e1bdcef50 Mon Sep 17 00:00:00 2001 From: Louis Murerwa Date: Tue, 21 May 2024 20:06:54 -0700 Subject: [PATCH 1/6] Add Chat Streaming Functionality --- .../plugins/open-ai/src/__tests__/open-ai | 52 --------- .../plugins/open-ai/src/__tests__/open-ai.ts | 64 +++++++++++ .../plugins/open-ai/src/services/open-ai.ts | 103 ++++++++++++------ 3 files changed, 131 insertions(+), 88 deletions(-) delete mode 100644 packages/plugins/open-ai/src/__tests__/open-ai create mode 100644 packages/plugins/open-ai/src/__tests__/open-ai.ts diff --git a/packages/plugins/open-ai/src/__tests__/open-ai b/packages/plugins/open-ai/src/__tests__/open-ai deleted file mode 100644 index e58a2ea0..00000000 --- a/packages/plugins/open-ai/src/__tests__/open-ai +++ /dev/null @@ -1,52 +0,0 @@ -import { OpenAI } from 'openai'; -import OpenAIService from '../services/open-ai'; - -describe('OpenAIService', () => { - let service; - - beforeEach(() => { - service = new OpenAIService({ - rateLimiterService: { - getRequestQueue: jest.fn().mockReturnValue({ - removeTokens: jest.fn() - }) - } - }, { - embedding_model: process.env.OPEN_AI_EMBEDDING_MODEL, - open_ai_key: process.env.OPEN_AI_KEY, - chat_model: process.env.OPEN_AI_CHAT_MODEL - }); - }); - - it('should create embeddings', async () => { - const doc = { content: 't' }; - const embeddings = await service.createEmbeddings("test content"); - expect(embeddings).toEqual([1, 2, 3]); - expect(OpenAI).toHaveBeenCalledWith({ - apiKey: 'test-key', - defaultQuery: { 'api-version': 'test-version' }, - defaultHeaders: { 'api-key': 'test-key' }, - baseURL: 'test-endpoint/openai/deployments/test-deployment' - }); - expect(service.openai_.embeddings.create).toHaveBeenCalledWith({ - input: 'test content', - model: 'test-model' - }); - }); - - it('should return empty array if doc content is not provided', async () => { - const doc = {}; - const embeddings = await service.createEmbeddings("test"); - expect(embeddings).toEqual([]); - }); - - // it('should complete chat', async () => { - // const messages = [ - // {role: 'system', content: 'You are a helpful assistant.'}, - // {role: 'user', content: 'Translate the following English text to French: "Hello, how are you?"'} - // ]; - // const result = await service.completeChat(messages); - // expect(result).toEqual( "\"Bonjour, comment vas-tu ?\""); - // }); - -}); \ No newline at end of file diff --git a/packages/plugins/open-ai/src/__tests__/open-ai.ts b/packages/plugins/open-ai/src/__tests__/open-ai.ts new file mode 100644 index 00000000..94e09973 --- /dev/null +++ b/packages/plugins/open-ai/src/__tests__/open-ai.ts @@ -0,0 +1,64 @@ +import { OpenAI } from "openai"; +import OpenAIService from "../services/open-ai"; +import dotenv from "dotenv"; +dotenv.config({ path: "../../ocular/.env.dev" }); + +describe("OpenAIService", () => { + let service; + + beforeEach(() => { + service = new OpenAIService( + { + rateLimiterService: { + getRequestQueue: jest.fn().mockReturnValue({ + removeTokens: jest.fn(), + }), + }, + }, + { + embedding_model: process.env.OPEN_AI_EMBEDDING_MODEL, + open_ai_key: process.env.OPEN_AI_KEY, + chat_model: process.env.OPEN_AI_CHAT_MODEL, + } + ); + }); + + // it("should create embeddings", async () => { + // const doc = { content: "t" }; + // const embeddings = await service.createEmbeddings("test content"); + // expect(embeddings).toEqual([1, 2, 3]); + // expect(OpenAI).toHaveBeenCalledWith({ + // apiKey: "test-key", + // defaultQuery: { "api-version": "test-version" }, + // defaultHeaders: { "api-key": "test-key" }, + // baseURL: "test-endpoint/openai/deployments/test-deployment", + // }); + // expect(service.openai_.embeddings.create).toHaveBeenCalledWith({ + // input: "test content", + // model: "test-model", + // }); + // }); + + // it("should return empty array if doc content is not provided", async () => { + // const doc = {}; + // const embeddings = await service.createEmbeddings("test"); + // expect(embeddings).toEqual([]); + // }); + + it("should complete chat with stream", async () => { + const messages = [ + { role: "system", content: "You are a helpful assistant." }, + { + role: "user", + content: + 'Translate the following English text to French: "Hello, how are you?"', + }, + ]; + const stream = service.completeChatWithStreaming(messages); + let result = ""; + for await (const chunk of stream) { + result += chunk; + } + expect(result).toEqual('"Bonjour, comment vas-tu ?"'); + }); +}); diff --git a/packages/plugins/open-ai/src/services/open-ai.ts b/packages/plugins/open-ai/src/services/open-ai.ts index e767058e..0b122a91 100644 --- a/packages/plugins/open-ai/src/services/open-ai.ts +++ b/packages/plugins/open-ai/src/services/open-ai.ts @@ -1,71 +1,102 @@ -import { IndexableDocument, AbstractLLMService, IndexableDocChunk, Message, PluginNameDefinitions } from "@ocular/types"; -import { OpenAI } from 'openai'; -import { encoding_for_model, type TiktokenModel } from 'tiktoken'; +import { + IndexableDocument, + AbstractLLMService, + IndexableDocChunk, + Message, + PluginNameDefinitions, +} from "@ocular/types"; +import { OpenAI } from "openai"; +import { encoding_for_model, type TiktokenModel } from "tiktoken"; import { RateLimiterService } from "@ocular/ocular"; -import { RateLimiterQueue } from "rate-limiter-flexible" +import { RateLimiterQueue } from "rate-limiter-flexible"; export default class OpenAIService extends AbstractLLMService { + static identifier = PluginNameDefinitions.OPENAI; - static identifier = PluginNameDefinitions.OPENAI - - protected openAIKey_: string - protected openAI_: OpenAI - protected embeddingModel_: string - protected chatModel_: string - protected tokenLimit_:number = 4096 + protected openAIKey_: string; + protected openAI_: OpenAI; + protected embeddingModel_: string; + protected chatModel_: string; + protected tokenLimit_: number = 4096; protected rateLimiterService_: RateLimiterService; - protected requestQueue_: RateLimiterQueue + protected requestQueue_: RateLimiterQueue; constructor(container, options) { - super(arguments[0],options) + super(arguments[0], options); - // Rate Limiter Service + // Rate Limiter Service this.rateLimiterService_ = container.rateLimiterService; - this.requestQueue_ = this.rateLimiterService_.getRequestQueue(PluginNameDefinitions.OPENAI); + this.requestQueue_ = this.rateLimiterService_.getRequestQueue( + PluginNameDefinitions.OPENAI + ); // Models - this.embeddingModel_ = options.embedding_model - this.chatModel_ = options.chat_model - + this.embeddingModel_ = options.embedding_model; + this.chatModel_ = options.chat_model; + // Chat Deployment - this.openAIKey_ = options.open_ai_key + this.openAIKey_ = options.open_ai_key; this.openAI_ = new OpenAI({ - apiKey: this.openAIKey_ || "" - }) + apiKey: this.openAIKey_ || "", + }); } - async createEmbeddings(text:string): Promise { - try{ + async createEmbeddings(text: string): Promise { + try { // Rate Limiter Limits On Token Count - const tokenCount = this.getChatModelTokenCount(text) - await this.requestQueue_.removeTokens(tokenCount,PluginNameDefinitions.OPENAI) + const tokenCount = this.getChatModelTokenCount(text); + await this.requestQueue_.removeTokens( + tokenCount, + PluginNameDefinitions.OPENAI + ); const result = await this.openAI_.embeddings.create({ model: this.embeddingModel_, - input: text - }) + input: text, + }); return result.data[0].embedding; - } catch(error){ - console.log("Open AI: Error",error) + } catch (error) { + console.log("Open AI: Error", error); } } async completeChat(messages: Message[]): Promise { - try{ + try { const result = await this.openAI_.chat.completions.create({ model: this.chatModel_, messages, - temperature: 0.3, + temperature: 0.3, max_tokens: 1024, n: 1, }); - console.log("Result Open AI",result.choices[0].message.content) + console.log("Result Open AI", result.choices[0].message.content); return result.choices[0].message.content; - }catch(error){ - console.log("Open AI: Error",error) + } catch (error) { + console.log("Open AI: Error", error); + } + } + + async *completeChatWithStreaming( + messages: Message[] + ): AsyncGenerator { + try { + const result = await this.openAI_.chat.completions.create({ + model: this.chatModel_, + stream: true, + messages, + temperature: 0.3, + max_tokens: 1024, + n: 1, + }); + + for await (const chunk of result) { + yield chunk.choices[0]?.delta.content ?? ""; + } + } catch (error) { + console.log("Azure Open AI: Error", error); } } - getChatModelTokenCount(content : string): number { + getChatModelTokenCount(content: string): number { const encoder = encoding_for_model(this.chatModel_ as TiktokenModel); let tokens = 2; for (const value of Object.values(content)) { @@ -78,4 +109,4 @@ export default class OpenAIService extends AbstractLLMService { getTokenLimit(): number { return this.tokenLimit_; } -} \ No newline at end of file +} From 383bd72925f070eda3fe7984bee65bcffac9b8ac Mon Sep 17 00:00:00 2001 From: Louis Murerwa Date: Tue, 21 May 2024 20:07:43 -0700 Subject: [PATCH 2/6] Add Streaming Functionality --- package-lock.json | 247 +++++++++++++++++- packages/ocular/package.json | 2 + .../api/routes/admin/files/create-upload.ts | 2 + .../src/approaches/ask-retrieve-read.ts | 61 ++++- .../1716240473567-renamedoctypetxt.ts | 14 + .../1716247207925-adddoctypecsvjson.ts | 22 ++ packages/ocular/src/services/file.ts | 23 +- packages/ocular/src/utils/embedder.ts | 14 + packages/plugins/azure-open-ai/package.json | 2 +- .../azure-open-ai/src/__tests__/open-ai | 87 ++++-- .../azure-open-ai/src/services/open-ai.ts | 24 ++ .../document-processor/src/lib/index.ts | 2 +- .../plugins/document-processor/src/lib/md.ts | 139 ++++++++++ .../src/services/document-processor.ts | 11 + packages/types/src/common/document.ts | 2 + .../src/interfaces/approach-interface.ts | 2 +- .../types/src/interfaces/llm-interface.ts | 30 ++- 17 files changed, 635 insertions(+), 49 deletions(-) create mode 100644 packages/ocular/src/migrations/1716240473567-renamedoctypetxt.ts create mode 100644 packages/ocular/src/migrations/1716247207925-adddoctypecsvjson.ts create mode 100644 packages/ocular/src/utils/embedder.ts create mode 100644 packages/plugins/document-processor/src/lib/md.ts diff --git a/package-lock.json b/package-lock.json index a57225b7..705551d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -77,6 +77,22 @@ "undici-types": "~5.26.4" } }, + "node_modules/@azure-rest/core-client": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-1.4.0.tgz", + "integrity": "sha512-ozTDPBVUDR5eOnMIwhggbnVmOrka4fXCs8n8mvUo4WLLc38kki6bAOByDoVZZPz/pZy2jMt2kwfpvy/UjALj6w==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-rest-pipeline": "^1.5.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/abort-controller": { "version": "2.1.2", "license": "MIT", @@ -154,6 +170,17 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/core-sse": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/core-sse/-/core-sse-2.1.2.tgz", + "integrity": "sha512-yf+pFIu8yCzXu9RbH2+8kp9vITIKJLHgkLgFNA6hxiDHK3fxeP596cHUj4c8Cm8JlooaUnYdHmF84KCZt3jbmw==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/core-tracing": { "version": "1.1.2", "license": "MIT", @@ -185,6 +212,23 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/openai": { + "version": "1.0.0-beta.12", + "resolved": "https://registry.npmjs.org/@azure/openai/-/openai-1.0.0-beta.12.tgz", + "integrity": "sha512-qKblxr6oVa8GsyNzY+/Ub9VmEsPYKhBrUrPaNEQiM+qrxnBPVm9kaeqGFFb/U78Q2zOabmhF9ctYt3xBW0nWnQ==", + "dependencies": { + "@azure-rest/core-client": "^1.1.7", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.13.0", + "@azure/core-sse": "^2.0.0", + "@azure/core-util": "^1.4.0", + "@azure/logger": "^1.0.3", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/search-documents": { "version": "12.0.0", "license": "MIT", @@ -4344,6 +4388,14 @@ "onnxruntime-node": "1.14.0" } }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.10", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.10.tgz", + "integrity": "sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/abbrev": { "version": "1.1.1", "dev": true, @@ -5098,6 +5150,11 @@ "ieee754": "^1.1.13" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==" + }, "node_modules/body-parser": { "version": "1.20.2", "license": "MIT", @@ -6036,6 +6093,32 @@ "node": "*" } }, + "node_modules/d3-dsv": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-2.0.0.tgz", + "integrity": "sha512-E+Pn8UJYx9mViuIUkoc93gJGGYut6mSDKy2+XaPwccwkRGlR+LO97L2VCCRjQivTwLHkSnAJG7yo00BWY6QM+w==", + "dependencies": { + "commander": "2", + "iconv-lite": "0.4", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json", + "csv2tsv": "bin/dsv2dsv", + "dsv2dsv": "bin/dsv2dsv", + "dsv2json": "bin/dsv2json", + "json2csv": "bin/json2dsv", + "json2dsv": "bin/json2dsv", + "json2tsv": "bin/json2dsv", + "tsv2csv": "bin/dsv2dsv", + "tsv2json": "bin/dsv2json" + } + }, + "node_modules/d3-dsv/node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, "node_modules/damerau-levenshtein": { "version": "1.0.8", "dev": true, @@ -6308,6 +6391,11 @@ "md5": "^2.3.0" } }, + "node_modules/dingbat-to-unicode": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", + "integrity": "sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w==" + }, "node_modules/dir-glob": { "version": "3.0.1", "license": "MIT", @@ -6344,6 +6432,14 @@ "url": "https://github.com/motdotla/dotenv?sponsor=1" } }, + "node_modules/duck": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz", + "integrity": "sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==", + "dependencies": { + "underscore": "^1.13.1" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "license": "MIT" @@ -8136,6 +8232,11 @@ "dev": true, "license": "ISC" }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" + }, "node_modules/import-fresh": { "version": "3.3.0", "license": "MIT", @@ -9482,6 +9583,49 @@ "node": ">=4.0" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/jwa": { "version": "2.0.0", "license": "MIT", @@ -9654,6 +9798,14 @@ "version": "1.10.60", "license": "MIT" }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "license": "MIT" @@ -9836,6 +9988,16 @@ "loose-envify": "cli.js" } }, + "node_modules/lop": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.1.tgz", + "integrity": "sha512-9xyho9why2A2tzm5aIcMWKvzqKsnxrf9B5I+8O30olh6lQU8PH978LqZoI4++37RBgS1Em5i54v1TFs/3wnmXQ==", + "dependencies": { + "duck": "^0.1.12", + "option": "~0.2.1", + "underscore": "^1.13.1" + } + }, "node_modules/lower-case": { "version": "2.0.2", "license": "MIT", @@ -9890,6 +10052,42 @@ "tmpl": "1.0.5" } }, + "node_modules/mammoth": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.7.2.tgz", + "integrity": "sha512-MqWU2hcLf1I5QMKyAbfJCvrLxnv5WztrAQyorfZ+WPq7Hk82vZFmvfR2/64ajIPpM4jlq0TXp1xZvp/FFaL1Ug==", + "dependencies": { + "@xmldom/xmldom": "^0.8.6", + "argparse": "~1.0.3", + "base64-js": "^1.5.1", + "bluebird": "~3.4.0", + "dingbat-to-unicode": "^1.0.1", + "jszip": "^3.7.1", + "lop": "^0.4.1", + "path-is-absolute": "^1.0.0", + "underscore": "^1.13.1", + "xmlbuilder": "^10.0.0" + }, + "bin": { + "mammoth": "bin/mammoth" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/mammoth/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/mammoth/node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, "node_modules/math-random": { "version": "1.0.4", "license": "MIT" @@ -13240,6 +13438,11 @@ "version": "12.1.3", "license": "MIT" }, + "node_modules/option": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", + "integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==" + }, "node_modules/optionator": { "version": "0.9.3", "dev": true, @@ -13464,6 +13667,11 @@ "version": "1.0.0", "license": "MIT" }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, "node_modules/parent-module": { "version": "1.0.1", "license": "MIT", @@ -14551,6 +14759,11 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" + }, "node_modules/safe-array-concat": { "version": "1.1.2", "dev": true, @@ -14723,6 +14936,11 @@ "node": ">= 0.4" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + }, "node_modules/setprototypeof": { "version": "1.2.0", "license": "ISC" @@ -15907,6 +16125,11 @@ "dev": true, "license": "MIT" }, + "node_modules/underscore": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" + }, "node_modules/undici": { "version": "5.28.4", "license": "MIT", @@ -16384,6 +16607,14 @@ } } }, + "node_modules/xmlbuilder": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-10.1.1.tgz", + "integrity": "sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "license": "MIT", @@ -16705,6 +16936,7 @@ "cookie-parser": "^1.4.6", "core-js": "^3.36.1", "cors": "^2.8.5", + "d3-dsv": "^2.0.0", "express-session": "^1.18.0", "fast-glob": "^3.3.2", "fs-exists-cached": "^1.0.0", @@ -16713,6 +16945,7 @@ "kafka-js": "^0.0.0", "kafkajs": "^2.2.4", "langchain": "^0.1.37", + "mammoth": "^1.7.2", "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "ora": "^3.4.0", @@ -17757,9 +17990,10 @@ "packages/plugins/azure-open-ai": { "version": "0.0.0", "dependencies": { + "@azure/openai": "^1.0.0-beta.6", "@ocular/types": "*", "@ocular/utils": "*", - "openai": "^4.29.2", + "dotenv": "^16.4.5", "tiktoken": "^1.0.13" }, "devDependencies": { @@ -17768,6 +18002,17 @@ "typescript": "^4.9.5" } }, + "packages/plugins/azure-open-ai/node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "packages/plugins/azure-open-ai/node_modules/typescript": { "version": "4.9.5", "dev": true, diff --git a/packages/ocular/package.json b/packages/ocular/package.json index 5db90560..1e4b5e7f 100644 --- a/packages/ocular/package.json +++ b/packages/ocular/package.json @@ -39,6 +39,7 @@ "cookie-parser": "^1.4.6", "core-js": "^3.36.1", "cors": "^2.8.5", + "d3-dsv": "^2.0.0", "express-session": "^1.18.0", "fast-glob": "^3.3.2", "fs-exists-cached": "^1.0.0", @@ -47,6 +48,7 @@ "kafka-js": "^0.0.0", "kafkajs": "^2.2.4", "langchain": "^0.1.37", + "mammoth": "^1.7.2", "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", "ora": "^3.4.0", diff --git a/packages/ocular/src/api/routes/admin/files/create-upload.ts b/packages/ocular/src/api/routes/admin/files/create-upload.ts index 39af7ac1..5553a5b0 100644 --- a/packages/ocular/src/api/routes/admin/files/create-upload.ts +++ b/packages/ocular/src/api/routes/admin/files/create-upload.ts @@ -93,6 +93,8 @@ enum Mimetype { PDF = "application/pdf", DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document", TXT = "text/plain", + MD = "text/markdown", + CSV = "text/csv", } export class UploadReq { diff --git a/packages/ocular/src/approaches/ask-retrieve-read.ts b/packages/ocular/src/approaches/ask-retrieve-read.ts index 486ee529..c1b72f0d 100644 --- a/packages/ocular/src/approaches/ask-retrieve-read.ts +++ b/packages/ocular/src/approaches/ask-retrieve-read.ts @@ -111,9 +111,64 @@ export default class AskRetrieveThenRead implements ISearchApproach { } async *runWithStreaming( - query: string, + userQuery: string, context?: SearchContext - ): AsyncGenerator { - throw new Error("Streaming not supported for this approach."); + ): AsyncGenerator { + if (context?.ai_completion) { + context.retrieve_chunks = true; + } + let searchResults: SearchResults = await this.searchService_.search( + null, + userQuery, + context + ); + + let message = null; + if (context && context.ai_completion) { + // Initial System Message + // const prompt = context?.prompt_template || SYSTEM_CHAT_TEMPLATE; + + const prompt = SYSTEM_CHAT_TEMPLATE; + const tokens = this.openai_.getChatModelTokenCount(prompt); + const messageBuilder = new MessageBuilder(prompt, tokens); + + // Use Top 3 Sources To Generate AI Completion. + // TODO: Use More Sophisticated Logic To Select Sources. + const sources = searchResults.chat_completion.citations + .map((c) => c.content) + .join(""); + const userContent = `${userQuery}\nSources:\n${sources}`; + + const roleTokens: number = this.openai_.getChatModelTokenCount("user"); + const messageTokens: number = + this.openai_.getChatModelTokenCount(userContent); + messageBuilder.appendMessage( + "user", + userContent, + 1, + roleTokens + messageTokens + ); + + // Add shots/samples. This helps model to mimic response and make sure they match rules laid out in system message. + // messageBuilder.appendMessage('assistant', QUESTION); + // messageBuilder.appendMessage('user', ANSWER) + const messages = messageBuilder.messages; + const chatCompletion = await this.openai_.completeChat(messages); + // Add Sources To The Search Results + + for await (const chunk of chatCompletion) { + searchResults.chat_completion.content = chunk; + + yield searchResults; + } + searchResults.chat_completion.content = chatCompletion; + } + + // const sources = new Set( + // searchResults.hits.map( + // (hit) => hit.documentMetadata?.source as AppNameDefinitions + // ) + // ); + // searchResults.sources = [...sources]; } } diff --git a/packages/ocular/src/migrations/1716240473567-renamedoctypetxt.ts b/packages/ocular/src/migrations/1716240473567-renamedoctypetxt.ts new file mode 100644 index 00000000..e6fa1dc8 --- /dev/null +++ b/packages/ocular/src/migrations/1716240473567-renamedoctypetxt.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class Renamedoctypetxt1716240473567 implements MigrationInterface { + name = 'Renamedoctypetxt1716240473567' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "o_auth" ALTER COLUMN "last_sync" SET DEFAULT NULL`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "o_auth" ALTER COLUMN "last_sync" DROP DEFAULT`); + } + +} diff --git a/packages/ocular/src/migrations/1716247207925-adddoctypecsvjson.ts b/packages/ocular/src/migrations/1716247207925-adddoctypecsvjson.ts new file mode 100644 index 00000000..2b3ae0e5 --- /dev/null +++ b/packages/ocular/src/migrations/1716247207925-adddoctypecsvjson.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class Adddoctypecsvjson1716247207925 implements MigrationInterface { + name = 'Adddoctypecsvjson1716247207925' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "o_auth" ALTER COLUMN "last_sync" SET DEFAULT NULL`); + await queryRunner.query(`ALTER TYPE "public"."document_metadata_type_enum" RENAME TO "document_metadata_type_enum_old"`); + await queryRunner.query(`CREATE TYPE "public"."document_metadata_type_enum" AS ENUM('pdf', 'txt', 'docx', 'html', 'md', 'json', 'csv')`); + await queryRunner.query(`ALTER TABLE "document_metadata" ALTER COLUMN "type" TYPE "public"."document_metadata_type_enum" USING "type"::"text"::"public"."document_metadata_type_enum"`); + await queryRunner.query(`DROP TYPE "public"."document_metadata_type_enum_old"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TYPE "public"."document_metadata_type_enum_old" AS ENUM('pdf', 'txt', 'docx', 'html', 'md')`); + await queryRunner.query(`ALTER TABLE "document_metadata" ALTER COLUMN "type" TYPE "public"."document_metadata_type_enum_old" USING "type"::"text"::"public"."document_metadata_type_enum_old"`); + await queryRunner.query(`DROP TYPE "public"."document_metadata_type_enum"`); + await queryRunner.query(`ALTER TYPE "public"."document_metadata_type_enum_old" RENAME TO "document_metadata_type_enum"`); + await queryRunner.query(`ALTER TABLE "o_auth" ALTER COLUMN "last_sync" DROP DEFAULT`); + } + +} diff --git a/packages/ocular/src/services/file.ts b/packages/ocular/src/services/file.ts index bdb60a6a..9de9fbaa 100644 --- a/packages/ocular/src/services/file.ts +++ b/packages/ocular/src/services/file.ts @@ -11,6 +11,9 @@ import path from "path"; import fs from "fs"; import * as fsPromise from "fs/promises"; import { parse } from "path"; +import mammoth from "mammoth"; +import { CSVLoader } from "langchain/document_loaders/fs/csv"; +import { JSONLoader } from "langchain/document_loaders/fs/json"; export default class FileService extends AbstractFileService { static identifier = "localfs"; @@ -95,8 +98,26 @@ export default class FileService extends AbstractFileService { case ".txt": completeText = await fsPromise.readFile(filePath, "utf-8"); break; + case ".docx": + const result = await mammoth.extractRawText({ path: filePath }); + completeText = result.value; + break; + case ".md": + completeText = await fsPromise.readFile(filePath, "utf-8"); + break; + case ".csv": + const csvLoader = new CSVLoader(filePath); + const csvData = await csvLoader.load(); + completeText = await csvData.map((doc) => doc.pageContent).join(" "); + console.log("CSV Data", completeText); + break; + case ".json": + const jsonLoader = new JSONLoader(filePath); + const jsonData = await jsonLoader.load(); + completeText = await docs.map((doc) => doc.pageContent).join(" "); + break; default: - throw new Error(`Unsupported file extension: ${extension}`); + throw new Error(`Failed to File With: ${extension} `); } return completeText; } diff --git a/packages/ocular/src/utils/embedder.ts b/packages/ocular/src/utils/embedder.ts new file mode 100644 index 00000000..1c38dae1 --- /dev/null +++ b/packages/ocular/src/utils/embedder.ts @@ -0,0 +1,14 @@ +const TransformersApi = Function('return import("@xenova/transformers")')(); +export async function generateEmbedding(content: string): Promise { + const { pipeline } = await TransformersApi; + const generateEmbedding = await pipeline( + "feature-extraction", + "Xenova/all-MiniLM-L6-v2" + ); + const output = await generateEmbedding(content, { + pooling: "mean", + normalize: true, + }); + const embedding: number[] = Array.from(output.data); + return embedding; +} diff --git a/packages/plugins/azure-open-ai/package.json b/packages/plugins/azure-open-ai/package.json index 74f14df9..79f36402 100644 --- a/packages/plugins/azure-open-ai/package.json +++ b/packages/plugins/azure-open-ai/package.json @@ -20,7 +20,7 @@ "dependencies": { "@ocular/types": "*", "@ocular/utils": "*", - "openai": "^4.29.2", + "dotenv": "^16.4.5", "tiktoken": "^1.0.13" } } diff --git a/packages/plugins/azure-open-ai/src/__tests__/open-ai b/packages/plugins/azure-open-ai/src/__tests__/open-ai index 181ac09b..1b636e52 100644 --- a/packages/plugins/azure-open-ai/src/__tests__/open-ai +++ b/packages/plugins/azure-open-ai/src/__tests__/open-ai @@ -1,49 +1,80 @@ -import { OpenAI } from 'openai'; -import OpenAIService from '../services/open-ai'; -describe('OpenAIService', () => { - let service; +import { OpenAI } from "openai"; +import OpenAIService from "../services/open-ai"; +import dotenv from "dotenv"; +dotenv.config({ path: "../../ocular/.env.dev" }); + +console.log(process.env.AZURE_OPEN_AI_KEY); // This should print your key +describe("OpenAIService", () => { + let service; beforeEach(() => { - service = new OpenAIService({}, { - open_ai_version: "2023-05-15", - endpoint: process.env.AZURE_OPEN_AI_ENDPOINT, - embedding_deployment_name: process.env.AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME, - embedding_model: process.env.AZURE_OPEN_AI_EMBEDDING_MODEL, - open_ai_key: process.env.AZURE_OPEN_AI_KEY, - chat_deployment: process.env.AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME, - chat_model: process.env.AZURE_OPEN_AI_CHAT_MODEL - }); + service = new OpenAIService( + { + rateLimiterService: { + getRequestQueue: jest.fn(), + }, + }, + { + open_ai_version: "2023-05-15", + endpoint: process.env.AZURE_OPEN_AI_ENDPOINT, + embedding_deployment_name: + process.env.AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME, + embedding_model: process.env.AZURE_OPEN_AI_EMBEDDING_MODEL, + open_ai_key: process.env.AZURE_OPEN_AI_KEY, + chat_deployment: process.env.AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME, + chat_model: process.env.AZURE_OPEN_AI_CHAT_MODEL, + } + ); }); - - // it('should create embeddings', async () => { - // const doc = { content: 't' }; + // it("should create embeddings", async () => { + // const doc = { content: "t" }; // const embeddings = await service.createEmbeddings(doc); // expect(embeddings).toEqual([1, 2, 3]); // expect(OpenAI).toHaveBeenCalledWith({ - // apiKey: 'test-key', - // defaultQuery: { 'api-version': 'test-version' }, - // defaultHeaders: { 'api-key': 'test-key' }, - // baseURL: 'test-endpoint/openai/deployments/test-deployment' + // apiKey: "test-key", + // defaultQuery: { "api-version": "test-version" }, + // defaultHeaders: { "api-key": "test-key" }, + // baseURL: "test-endpoint/openai/deployments/test-deployment", // }); // expect(service.openai_.embeddings.create).toHaveBeenCalledWith({ - // input: 'test content', - // model: 'test-model' + // input: "test content", + // model: "test-model", // }); // }); - // it('should return empty array if doc content is not provided', async () => { + // it("should return empty array if doc content is not provided", async () => { // const doc = {}; // const embeddings = await service.createEmbeddings(doc); // expect(embeddings).toEqual([]); // }); - it('should complete chat', async () => { + it("should complete chat", async () => { const messages = [ - {role: 'system', content: 'You are a helpful assistant.'}, - {role: 'user', content: 'Translate the following English text to French: "Hello, how are you?"'} + { role: "system", content: "You are a helpful assistant." }, + { + role: "user", + content: + 'Translate the following English text to French: "Hello, how are you?"', + }, ]; const result = await service.completeChat(messages); - expect(result).toEqual( "\"Bonjour, comment vas-tu ?\""); + expect(result).toEqual('"Bonjour, comment vas-tu ?"'); }); -}); \ No newline at end of file + // it("should complete chat with stream", async () => { + // const messages = [ + // { role: "system", content: "You are a helpful assistant." }, + // { + // role: "user", + // content: + // 'Translate the following English text to French: "Hello, how are you?"', + // }, + // ]; + // const stream = service.completeChatWithStream(messages); + // stream.on("data", (data) => { + // console.log(data); + // expect("result").toEqual('"Bonjour, comment vas-tu ?"'); + // }); + // expect("result").toEqual('"Bonjour, comment vas-tu ?"'); + // }); +}); diff --git a/packages/plugins/azure-open-ai/src/services/open-ai.ts b/packages/plugins/azure-open-ai/src/services/open-ai.ts index cd8c16ea..8900231f 100644 --- a/packages/plugins/azure-open-ai/src/services/open-ai.ts +++ b/packages/plugins/azure-open-ai/src/services/open-ai.ts @@ -112,6 +112,30 @@ export default class AzureOpenAIService extends AbstractLLMService { } } + completeChatWithStreaming( + messages: Message[] + ): AsyncGenerator { + const generator = async function* (messages: Message[]) { + try { + const result = await this.chatClient_.chat.completions.create({ + stream: true, + model: this.chatModel_, + messages, + temperature: 0.3, + max_tokens: 1024, + n: 1, + }); + + for await (const chunk of result) { + yield chunk.choices[0]?.delta.content ?? ""; + } + } catch (error) { + console.log("Azure Open AI: Error", error); + } + }; + return generator(messages); + } + getChatModelTokenCount(content: string): number { const encoder = encoding_for_model(this.chatModel_ as TiktokenModel); let tokens = 2; diff --git a/packages/plugins/document-processor/src/lib/index.ts b/packages/plugins/document-processor/src/lib/index.ts index ddb4e027..27405125 100644 --- a/packages/plugins/document-processor/src/lib/index.ts +++ b/packages/plugins/document-processor/src/lib/index.ts @@ -1 +1 @@ -export * from './txt'; \ No newline at end of file +export * from "./txt"; diff --git a/packages/plugins/document-processor/src/lib/md.ts b/packages/plugins/document-processor/src/lib/md.ts new file mode 100644 index 00000000..eb6e82f7 --- /dev/null +++ b/packages/plugins/document-processor/src/lib/md.ts @@ -0,0 +1,139 @@ +import { IndexableDocChunk, IndexableDocument, Section } from "@ocular/types"; +import e from "express"; +import { encode } from "gpt-tokenizer"; +import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"; + +export const processMarkdown = async ( + document: IndexableDocument, + max_chunk_size: number, + chunk_overlap: number +): Promise => { + const splitter = RecursiveCharacterTextSplitter.fromLanguage("markdown", { + chunkSize: max_chunk_size, + chunkOverlap: chunk_overlap, + }); + + const title = document.title; + const organisationId = document.organisationId; + const documentId = document.id; + const source = document.source; + const metadata = document.metadata; + const updatedAt = document.updatedAt; + + const sections: Section[] = document.sections; + // If the document has no sections, return the document as a single chunk + if (!sections || sections.length === 0) { + return [ + { + chunkId: 0, + organisationId: organisationId, + documentId: documentId, + source: source, + type: document.type, + title: title, + tokens: 0, + content: "", + metadata: metadata, + updatedAt: updatedAt, + }, + ]; + } + + // Split the document into chunks + let chunk_text = ""; + let chunk_links = {}; + let chunks: IndexableDocChunk[] = []; + for (const section of sections) { + // If the current section is bigger than the chunk size, split the section into chunks + if (encode(section.content).length > max_chunk_size) { + // Create a new document for the remaining chunk_text + if (chunk_text.length > 0) { + const tokens = encode(chunk_text).length; + const chunk = { + chunkId: chunks.length, + organisationId: organisationId, + documentId: documentId, + source: source, + type: document.type, + title: title, + content: chunk_text, + tokens: tokens, + chunkLinks: chunk_links, + metadata: metadata, + updatedAt: updatedAt, + }; + chunks.push(chunk); + chunk_text = ""; + chunk_links = {}; + } + + // Split the big section into chunks + const splitDocs = await splitter.createDocuments([section.content]); + for (let i = 0; i < splitDocs.length; i++) { + const doc = splitDocs[i]; + chunks.push({ + chunkId: chunks.length, + organisationId: organisationId, + documentId: documentId, + source: source, + type: document.type, + title: title, + content: doc.pageContent, + tokens: encode(doc.pageContent).length, + chunkLinks: { 0: section.link }, + metadata: metadata, + updatedAt: updatedAt, + }); + } + continue; + } + + // If the current chunk_text + section.content is bigger than the chunk size, create a new chunk. + if ( + chunk_text.length > 0 && + chunk_text.length + section.content.length > max_chunk_size + ) { + const tokens = encode(chunk_text).length; + const chunk = { + chunkId: chunks.length, + organisationId: document.organisationId, + documentId: document.id, + source: document.source, + type: document.type, + title: document.title, + content: chunk_text, + tokens: tokens, + chunkLinks: chunk_links, + metadata: document.metadata, + updatedAt: document.updatedAt, + }; + chunks.push(chunk); + chunk_text = section.content; + chunk_links = { 0: section.link }; + } else { + chunk_text += section.content; + let links_size = Object.keys(chunk_links).length; + chunk_links[links_size] = section.link; + } + } + + // Add the last chunk + if (chunk_text.length > 0) { + const tokens = encode(chunk_text).length; + const chunk = { + chunkId: chunks.length, + organisationId: document.organisationId, + documentId: document.id, + source: document.source, + title: document.title, + type: document.type, + content: chunk_text, + tokens: tokens, + chunkLinks: chunk_links, + metadata: document.metadata, + updatedAt: document.updatedAt, + }; + chunks.push(chunk); + } + return chunks; +}; diff --git a/packages/plugins/document-processor/src/services/document-processor.ts b/packages/plugins/document-processor/src/services/document-processor.ts index 32e154b9..46543276 100644 --- a/packages/plugins/document-processor/src/services/document-processor.ts +++ b/packages/plugins/document-processor/src/services/document-processor.ts @@ -6,6 +6,7 @@ import { DocType, } from "@ocular/types"; import { processTxt } from "../lib/txt"; +import { processMarkdown } from "../lib/md"; export default class documentProcessorService extends AbstractDocumentProcesserService { static identifier = "document-processor"; @@ -30,12 +31,22 @@ export default class documentProcessorService extends AbstractDocumentProcesserS switch (document.type as DocType) { case DocType.PDF: case DocType.TXT: + case DocType.DOCX: + case DocType.JSON: + case DocType.CSV: chunks = await processTxt( document, this.max_chunk_length_, this.chunk_over_lap_ ); break; + case DocType.MD: + chunks = await processMarkdown( + document, + this.max_chunk_length_, + this.chunk_over_lap_ + ); + break; default: console.log( "chunkIndexableDocument:Document Type Not Supported", diff --git a/packages/types/src/common/document.ts b/packages/types/src/common/document.ts index 5dde8e15..4e91406b 100644 --- a/packages/types/src/common/document.ts +++ b/packages/types/src/common/document.ts @@ -13,6 +13,8 @@ export enum DocType { DOCX = "docx", HTML = "html", MD = "md", + JSON = "json", + CSV = "csv", } // Document containing infomation from external intergrations to be indexed in the search engine diff --git a/packages/types/src/interfaces/approach-interface.ts b/packages/types/src/interfaces/approach-interface.ts index 1805bd45..7926852e 100644 --- a/packages/types/src/interfaces/approach-interface.ts +++ b/packages/types/src/interfaces/approach-interface.ts @@ -23,7 +23,7 @@ export interface ISearchApproach { runWithStreaming( query: string, context?: SearchContext - ): AsyncGenerator; + ): AsyncGenerator; } export interface IChatApproach { diff --git a/packages/types/src/interfaces/llm-interface.ts b/packages/types/src/interfaces/llm-interface.ts index 35949b9b..660a91cc 100644 --- a/packages/types/src/interfaces/llm-interface.ts +++ b/packages/types/src/interfaces/llm-interface.ts @@ -1,12 +1,13 @@ -import { AutoflowContainer, Message } from '../common'; -import { IndexableDocument,IndexableDocChunk } from '../common/document'; -import { TransactionBaseService } from './transaction-base-service'; +import { AutoflowContainer, Message } from "../common"; +import { IndexableDocument, IndexableDocChunk } from "../common/document"; +import { TransactionBaseService } from "./transaction-base-service"; export interface ILLMInterface extends TransactionBaseService { - createEmbeddings(text:string): Promise ; + createEmbeddings(text: string): Promise; completeChat(messages: Message[]): Promise; + completeChatWithStreaming(messages: Message[]): AsyncGenerator; getChatModelTokenCount(content: string): number; - getTokenLimit(): number + getTokenLimit(): number; } /** @@ -16,25 +17,28 @@ export abstract class AbstractLLMService extends TransactionBaseService implements ILLMInterface { - static _isLLM = true - static identifier: string + static _isLLM = true; + static identifier: string; static isLLMService(object): boolean { - return object?.constructor?._isLLM + return object?.constructor?._isLLM; } getIdentifier(): string { - return (this.constructor as any).identifier + return (this.constructor as any).identifier; } protected constructor( protected readonly container: AutoflowContainer, protected readonly config?: Record // eslint-disable-next-line @typescript-eslint/no-empty-function ) { - super(container, config) + super(container, config); } - abstract createEmbeddings(text:string): Promise ; + abstract createEmbeddings(text: string): Promise; abstract completeChat(messages: Message[]): Promise; + abstract completeChatWithStreaming( + messages: Message[] + ): AsyncGenerator; abstract getChatModelTokenCount(content: string): number; - abstract getTokenLimit(): number -} \ No newline at end of file + abstract getTokenLimit(): number; +} From 78eed123316bbfb8426c5fd843ee11e4bdb40bb3 Mon Sep 17 00:00:00 2001 From: Louis Murerwa Date: Wed, 22 May 2024 10:34:25 -0700 Subject: [PATCH 3/6] Add AI Streaming API --- packages/ocular/core-config.js | 46 ++--- .../ocular/src/api/routes/member/index.ts | 34 ++-- .../src/api/routes/member/search/ask.ts | 90 ++++++++++ .../src/api/routes/member/search/search.ts | 35 +--- .../src/approaches/ask-retrieve-read.ts | 159 ++++++++---------- .../approaches/chat-retrieve-read-retrieve.ts | 3 +- packages/ocular/src/services/search.ts | 40 +++-- .../azure-open-ai/src/__tests__/open-ai | 4 - .../plugins/open-ai/src/services/open-ai.ts | 5 +- packages/types/src/common/search.ts | 2 +- .../types/src/interfaces/search-interface.ts | 20 ++- 11 files changed, 256 insertions(+), 182 deletions(-) create mode 100644 packages/ocular/src/api/routes/member/search/ask.ts diff --git a/packages/ocular/core-config.js b/packages/ocular/core-config.js index ad85a696..bee7c411 100644 --- a/packages/ocular/core-config.js +++ b/packages/ocular/core-config.js @@ -180,35 +180,35 @@ module.exports = { chunk_over_lap: 0, }, }, - { - resolve: PluginNameDefinitions.AZUREOPENAI, - options: { - open_ai_key: process.env.AZURE_OPEN_AI_KEY, - open_ai_version: "2023-05-15", - endpoint: process.env.AZURE_OPEN_AI_ENDPOINT, - embedding_deployment_name: - process.env.AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME, - embedding_model: process.env.AZURE_OPEN_AI_EMBEDDING_MODEL, - chat_deployment_name: process.env.AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME, - chat_model: process.env.AZURE_OPEN_AI_CHAT_MODEL, - rate_limiter_opts: { - requests: 1, // Number of Tokens - interval: 1, // Interval in Seconds - }, - }, - }, // { - // resolve: PluginNameDefinitions.OPENAI, + // resolve: PluginNameDefinitions.AZUREOPENAI, // options: { - // open_ai_key: process.env.OPEN_AI_KEY, - // embedding_model: process.env.OPEN_AI_EMBEDDING_MODEL, - // chat_model: process.env.OPEN_AI_CHAT_MODEL, + // open_ai_key: process.env.AZURE_OPEN_AI_KEY, + // open_ai_version: "2023-05-15", + // endpoint: process.env.AZURE_OPEN_AI_ENDPOINT, + // embedding_deployment_name: + // process.env.AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME, + // embedding_model: process.env.AZURE_OPEN_AI_EMBEDDING_MODEL, + // chat_deployment_name: process.env.AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME, + // chat_model: process.env.AZURE_OPEN_AI_CHAT_MODEL, // rate_limiter_opts: { - // requests: 1000000, // Number of Tokens - // interval: 60, // Interval in Seconds + // requests: 1, // Number of Tokens + // interval: 1, // Interval in Seconds // }, // }, // }, + { + resolve: PluginNameDefinitions.OPENAI, + options: { + open_ai_key: process.env.OPEN_AI_KEY, + embedding_model: process.env.OPEN_AI_EMBEDDING_MODEL, + chat_model: process.env.OPEN_AI_CHAT_MODEL, + rate_limiter_opts: { + requests: 1000000, // Number of Tokens + interval: 60, // Interval in Seconds + }, + }, + }, { resolve: `qdrant-vector-search-service`, options: { diff --git a/packages/ocular/src/api/routes/member/index.ts b/packages/ocular/src/api/routes/member/index.ts index 0be45a06..ff22aa5f 100644 --- a/packages/ocular/src/api/routes/member/index.ts +++ b/packages/ocular/src/api/routes/member/index.ts @@ -1,24 +1,24 @@ -import { Router } from "express" +import { Router } from "express"; -import auth from "./auth" -import middlewares from "../../middlewares" +import auth from "./auth"; +import middlewares from "../../middlewares"; // import apps from "./apps" // import components from "./member/components" -import search from "./search" -import chat from "./chat" -// import teams from "./member/teams" +import search from "./search"; +import chat from "./chat"; +import { ask } from "./search"; // import organisation from "./member/organisation" export default (app, container, config) => { - const route = Router() - app.use("/",route) + const route = Router(); + app.use("/", route); // Unauthenticated Routes - auth(route) - + auth(route); + // Authenticated routes - route.use(middlewares.authenticate()) - route.use(middlewares.registeredLoggedinUser) + route.use(middlewares.authenticate()); + route.use(middlewares.registeredLoggedinUser); // // Authenticated routes // route.use(middlewares.authenticate()) @@ -27,11 +27,11 @@ export default (app, container, config) => { // apps(route) // components(route) // invites(route) - chat(route) - search(route) - // teams(route) + chat(route); + search(route); + ask(route); // organisation(route) // users(route) - return app -} \ No newline at end of file + return app; +}; diff --git a/packages/ocular/src/api/routes/member/search/ask.ts b/packages/ocular/src/api/routes/member/search/ask.ts new file mode 100644 index 00000000..fd6b1954 --- /dev/null +++ b/packages/ocular/src/api/routes/member/search/ask.ts @@ -0,0 +1,90 @@ +import { + ValidateNested, + IsOptional, + IsString, + IsBoolean, + IsEnum, + IsNumber, + IsDateString, +} from "class-validator"; +import { SearchService } from "../../../../services"; +import { validator } from "@ocular/utils"; +import { AppNameDefinitions, DocType } from "@ocular/types"; +import { Type } from "class-transformer"; + +export default async (req, res) => { + try { + const { q, context } = await validator(PostAskReq, req.body); + const searchApproach = req.scope.resolve("askRetrieveReadApproache"); + const loggedInUser = req.scope.resolve("loggedInUser"); + if (context && context.stream) { + const chunks = await searchApproach.runWithStreaming( + q, + (context as any) ?? {} + ); + // Stream the Search Results + for await (const chunk of chunks) { + res.write(JSON.stringify(chunk) + "\n"); + } + res.status(200); + res.end(); + } else { + const results = await searchApproach.run( + loggedInUser.organisation_id.toLowerCase().substring(4), + q, + (context as any) ?? {} + ); + return res.status(200).send(results); + } + } catch (_error: unknown) { + console.log(_error); + return res.status(500).send(`Error: Failed to execute SearchApproach.`); + } +}; + +class PostAskReq { + @IsString() + q: string; + + @IsOptional() + @ValidateNested() + @Type(() => AskContextReq) + context?: AskContextReq; +} + +class AskContextReq { + @IsOptional() + @IsBoolean() + suggest_followup_questions?: boolean; + + @IsOptional() + @IsNumber() + top?: number; + + @IsOptional() + @IsEnum(AppNameDefinitions, { each: true }) + sources?: Set; + + @IsOptional() + @IsEnum(DocType, { each: true }) + types?: Set; + + @IsOptional() + @ValidateNested() + @Type(() => DateRange) + date: DateRange; + + @IsOptional() + @IsBoolean() + stream?: boolean; +} + +class DateRange { + @IsOptional() + @IsDateString() + from: string; + + @IsOptional() + @IsDateString() + to: string; +} diff --git a/packages/ocular/src/api/routes/member/search/search.ts b/packages/ocular/src/api/routes/member/search/search.ts index a8c18957..01095111 100644 --- a/packages/ocular/src/api/routes/member/search/search.ts +++ b/packages/ocular/src/api/routes/member/search/search.ts @@ -12,26 +12,21 @@ import { validator } from "@ocular/utils"; import { AppNameDefinitions, DocType } from "@ocular/types"; import { Type } from "class-transformer"; -// TODO: The SearchApproach used currently is hardcoded to be AskRetrieveReadApproach. -// Improve to dynamically resolve the SearchApproaches based on the approach enum. export default async (req, res) => { try { - console.log("PostSearchReq", req.body); - const validated = await validator(PostSearchReq, req.body); - const { q, context } = validated; + const { q, context } = await validator(PostSearchReq, req.body); + const searchService = req.scope.resolve("searchService"); const loggedInUser = req.scope.resolve("loggedInUser"); - const searchApproach = req.scope.resolve("askRetrieveReadApproache"); - // Add Organization Id And User ID As Required Fields - console.log("context", context); - const results = await searchApproach.run( - loggedInUser.organisation_id.toLowerCase().substring(4), - q, - (context as any) ?? {} + const searchResults = await searchService.searchDocuments(null, q, context); + const sources = new Set( + searchResults.map( + (hit) => hit.documentMetadata?.source as AppNameDefinitions + ) ); - return res.status(200).send(results); + return res.status(200).send({ hits: searchResults, sources: [...sources] }); } catch (_error: unknown) { console.log(_error); - return res.status(500).send(`Error: Failed to execute SearchApproach.`); + return res.status(500).send(`Error: Failed to Search .`); } }; @@ -46,18 +41,6 @@ class PostSearchReq { } class SearchContextReq { - @IsOptional() - @IsBoolean() - ai_completion?: boolean; - - @IsOptional() - @IsString() - prompt_template?: string; - - @IsOptional() - @IsBoolean() - suggest_followup_questions?: boolean; - @IsOptional() @IsNumber() top?: number; diff --git a/packages/ocular/src/approaches/ask-retrieve-read.ts b/packages/ocular/src/approaches/ask-retrieve-read.ts index c1b72f0d..615a930f 100644 --- a/packages/ocular/src/approaches/ask-retrieve-read.ts +++ b/packages/ocular/src/approaches/ask-retrieve-read.ts @@ -60,53 +60,44 @@ export default class AskRetrieveThenRead implements ISearchApproach { if (context?.ai_completion) { context.retrieve_chunks = true; } - let searchResults: SearchResults = await this.searchService_.search( + let chunks = await this.searchService_.searchChunks( null, userQuery, context ); let message = null; - if (context && context.ai_completion) { - // Initial System Message - // const prompt = context?.prompt_template || SYSTEM_CHAT_TEMPLATE; - - const prompt = SYSTEM_CHAT_TEMPLATE; - const tokens = this.openai_.getChatModelTokenCount(prompt); - const messageBuilder = new MessageBuilder(prompt, tokens); - - // Use Top 3 Sources To Generate AI Completion. - // TODO: Use More Sophisticated Logic To Select Sources. - const sources = searchResults.chat_completion.citations - .map((c) => c.content) - .join(""); - const userContent = `${userQuery}\nSources:\n${sources}`; - - const roleTokens: number = this.openai_.getChatModelTokenCount("user"); - const messageTokens: number = - this.openai_.getChatModelTokenCount(userContent); - messageBuilder.appendMessage( - "user", - userContent, - 1, - roleTokens + messageTokens - ); - - // Add shots/samples. This helps model to mimic response and make sure they match rules laid out in system message. - // messageBuilder.appendMessage('assistant', QUESTION); - // messageBuilder.appendMessage('user', ANSWER) - const messages = messageBuilder.messages; - const chatCompletion = await this.openai_.completeChat(messages); - searchResults.chat_completion.content = chatCompletion; - } - - // Add Sources To The Search Results - const sources = new Set( - searchResults.hits.map( - (hit) => hit.documentMetadata?.source as AppNameDefinitions - ) + // Initial System Message + // const prompt = context?.prompt_template || SYSTEM_CHAT_TEMPLATE; + + const prompt = SYSTEM_CHAT_TEMPLATE; + const tokens = this.openai_.getChatModelTokenCount(prompt); + const messageBuilder = new MessageBuilder(prompt, tokens); + + // Use Top 3 Sources To Generate AI Completion. + // TODO: Use More Sophisticated Logic To Select Sources. + const sources = chunks.map((c) => c.content).join(""); + const userContent = `${userQuery}\nSources:\n${sources}`; + + const roleTokens: number = this.openai_.getChatModelTokenCount("user"); + const messageTokens: number = + this.openai_.getChatModelTokenCount(userContent); + messageBuilder.appendMessage( + "user", + userContent, + 1, + roleTokens + messageTokens ); - searchResults.sources = [...sources]; + + // Add shots/samples. This helps model to mimic response and make sure they match rules laid out in system message. + // messageBuilder.appendMessage('assistant', QUESTION); + // messageBuilder.appendMessage('user', ANSWER) + const messages = messageBuilder.messages; + const chatCompletion = await this.openai_.completeChat(messages); + let searchResults: SearchResults = { + chat_completion: { content: chatCompletion, citations: chunks }, + }; + return searchResults; } @@ -114,61 +105,51 @@ export default class AskRetrieveThenRead implements ISearchApproach { userQuery: string, context?: SearchContext ): AsyncGenerator { - if (context?.ai_completion) { - context.retrieve_chunks = true; - } - let searchResults: SearchResults = await this.searchService_.search( + let chunks = await this.searchService_.searchChunks( null, userQuery, context ); let message = null; - if (context && context.ai_completion) { - // Initial System Message - // const prompt = context?.prompt_template || SYSTEM_CHAT_TEMPLATE; - - const prompt = SYSTEM_CHAT_TEMPLATE; - const tokens = this.openai_.getChatModelTokenCount(prompt); - const messageBuilder = new MessageBuilder(prompt, tokens); - - // Use Top 3 Sources To Generate AI Completion. - // TODO: Use More Sophisticated Logic To Select Sources. - const sources = searchResults.chat_completion.citations - .map((c) => c.content) - .join(""); - const userContent = `${userQuery}\nSources:\n${sources}`; - - const roleTokens: number = this.openai_.getChatModelTokenCount("user"); - const messageTokens: number = - this.openai_.getChatModelTokenCount(userContent); - messageBuilder.appendMessage( - "user", - userContent, - 1, - roleTokens + messageTokens - ); - - // Add shots/samples. This helps model to mimic response and make sure they match rules laid out in system message. - // messageBuilder.appendMessage('assistant', QUESTION); - // messageBuilder.appendMessage('user', ANSWER) - const messages = messageBuilder.messages; - const chatCompletion = await this.openai_.completeChat(messages); - // Add Sources To The Search Results - - for await (const chunk of chatCompletion) { - searchResults.chat_completion.content = chunk; - - yield searchResults; - } - searchResults.chat_completion.content = chatCompletion; - } + // Initial System Message + // const prompt = context?.prompt_template || SYSTEM_CHAT_TEMPLATE; + + const prompt = SYSTEM_CHAT_TEMPLATE; + const tokens = this.openai_.getChatModelTokenCount(prompt); + const messageBuilder = new MessageBuilder(prompt, tokens); + + // Use Top 3 Sources To Generate AI Completion. + // TODO: Use More Sophisticated Logic To Select Sources. + const sources = chunks.map((c) => c.content).join(""); + const userContent = `${userQuery}\nSources:\n${sources}`; + + const roleTokens: number = this.openai_.getChatModelTokenCount("user"); + const messageTokens: number = + this.openai_.getChatModelTokenCount(userContent); + messageBuilder.appendMessage( + "user", + userContent, + 1, + roleTokens + messageTokens + ); - // const sources = new Set( - // searchResults.hits.map( - // (hit) => hit.documentMetadata?.source as AppNameDefinitions - // ) - // ); - // searchResults.sources = [...sources]; + // Add shots/samples. This helps model to mimic response and make sure they match rules laid out in system message. + // messageBuilder.appendMessage('assistant', QUESTION); + // messageBuilder.appendMessage('user', ANSWER) + const messages = messageBuilder.messages; + const chatCompletion = await this.openai_.completeChatWithStreaming( + messages + ); + let searchResults: SearchResults = { + chat_completion: { + content: "", + citations: chunks, + }, + }; + for await (const chunk of chatCompletion) { + searchResults.chat_completion.content = chunk; + yield searchResults; + } } } diff --git a/packages/ocular/src/approaches/chat-retrieve-read-retrieve.ts b/packages/ocular/src/approaches/chat-retrieve-read-retrieve.ts index 03e16b15..3f02a5ea 100644 --- a/packages/ocular/src/approaches/chat-retrieve-read-retrieve.ts +++ b/packages/ocular/src/approaches/chat-retrieve-read-retrieve.ts @@ -169,7 +169,8 @@ export default class ChatReadRetrieveRead implements IChatApproach { // STEP 2: Retrieve relevant documents from the search index with the GPT optimized query // ----------------------------------------------------------------------- - let hits = await this.searchService_.search(null, queryText, context); + // let hits = await this.searchService_.search(null, queryText, context); + let hits: SearchResults = { hits: [] }; // hits = hits.filter((doc) => doc !== null); // hits diff --git a/packages/ocular/src/services/search.ts b/packages/ocular/src/services/search.ts index d4558b3c..050edc20 100644 --- a/packages/ocular/src/services/search.ts +++ b/packages/ocular/src/services/search.ts @@ -9,6 +9,7 @@ import { SearchResults, SearchChunk, IEmbedderInterface, + SearchDocument, } from "@ocular/types"; import { AzureOpenAIOptions, @@ -47,11 +48,11 @@ class SearchService extends AbstractSearchService { this.embedderService_ = container.embedderService; } - async search( + async searchDocuments( indexName?: string, query?: string, context?: SearchContext - ): Promise { + ): Promise { indexName = indexName ? indexName : this.defaultIndexName_; // Compute Embeddings For The Query @@ -72,23 +73,32 @@ class SearchService extends AbstractSearchService { const docMetadata = await this.documentMetadataService_.list(docIds); // Join The Document Metadata With The Search Results - searchResults.hits = searchResults.hits.map((hit) => { + const hits = searchResults.hits.map((hit) => { const metadata = docMetadata.find((doc) => doc.id === hit.documentId); return { ...hit, documentMetadata: { ...metadata } }; }); + return hits; + } + + async searchChunks( + indexName?: string, + query?: string, + context?: SearchContext + ): Promise { + indexName = indexName ? indexName : this.defaultIndexName_; + + // Compute Embeddings For The Query + const queryVector = await this.embedderService_.createEmbeddings([query!]); + + // Search the index for the query vector + const chunks: SearchChunk[] = + await this.vectorDBService_.searchDocumentChunks( + indexName, + queryVector[0], + context + ); - // Retrieve Chunks Top Indipendent Chunks Irrespective of the Document - if (context.retrieve_chunks) { - searchResults.chat_completion = {}; - searchResults.chat_completion.citations = - await this.vectorDBService_.searchDocumentChunks( - indexName, - queryVector[0], - context - ); - } - console.log("Search Results", searchResults); - return searchResults; + return chunks; } } diff --git a/packages/plugins/azure-open-ai/src/__tests__/open-ai b/packages/plugins/azure-open-ai/src/__tests__/open-ai index 1b636e52..f9902d3f 100644 --- a/packages/plugins/azure-open-ai/src/__tests__/open-ai +++ b/packages/plugins/azure-open-ai/src/__tests__/open-ai @@ -1,9 +1,5 @@ import { OpenAI } from "openai"; import OpenAIService from "../services/open-ai"; -import dotenv from "dotenv"; -dotenv.config({ path: "../../ocular/.env.dev" }); - -console.log(process.env.AZURE_OPEN_AI_KEY); // This should print your key describe("OpenAIService", () => { let service; diff --git a/packages/plugins/open-ai/src/services/open-ai.ts b/packages/plugins/open-ai/src/services/open-ai.ts index 0b122a91..9be8856b 100644 --- a/packages/plugins/open-ai/src/services/open-ai.ts +++ b/packages/plugins/open-ai/src/services/open-ai.ts @@ -87,9 +87,10 @@ export default class OpenAIService extends AbstractLLMService { max_tokens: 1024, n: 1, }); - + let content = ""; for await (const chunk of result) { - yield chunk.choices[0]?.delta.content ?? ""; + content += chunk.choices[0]?.delta.content ?? ""; + yield content; } } catch (error) { console.log("Azure Open AI: Error", error); diff --git a/packages/types/src/common/search.ts b/packages/types/src/common/search.ts index 17375a6d..3a8a4ad2 100644 --- a/packages/types/src/common/search.ts +++ b/packages/types/src/common/search.ts @@ -46,7 +46,7 @@ export interface SearchResults { content?: string; citations?: SearchChunk[]; }; - hits: SearchDocument[]; + hits?: SearchDocument[]; sources?: AppNameDefinitions[]; } diff --git a/packages/types/src/interfaces/search-interface.ts b/packages/types/src/interfaces/search-interface.ts index ecfe3224..1765ae54 100644 --- a/packages/types/src/interfaces/search-interface.ts +++ b/packages/types/src/interfaces/search-interface.ts @@ -1,15 +1,22 @@ import { IndexableDocChunk, Message, + SearchChunk, SearchContext, + SearchDocument, SearchResults, } from "../common"; export interface ISearchService { - search( + searchDocuments( indexName: string, query: string, context?: SearchContext - ): Promise; + ): Promise; + searchChunks( + indexName: string, + query: string, + context?: SearchContext + ): Promise; } export abstract class AbstractSearchService implements ISearchService { @@ -29,9 +36,14 @@ export abstract class AbstractSearchService implements ISearchService { protected constructor(container, options) { this.options_ = options; } - abstract search( + abstract searchDocuments( + indexName: string, + query: string, + context?: SearchContext + ): Promise; + abstract searchChunks( indexName: string, query: string, context?: SearchContext - ): Promise; + ): Promise; } From 6b257597c6c8d6dea8e1291b75699ff4786eef34 Mon Sep 17 00:00:00 2001 From: Louis Murerwa Date: Wed, 22 May 2024 16:15:28 -0700 Subject: [PATCH 4/6] Ocular CoPilot Streaming --- packages/ocular-ui/lib/stream.ts | 46 +++++++++++++++++ .../pages/dashboard/search/results/index.tsx | 30 +++++++++-- packages/ocular-ui/services/api.js | 19 ++++++- .../ocular-ui/services/streaming-request.ts | 50 +++++++++++++++++++ .../src/api/routes/member/search/ask.ts | 1 + 5 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 packages/ocular-ui/lib/stream.ts create mode 100644 packages/ocular-ui/services/streaming-request.ts diff --git a/packages/ocular-ui/lib/stream.ts b/packages/ocular-ui/lib/stream.ts new file mode 100644 index 00000000..ce98b4f2 --- /dev/null +++ b/packages/ocular-ui/lib/stream.ts @@ -0,0 +1,46 @@ +class NdJsonParserStream extends TransformStream { + private buffer: string = ''; + constructor() { + let controller: TransformStreamDefaultController; + super({ + start: (_controller) => { + controller = _controller; + }, + transform: (chunk) => { + const jsonChunks = chunk.split('\n').filter(Boolean); + for (const jsonChunk of jsonChunks) { + try { + this.buffer += jsonChunk; + controller.enqueue(JSON.parse(this.buffer)); + this.buffer = ''; + } catch { + // Invalid JSON, wait for next chunk + } + } + }, + }); + } +} + +export function createReader(responseBody: ReadableStream | null) { + return responseBody + ?.pipeThrough(new TextDecoderStream()) + .pipeThrough(new NdJsonParserStream()) + .getReader(); +} + +export async function* readStream(reader: any): AsyncGenerator { + if (!reader) { + throw new Error('No response body or body is not readable'); + } + + let value: JSON | undefined; + let done: boolean; + while ((({ value, done } = await reader.read()), !done)) { + yield new Promise((resolve) => { + setTimeout(() => { + resolve(value as T); + }, 1000); + }); + } +} diff --git a/packages/ocular-ui/pages/dashboard/search/results/index.tsx b/packages/ocular-ui/pages/dashboard/search/results/index.tsx index 5f54c8d5..e375c5d7 100644 --- a/packages/ocular-ui/pages/dashboard/search/results/index.tsx +++ b/packages/ocular-ui/pages/dashboard/search/results/index.tsx @@ -6,6 +6,7 @@ import Header from "@/components/search/header"; import { useRouter } from "next/router"; import SearchResults from "@/components/search/search-results"; import { ApplicationContext } from "@/context/context" +import { createReader,readStream } from '@/lib/stream'; // Importing API End Points import api from "@/services/api" @@ -34,19 +35,38 @@ const selectedDate = useMemo(() => { useEffect(() => { setIsLoadingResults(true); setIsLoadingCopilot(true); + // Search api.search.search(router.query.q, selectedResultSources, selectedDate) .then(data => { - setAiResults(data.data.chat_completion.content); - setai_citations(data.data.chat_completion.citations); - setIsLoadingCopilot(false); setSearchResults(data.data.hits); setResultSources(data.data.sources); setIsLoadingResults(false); }) .catch(error => { - setIsLoadingResults(false); - setIsLoadingCopilot(false); + setIsLoadingResults(false); }); + // Copilot + const stream = true; + api.search.ask(router.query.q, selectedResultSources, selectedDate, stream) + .then(async response => { + if(stream){ + const reader = createReader(response.body); + const chunks = readStream(reader); + for await (const chunk of chunks) { + setAiResults(chunk.chat_completion.content); + setai_citations(chunk.chat_completion.citations); + setIsLoadingCopilot(false); + } + } else { + setAiResults(response.data.chat_completion.content); + setai_citations(response.data.chat_completion.citations); + setIsLoadingCopilot(false); + } + }) + .catch(error => { + console.error(error); + setIsLoadingCopilot(false); + }); }, [router.query.q, selectedResultSources, setResultSources, selectedDate]); return ( diff --git a/packages/ocular-ui/services/api.js b/packages/ocular-ui/services/api.js index 1b64f062..ee88a691 100644 --- a/packages/ocular-ui/services/api.js +++ b/packages/ocular-ui/services/api.js @@ -1,4 +1,5 @@ import ocularRequest from './request'; +import ocularStreamingRequest from './streaming-request'; export default { apps: { @@ -90,12 +91,28 @@ export default { }, }, search: { + ask(q, sources, date,stream=false) { + const path = `/ask`; + const body = { + context: { + top: 5, + stream:stream + }, + q: q + }; + if (date.from || date.to) { + body.context.date = date; + } + if (sources && sources.length > 0) { + body.context.sources = sources; + } + return ocularStreamingRequest("POST", path, body,stream); + }, search(q, sources, date) { const path = `/search`; const body = { context: { top: 20, - ai_completion: true // date: date, }, q: q diff --git a/packages/ocular-ui/services/streaming-request.ts b/packages/ocular-ui/services/streaming-request.ts new file mode 100644 index 00000000..996d5ba7 --- /dev/null +++ b/packages/ocular-ui/services/streaming-request.ts @@ -0,0 +1,50 @@ +// Ocular uses Axios on all its services to make HTTP requests. However Axios does support client streaming in the +//browser so we created this stream api as a workaround. + +// The streaming api is a simple wrapper around the fetch api that allows you to make streaming requests to the server. +export const OCULAR_BACKEND_URL = + process.env.OCULAR_BACKEND_URL || 'http://localhost:9000/v1'; + +export default async function ocularStreamingRequest( + method: + | 'GET' + | 'POST' + | 'PUT' + | 'DELETE' + | 'PATCH' + | 'OPTIONS' + | 'HEAD' + | 'CONNECT' + | 'TRACE', + path = '', + payload = {}, + stream = false, + cancelTokenSource: AbortController | null +) { + const headers = { + 'Access-Control-Allow-Origin': '*', + 'Content-Type': 'application/json', + }; + + const options = { + method, + headers, + credentials: 'include', + body: JSON.stringify(payload), + signal: cancelTokenSource ? cancelTokenSource.signal : undefined, + }; + + if (method === 'GET') { + delete options.body; + } + + const response = await fetch(`${OCULAR_BACKEND_URL}${path}`, options); + + if (!response.ok) { + const error = await response.text(); + console.error('Error', error); + throw new Error(error); + } + + return response; +} diff --git a/packages/ocular/src/api/routes/member/search/ask.ts b/packages/ocular/src/api/routes/member/search/ask.ts index fd6b1954..34ba3969 100644 --- a/packages/ocular/src/api/routes/member/search/ask.ts +++ b/packages/ocular/src/api/routes/member/search/ask.ts @@ -14,6 +14,7 @@ import { Type } from "class-transformer"; export default async (req, res) => { try { + console.log("Search API: Ask Request Received", req.body); const { q, context } = await validator(PostAskReq, req.body); const searchApproach = req.scope.resolve("askRetrieveReadApproache"); const loggedInUser = req.scope.resolve("loggedInUser"); From ccb86b0ad35146b7e2eb72585e72a529349fbe6f Mon Sep 17 00:00:00 2001 From: Louis Murerwa Date: Wed, 22 May 2024 21:03:03 -0700 Subject: [PATCH 5/6] Add Streaming Functionality --- packages/ocular-ui/lib/stream.ts | 2 +- packages/ocular-ui/pages/dashboard/search/results/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ocular-ui/lib/stream.ts b/packages/ocular-ui/lib/stream.ts index ce98b4f2..01010d9c 100644 --- a/packages/ocular-ui/lib/stream.ts +++ b/packages/ocular-ui/lib/stream.ts @@ -40,7 +40,7 @@ export async function* readStream(reader: any): AsyncGenerator { yield new Promise((resolve) => { setTimeout(() => { resolve(value as T); - }, 1000); + }, 50); }); } } diff --git a/packages/ocular-ui/pages/dashboard/search/results/index.tsx b/packages/ocular-ui/pages/dashboard/search/results/index.tsx index e375c5d7..5b08627a 100644 --- a/packages/ocular-ui/pages/dashboard/search/results/index.tsx +++ b/packages/ocular-ui/pages/dashboard/search/results/index.tsx @@ -49,13 +49,13 @@ const selectedDate = useMemo(() => { const stream = true; api.search.ask(router.query.q, selectedResultSources, selectedDate, stream) .then(async response => { + setIsLoadingCopilot(false); if(stream){ const reader = createReader(response.body); const chunks = readStream(reader); for await (const chunk of chunks) { setAiResults(chunk.chat_completion.content); setai_citations(chunk.chat_completion.citations); - setIsLoadingCopilot(false); } } else { setAiResults(response.data.chat_completion.content); From d98e15e24537c92bf42453709c18694c4796099a Mon Sep 17 00:00:00 2001 From: Louis Murerwa Date: Wed, 22 May 2024 21:56:27 -0700 Subject: [PATCH 6/6] Add Azure Open AI --- .../ocular-ui/services/streaming-request.ts | 4 +- packages/ocular/core-config.js | 46 +++++++++---------- .../azure-open-ai/src/services/open-ai.ts | 36 +++++++-------- 3 files changed, 41 insertions(+), 45 deletions(-) diff --git a/packages/ocular-ui/services/streaming-request.ts b/packages/ocular-ui/services/streaming-request.ts index 996d5ba7..c04c1a52 100644 --- a/packages/ocular-ui/services/streaming-request.ts +++ b/packages/ocular-ui/services/streaming-request.ts @@ -1,7 +1,5 @@ // Ocular uses Axios on all its services to make HTTP requests. However Axios does support client streaming in the -//browser so we created this stream api as a workaround. - -// The streaming api is a simple wrapper around the fetch api that allows you to make streaming requests to the server. +// browser so we created this stream api as a workaround. The streaming api is a simple wrapper around the fetch api that allows you to make streaming requests to the server. export const OCULAR_BACKEND_URL = process.env.OCULAR_BACKEND_URL || 'http://localhost:9000/v1'; diff --git a/packages/ocular/core-config.js b/packages/ocular/core-config.js index 5b340799..7f3451a1 100644 --- a/packages/ocular/core-config.js +++ b/packages/ocular/core-config.js @@ -180,35 +180,35 @@ module.exports = { chunk_over_lap: 0, }, }, - // { - // resolve: PluginNameDefinitions.AZUREOPENAI, - // options: { - // open_ai_key: process.env.AZURE_OPEN_AI_KEY, - // open_ai_version: "2023-05-15", - // endpoint: process.env.AZURE_OPEN_AI_ENDPOINT, - // embedding_deployment_name: - // process.env.AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME, - // embedding_model: process.env.AZURE_OPEN_AI_EMBEDDING_MODEL, - // chat_deployment_name: process.env.AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME, - // chat_model: process.env.AZURE_OPEN_AI_CHAT_MODEL, - // rate_limiter_opts: { - // requests: 1, // Number of Tokens - // interval: 1, // Interval in Seconds - // }, - // }, - // }, { - resolve: PluginNameDefinitions.OPENAI, + resolve: PluginNameDefinitions.AZUREOPENAI, options: { - open_ai_key: process.env.OPEN_AI_KEY, - embedding_model: process.env.OPEN_AI_EMBEDDING_MODEL, - chat_model: process.env.OPEN_AI_CHAT_MODEL, + open_ai_key: process.env.AZURE_OPEN_AI_KEY, + open_ai_version: "2023-05-15", + endpoint: process.env.AZURE_OPEN_AI_ENDPOINT, + embedding_deployment_name: + process.env.AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME, + embedding_model: process.env.AZURE_OPEN_AI_EMBEDDING_MODEL, + chat_deployment_name: process.env.AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME, + chat_model: process.env.AZURE_OPEN_AI_CHAT_MODEL, rate_limiter_opts: { - requests: 1000000, // Number of Tokens - interval: 60, // Interval in Seconds + requests: 1, // Number of Tokens + interval: 1, // Interval in Seconds }, }, }, + // { + // resolve: PluginNameDefinitions.OPENAI, + // options: { + // open_ai_key: process.env.OPEN_AI_KEY, + // embedding_model: process.env.OPEN_AI_EMBEDDING_MODEL, + // chat_model: process.env.OPEN_AI_CHAT_MODEL, + // rate_limiter_opts: { + // requests: 1000000, // Number of Tokens + // interval: 60, // Interval in Seconds + // }, + // }, + // }, { resolve: `qdrant-vector-search-service`, options: { diff --git a/packages/plugins/azure-open-ai/src/services/open-ai.ts b/packages/plugins/azure-open-ai/src/services/open-ai.ts index 8900231f..c3f94474 100644 --- a/packages/plugins/azure-open-ai/src/services/open-ai.ts +++ b/packages/plugins/azure-open-ai/src/services/open-ai.ts @@ -112,28 +112,26 @@ export default class AzureOpenAIService extends AbstractLLMService { } } - completeChatWithStreaming( + async *completeChatWithStreaming( messages: Message[] ): AsyncGenerator { - const generator = async function* (messages: Message[]) { - try { - const result = await this.chatClient_.chat.completions.create({ - stream: true, - model: this.chatModel_, - messages, - temperature: 0.3, - max_tokens: 1024, - n: 1, - }); - - for await (const chunk of result) { - yield chunk.choices[0]?.delta.content ?? ""; - } - } catch (error) { - console.log("Azure Open AI: Error", error); + try { + const result = await this.chatClient_.chat.completions.create({ + model: this.chatModel_, + stream: true, + messages, + temperature: 0.3, + max_tokens: 1024, + n: 1, + }); + let content = ""; + for await (const chunk of result) { + content += chunk.choices[0]?.delta.content ?? ""; + yield content; } - }; - return generator(messages); + } catch (error) { + console.log("Azure Open AI: Error", error); + } } getChatModelTokenCount(content: string): number {