From 509b1e5c63e8108408846f610e2ce9ded7502cf4 Mon Sep 17 00:00:00 2001 From: Danny Avila <110412045+danny-avila@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:47:40 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=84=20refactor:=20Consolidate=20Ask/Ed?= =?UTF-8?q?it=20Controllers=20(#1365)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(Ask/Edit): consolidate ask/edit controllers between the main modules and openAI controllers to reduce repetition of code and increase reusability * fix(winston/logger): circular dependency issue * fix(config/scripts): fix script imports * refactor(indexSync): make not configured message an info log message * chore: create a rollup script for api/server/index.js to check circular dependencies * chore: bump @keyv/redis --- api/app/clients/AnthropicClient.js | 2 +- api/config/winston.js | 11 +- api/lib/db/indexSync.js | 2 + api/package.json | 2 +- api/server/controllers/AskController.js | 70 ++++---- api/server/controllers/EditController.js | 47 +++--- api/server/routes/ask/openAI.js | 151 +----------------- api/server/routes/edit/openAI.js | 141 +--------------- config/add-balance.js | 8 +- config/ban-user.js | 8 +- config/create-user.js | 8 +- config/loader.js | 4 +- config/packages.js | 4 +- config/stop-backend.js | 4 +- config/update.js | 2 +- config/upgrade.js | 2 +- package-lock.json | 134 +++++++++++++++- packages/data-provider/.gitignore | 3 +- packages/data-provider/package.json | 6 + .../data-provider/server-rollup.config.js | 54 +++++++ 20 files changed, 297 insertions(+), 366 deletions(-) create mode 100644 packages/data-provider/server-rollup.config.js diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 25c642412ab..0441a49334e 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -290,7 +290,7 @@ class AnthropicClient extends BaseClient { top_p, top_k, }; - logger.debug('[AnthropicClient]', { requestOptions }); + logger.debug('[AnthropicClient]', { ...requestOptions }); const response = await client.completions.create(requestOptions); signal.addEventListener('abort', () => { diff --git a/api/config/winston.js b/api/config/winston.js index d689cee8bdd..8038b191066 100644 --- a/api/config/winston.js +++ b/api/config/winston.js @@ -2,7 +2,6 @@ const path = require('path'); const winston = require('winston'); require('winston-daily-rotate-file'); const { redact, deepObjectFormat } = require('./parsers'); -const { isEnabled } = require('~/server/utils/handleText'); const logDir = path.join(__dirname, '..', 'logs'); @@ -67,7 +66,10 @@ const transports = [ // ); // } -if (isEnabled && isEnabled(DEBUG_LOGGING)) { +if ( + (typeof DEBUG_LOGGING === 'string' && DEBUG_LOGGING?.toLowerCase() === 'true') || + DEBUG_LOGGING === true +) { transports.push( new winston.transports.DailyRotateFile({ level: 'debug', @@ -88,7 +90,10 @@ const consoleFormat = winston.format.combine( winston.format.printf((info) => `${info.timestamp} ${info.level}: ${info.message}`), ); -if (isEnabled && isEnabled(DEBUG_CONSOLE)) { +if ( + (typeof DEBUG_CONSOLE === 'string' && DEBUG_CONSOLE?.toLowerCase() === 'true') || + DEBUG_CONSOLE === true +) { transports.push( new winston.transports.Console({ level: 'debug', diff --git a/api/lib/db/indexSync.js b/api/lib/db/indexSync.js index 5413d7b6320..53ac3d3a266 100644 --- a/api/lib/db/indexSync.js +++ b/api/lib/db/indexSync.js @@ -65,6 +65,8 @@ async function indexSync(req, res, next) { logger.error('[indexSync] Trouble creating indices, try restarting the server.', err); } }, 750); + } else if (err.message.includes('Meilisearch not configured')) { + logger.info('[indexSync] Meilisearch not configured, search will be disabled.'); } else { logger.error('[indexSync] error', err); // res.status(500).json({ error: 'Server error' }); diff --git a/api/package.json b/api/package.json index f7d89afb9e3..4d09a96e156 100644 --- a/api/package.json +++ b/api/package.json @@ -30,7 +30,7 @@ "@anthropic-ai/sdk": "^0.5.4", "@azure/search-documents": "^12.0.0", "@keyv/mongo": "^2.1.8", - "@keyv/redis": "^2.8.0", + "@keyv/redis": "^2.8.1", "@langchain/google-genai": "^0.0.2", "axios": "^1.3.4", "bcryptjs": "^2.4.3", diff --git a/api/server/controllers/AskController.js b/api/server/controllers/AskController.js index 7ef25b93d1e..11be3afd301 100644 --- a/api/server/controllers/AskController.js +++ b/api/server/controllers/AskController.js @@ -4,7 +4,7 @@ const { saveMessage, getConvoTitle, getConvo } = require('~/models'); const { createAbortController, handleAbortError } = require('~/server/middleware'); const { logger } = require('~/config'); -const AskController = async (req, res, next, initializeClient) => { +const AskController = async (req, res, next, initializeClient, addTitle) => { let { text, endpointOption, @@ -12,7 +12,9 @@ const AskController = async (req, res, next, initializeClient) => { parentMessageId = null, overrideParentMessageId = null, } = req.body; + logger.debug('[AskController]', { text, conversationId, ...endpointOption }); + let metadata; let userMessage; let promptTokens; @@ -21,8 +23,11 @@ const AskController = async (req, res, next, initializeClient) => { let lastSavedTimestamp = 0; let saveDelay = 100; const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model }); + const newConvo = !conversationId; const user = req.user.id; + const addMetadata = (data) => (metadata = data); + const getReqData = (data = {}) => { for (let key in data) { if (key === 'userMessage') { @@ -50,6 +55,7 @@ const AskController = async (req, res, next, initializeClient) => { conversationId, parentMessageId: overrideParentMessageId ?? userMessageId, text: partialText, + model: endpointOption.modelOptions.model, unfinished: true, cancelled: false, error: false, @@ -62,46 +68,50 @@ const AskController = async (req, res, next, initializeClient) => { } }, }); - try { - const addMetadata = (data) => (metadata = data); - const getAbortData = () => ({ - conversationId, - messageId: responseMessageId, - sender, - parentMessageId: overrideParentMessageId ?? userMessageId, - text: getPartialText(), - userMessage, - promptTokens, - }); - const { abortController, onStart } = createAbortController(req, res, getAbortData); + const getAbortData = () => ({ + sender, + conversationId, + messageId: responseMessageId, + parentMessageId: overrideParentMessageId ?? userMessageId, + text: getPartialText(), + userMessage, + promptTokens, + }); - const { client } = await initializeClient({ req, res, endpointOption }); + const { abortController, onStart } = createAbortController(req, res, getAbortData); - let response = await client.sendMessage(text, { - // debug: true, + try { + const { client } = await initializeClient({ req, res, endpointOption }); + const messageOptions = { user, - conversationId, parentMessageId, + conversationId, overrideParentMessageId, - ...endpointOption, + getReqData, + onStart, + addMetadata, + abortController, onProgress: progressCallback.call(null, { res, text, - parentMessageId: overrideParentMessageId ?? userMessageId, + parentMessageId: overrideParentMessageId || userMessageId, }), - onStart, - getReqData, - addMetadata, - abortController, - }); + }; + + let response = await client.sendMessage(text, messageOptions); + + if (overrideParentMessageId) { + response.parentMessageId = overrideParentMessageId; + } if (metadata) { response = { ...response, ...metadata }; } - if (overrideParentMessageId) { - response.parentMessageId = overrideParentMessageId; + if (client.options.attachments) { + userMessage.files = client.options.attachments; + delete userMessage.image_urls; } sendMessage(res, { @@ -116,7 +126,13 @@ const AskController = async (req, res, next, initializeClient) => { await saveMessage({ ...response, user }); await saveMessage(userMessage); - // TODO: add title service + if (addTitle && parentMessageId === '00000000-0000-0000-0000-000000000000' && newConvo) { + addTitle(req, { + text, + response, + client, + }); + } } catch (error) { const partialText = getPartialText(); handleAbortError(res, req, error, { diff --git a/api/server/controllers/EditController.js b/api/server/controllers/EditController.js index 8537d78098f..023dc35a832 100644 --- a/api/server/controllers/EditController.js +++ b/api/server/controllers/EditController.js @@ -15,6 +15,7 @@ const EditController = async (req, res, next, initializeClient) => { parentMessageId = null, overrideParentMessageId = null, } = req.body; + logger.debug('[EditController]', { text, generation, @@ -22,6 +23,7 @@ const EditController = async (req, res, next, initializeClient) => { conversationId, ...endpointOption, }); + let metadata; let userMessage; let promptTokens; @@ -48,6 +50,7 @@ const EditController = async (req, res, next, initializeClient) => { generation, onProgress: ({ text: partialText }) => { const currentTimestamp = Date.now(); + if (currentTimestamp - lastSavedTimestamp > saveDelay) { lastSavedTimestamp = currentTimestamp; saveMessage({ @@ -56,6 +59,7 @@ const EditController = async (req, res, next, initializeClient) => { conversationId, parentMessageId: overrideParentMessageId ?? userMessageId, text: partialText, + model: endpointOption.modelOptions.model, unfinished: true, cancelled: false, isEdited: true, @@ -69,19 +73,20 @@ const EditController = async (req, res, next, initializeClient) => { } }, }); - try { - const getAbortData = () => ({ - conversationId, - messageId: responseMessageId, - sender, - parentMessageId: overrideParentMessageId ?? userMessageId, - text: getPartialText(), - userMessage, - promptTokens, - }); - const { abortController, onStart } = createAbortController(req, res, getAbortData); + const getAbortData = () => ({ + conversationId, + messageId: responseMessageId, + sender, + parentMessageId: overrideParentMessageId ?? userMessageId, + text: getPartialText(), + userMessage, + promptTokens, + }); + + const { abortController, onStart } = createAbortController(req, res, getAbortData); + try { const { client } = await initializeClient({ req, res, endpointOption }); let response = await client.sendMessage(text, { @@ -93,25 +98,22 @@ const EditController = async (req, res, next, initializeClient) => { parentMessageId, responseMessageId, overrideParentMessageId, - ...endpointOption, - onProgress: progressCallback.call(null, { - res, - text, - parentMessageId: overrideParentMessageId ?? userMessageId, - }), getReqData, onStart, addMetadata, abortController, + onProgress: progressCallback.call(null, { + res, + text, + parentMessageId: overrideParentMessageId || userMessageId, + }), }); if (metadata) { response = { ...response, ...metadata }; } - if (overrideParentMessageId) { - response.parentMessageId = overrideParentMessageId; - } + await saveMessage({ ...response, user }); sendMessage(res, { title: await getConvoTitle(user, conversationId), @@ -121,11 +123,6 @@ const EditController = async (req, res, next, initializeClient) => { responseMessage: response, }); res.end(); - - await saveMessage({ ...response, user }); - await saveMessage(userMessage); - - // TODO: add title service } catch (error) { const partialText = getPartialText(); handleAbortError(res, req, error, { diff --git a/api/server/routes/ask/openAI.js b/api/server/routes/ask/openAI.js index e91692d8a8e..180ee27f299 100644 --- a/api/server/routes/ask/openAI.js +++ b/api/server/routes/ask/openAI.js @@ -1,160 +1,19 @@ const express = require('express'); -const router = express.Router(); -const { sendMessage, createOnProgress } = require('~/server/utils'); -const { saveMessage, getConvoTitle, getConvo } = require('~/models'); -const { getResponseSender } = require('librechat-data-provider'); +const AskController = require('~/server/controllers/AskController'); const { addTitle, initializeClient } = require('~/server/services/Endpoints/openAI'); const { handleAbort, - createAbortController, - handleAbortError, setHeaders, validateEndpoint, buildEndpointOption, } = require('~/server/middleware'); -const { logger } = require('~/config'); - -router.post('/abort', handleAbort()); - -router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res) => { - let { - text, - endpointOption, - conversationId, - parentMessageId = null, - overrideParentMessageId = null, - } = req.body; - - logger.debug('[/ask/openAI]', { text, conversationId, ...endpointOption }); - - let metadata; - let userMessage; - let promptTokens; - let userMessageId; - let responseMessageId; - let lastSavedTimestamp = 0; - let saveDelay = 100; - const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model }); - const newConvo = !conversationId; - const user = req.user.id; - - const addMetadata = (data) => (metadata = data); - - const getReqData = (data = {}) => { - for (let key in data) { - if (key === 'userMessage') { - userMessage = data[key]; - userMessageId = data[key].messageId; - } else if (key === 'responseMessageId') { - responseMessageId = data[key]; - } else if (key === 'promptTokens') { - promptTokens = data[key]; - } else if (!conversationId && key === 'conversationId') { - conversationId = data[key]; - } - } - }; - - const { onProgress: progressCallback, getPartialText } = createOnProgress({ - onProgress: ({ text: partialText }) => { - const currentTimestamp = Date.now(); - - if (currentTimestamp - lastSavedTimestamp > saveDelay) { - lastSavedTimestamp = currentTimestamp; - saveMessage({ - messageId: responseMessageId, - sender, - conversationId, - parentMessageId: overrideParentMessageId ?? userMessageId, - text: partialText, - model: endpointOption.modelOptions.model, - unfinished: true, - cancelled: false, - error: false, - user, - }); - } - if (saveDelay < 500) { - saveDelay = 500; - } - }, - }); - - const getAbortData = () => ({ - sender, - conversationId, - messageId: responseMessageId, - parentMessageId: overrideParentMessageId ?? userMessageId, - text: getPartialText(), - userMessage, - promptTokens, - }); - - const { abortController, onStart } = createAbortController(req, res, getAbortData); - - try { - const { client } = await initializeClient({ req, res, endpointOption }); - const messageOptions = { - user, - parentMessageId, - conversationId, - overrideParentMessageId, - getReqData, - onStart, - addMetadata, - abortController, - onProgress: progressCallback.call(null, { - res, - text, - parentMessageId: overrideParentMessageId || userMessageId, - }), - }; - - let response = await client.sendMessage(text, messageOptions); - - if (overrideParentMessageId) { - response.parentMessageId = overrideParentMessageId; - } - - if (metadata) { - response = { ...response, ...metadata }; - } - - if (client.options.attachments) { - userMessage.files = client.options.attachments; - delete userMessage.image_urls; - } - - sendMessage(res, { - title: await getConvoTitle(user, conversationId), - final: true, - conversation: await getConvo(user, conversationId), - requestMessage: userMessage, - responseMessage: response, - }); - res.end(); +const router = express.Router(); - await saveMessage({ ...response, user }); - await saveMessage(userMessage); +router.post('/abort', handleAbort()); - if (parentMessageId === '00000000-0000-0000-0000-000000000000' && newConvo) { - addTitle(req, { - text, - response, - client, - }); - } - } catch (error) { - const partialText = getPartialText(); - handleAbortError(res, req, error, { - partialText, - conversationId, - sender, - messageId: responseMessageId, - parentMessageId: userMessageId ?? parentMessageId, - }); - } +router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res, next) => { + await AskController(req, res, next, initializeClient, addTitle); }); module.exports = router; diff --git a/api/server/routes/edit/openAI.js b/api/server/routes/edit/openAI.js index ec0b62330d9..47f36d6cb4d 100644 --- a/api/server/routes/edit/openAI.js +++ b/api/server/routes/edit/openAI.js @@ -1,150 +1,19 @@ const express = require('express'); -const router = express.Router(); -const { getResponseSender } = require('librechat-data-provider'); +const EditController = require('~/server/controllers/EditController'); const { initializeClient } = require('~/server/services/Endpoints/openAI'); -const { saveMessage, getConvoTitle, getConvo } = require('~/models'); -const { sendMessage, createOnProgress } = require('~/server/utils'); const { handleAbort, - createAbortController, - handleAbortError, setHeaders, validateEndpoint, buildEndpointOption, } = require('~/server/middleware'); -const { logger } = require('~/config'); - -router.post('/abort', handleAbort()); - -router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res) => { - let { - text, - generation, - endpointOption, - conversationId, - responseMessageId, - isContinued = false, - parentMessageId = null, - overrideParentMessageId = null, - } = req.body; - - logger.debug('[/edit/openAI]', { - text, - generation, - isContinued, - conversationId, - ...endpointOption, - }); - - let metadata; - let userMessage; - let promptTokens; - let lastSavedTimestamp = 0; - let saveDelay = 100; - const sender = getResponseSender({ ...endpointOption, model: endpointOption.modelOptions.model }); - const userMessageId = parentMessageId; - const user = req.user.id; - - const addMetadata = (data) => (metadata = data); - const getReqData = (data = {}) => { - for (let key in data) { - if (key === 'userMessage') { - userMessage = data[key]; - } else if (key === 'responseMessageId') { - responseMessageId = data[key]; - } else if (key === 'promptTokens') { - promptTokens = data[key]; - } - } - }; - - const { onProgress: progressCallback, getPartialText } = createOnProgress({ - generation, - onProgress: ({ text: partialText }) => { - const currentTimestamp = Date.now(); - if (currentTimestamp - lastSavedTimestamp > saveDelay) { - lastSavedTimestamp = currentTimestamp; - saveMessage({ - messageId: responseMessageId, - sender, - conversationId, - parentMessageId: overrideParentMessageId || userMessageId, - text: partialText, - model: endpointOption.modelOptions.model, - unfinished: true, - cancelled: false, - isEdited: true, - error: false, - user, - }); - } - - if (saveDelay < 500) { - saveDelay = 500; - } - }, - }); - - const getAbortData = () => ({ - sender, - conversationId, - messageId: responseMessageId, - parentMessageId: overrideParentMessageId ?? userMessageId, - text: getPartialText(), - userMessage, - promptTokens, - }); - - const { abortController, onStart } = createAbortController(req, res, getAbortData); - - try { - const { client } = await initializeClient({ req, res, endpointOption }); - - let response = await client.sendMessage(text, { - user, - generation, - isContinued, - isEdited: true, - conversationId, - parentMessageId, - responseMessageId, - overrideParentMessageId, - getReqData, - onStart, - addMetadata, - abortController, - onProgress: progressCallback.call(null, { - res, - text, - parentMessageId: overrideParentMessageId || userMessageId, - }), - }); - - if (metadata) { - response = { ...response, ...metadata }; - } +const router = express.Router(); - await saveMessage({ ...response, user }); +router.post('/abort', handleAbort()); - sendMessage(res, { - title: await getConvoTitle(user, conversationId), - final: true, - conversation: await getConvo(user, conversationId), - requestMessage: userMessage, - responseMessage: response, - }); - res.end(); - } catch (error) { - const partialText = getPartialText(); - handleAbortError(res, req, error, { - partialText, - conversationId, - sender, - messageId: responseMessageId, - parentMessageId: userMessageId ?? parentMessageId, - }); - } +router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res, next) => { + await EditController(req, res, next, initializeClient); }); module.exports = router; diff --git a/config/add-balance.js b/config/add-balance.js index ed2e6128983..3aa786c16e9 100644 --- a/config/add-balance.js +++ b/config/add-balance.js @@ -1,7 +1,9 @@ -const connectDb = require('../api/lib/db/connectDb'); +const path = require('path'); +require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); const { askQuestion, silentExit } = require('./helpers'); -const User = require('../api/models/User'); -const Transaction = require('../api/models/Transaction'); +const Transaction = require('~/models/Transaction'); +const connectDb = require('~/lib/db/connectDb'); +const User = require('~/models/User'); (async () => { /** diff --git a/config/ban-user.js b/config/ban-user.js index 45e628aeca4..6db29e8159a 100644 --- a/config/ban-user.js +++ b/config/ban-user.js @@ -1,7 +1,9 @@ -const connectDb = require('../api/lib/db/connectDb'); +const path = require('path'); +require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); const { askQuestion, silentExit } = require('./helpers'); -const banViolation = require('../api/cache/banViolation'); -const User = require('../api/models/User'); +const banViolation = require('~/cache/banViolation'); +const connectDb = require('~/lib/db/connectDb'); +const User = require('~/models/User'); (async () => { /** diff --git a/config/create-user.js b/config/create-user.js index c854b137a96..738460c8a3e 100644 --- a/config/create-user.js +++ b/config/create-user.js @@ -1,7 +1,9 @@ -const connectDb = require('../api/lib/db/connectDb'); -const { registerUser } = require('../api/server/services/AuthService'); +const path = require('path'); +require('module-alias')({ base: path.resolve(__dirname, '..', 'api') }); +const { registerUser } = require('~/server/services/AuthService'); const { askQuestion, silentExit } = require('./helpers'); -const User = require('../api/models/User'); +const connectDb = require('~/lib/db/connectDb'); +const User = require('~/models/User'); (async () => { /** diff --git a/config/loader.js b/config/loader.js index c0e3413528b..a29ca19a0a4 100644 --- a/config/loader.js +++ b/config/loader.js @@ -1,7 +1,7 @@ -const dotenv = require('dotenv'); -const path = require('path'); const fs = require('fs'); +const path = require('path'); const crypto = require('crypto'); +const dotenv = require('dotenv'); /** * This class is responsible for loading the environment variables diff --git a/config/packages.js b/config/packages.js index 7e4b0f011bc..0b457a7bfe8 100644 --- a/config/packages.js +++ b/config/packages.js @@ -1,6 +1,6 @@ -const { execSync } = require('child_process'); -const path = require('path'); const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); const { deleteNodeModules } = require('./helpers'); diff --git a/config/stop-backend.js b/config/stop-backend.js index e863a03eeec..8e8f23dbdc2 100644 --- a/config/stop-backend.js +++ b/config/stop-backend.js @@ -1,7 +1,5 @@ -// eslint-disable-next-line -const helpers = require('./helpers'); -const { exec } = require('child_process'); const { promisify } = require('util'); +const { exec } = require('child_process'); const isWindows = process.platform === 'win32'; const execAsync = promisify(exec); diff --git a/config/update.js b/config/update.js index 5a36326f23a..fd947eda393 100644 --- a/config/update.js +++ b/config/update.js @@ -1,5 +1,5 @@ -const { execSync } = require('child_process'); const path = require('path'); +const { execSync } = require('child_process'); const { askQuestion, isDockerRunning, deleteNodeModules, silentExit } = require('./helpers'); const config = { diff --git a/config/upgrade.js b/config/upgrade.js index d0477de50c0..c954cf0452a 100644 --- a/config/upgrade.js +++ b/config/upgrade.js @@ -1,8 +1,8 @@ /** * Upgrade script */ -const dotenv = require('dotenv'); const fs = require('fs'); +const dotenv = require('dotenv'); const { exit } = require('process'); // Suppress default warnings diff --git a/package-lock.json b/package-lock.json index 9b1db24c9b8..bdc835628ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,7 +44,7 @@ "@anthropic-ai/sdk": "^0.5.4", "@azure/search-documents": "^12.0.0", "@keyv/mongo": "^2.1.8", - "@keyv/redis": "^2.8.0", + "@keyv/redis": "^2.8.1", "@langchain/google-genai": "^0.0.2", "axios": "^1.3.4", "bcryptjs": "^2.4.3", @@ -2130,6 +2130,41 @@ "@babel/core": "^7.13.0" } }, + "node_modules/@babel/plugin-proposal-nullish-coalescing-operator": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-nullish-coalescing-operator/-/plugin-proposal-nullish-coalescing-operator-7.18.6.tgz", + "integrity": "sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==", + "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-nullish-coalescing-operator instead.", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-proposal-optional-chaining": { + "version": "7.21.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.21.0.tgz", + "integrity": "sha512-p4zeefM72gpmEe2fkUr/OnOXpWEf8nAgk7ZYVqqfFiyIG7oFfVZcCrU64hWn5xp4tQ9LkV4bTIa5rD0KANpKNA==", + "deprecated": "This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-chaining instead.", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.20.2", + "@babel/helper-skip-transparent-expression-wrappers": "^7.20.0", + "@babel/plugin-syntax-optional-chaining": "^7.8.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-proposal-private-property-in-object": { "version": "7.21.0-placeholder-for-preset-env.2", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.0-placeholder-for-preset-env.2.tgz", @@ -5464,9 +5499,9 @@ } }, "node_modules/@keyv/redis": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/@keyv/redis/-/redis-2.8.0.tgz", - "integrity": "sha512-6k7wG/KKSIGpruKlsEB4sFjECJEyQsuJbWoWdoq9Uv2L6Mm/SEqEidekRZI/QljE1A4WQkFsIE8hHl1Oc3UNGg==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/@keyv/redis/-/redis-2.8.1.tgz", + "integrity": "sha512-JjoNXtAcjT0r0CIWi69rVoXdwPD2nHqyWR80XHmsP1Psuzml1icNtehq6ZqKJjLLgLzH3DvQoWrXkmhxxerf3g==", "dependencies": { "ioredis": "^5.3.2" }, @@ -6898,6 +6933,64 @@ "node": ">=14.0.0" } }, + "node_modules/@rollup/plugin-alias": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-alias/-/plugin-alias-5.1.0.tgz", + "integrity": "sha512-lpA3RZ9PdIG7qqhEfv79tBffNaoDuukFDrmhLqg9ifv99u/ehn+lOg30x2zmhf8AQqQUZaMk/B9fZraQ6/acDQ==", + "dev": true, + "dependencies": { + "slash": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, + "node_modules/@rollup/plugin-alias/node_modules/slash": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-4.0.0.tgz", + "integrity": "sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@rollup/plugin-babel": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@rollup/plugin-babel/-/plugin-babel-6.0.4.tgz", + "integrity": "sha512-YF7Y52kFdFT/xVSuVdjkV5ZdX/3YtmX0QulG+x0taQOtJdHYzVU61aSSkAgVJ7NOv6qPkIYiJSgSWWN/DM5sGw==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.18.6", + "@rollup/pluginutils": "^5.0.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0", + "@types/babel__core": "^7.1.9", + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "@types/babel__core": { + "optional": true + }, + "rollup": { + "optional": true + } + } + }, "node_modules/@rollup/plugin-commonjs": { "version": "25.0.4", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-25.0.4.tgz", @@ -6957,6 +7050,26 @@ "node": ">=12" } }, + "node_modules/@rollup/plugin-json": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@rollup/plugin-json/-/plugin-json-6.1.0.tgz", + "integrity": "sha512-EGI2te5ENk1coGeADSIwZ7G2Q8CJS2sF120T7jLw4xFw9n7wIOXHo+kIYRAoVpJAN+kmqZSoO3Fp4JtoNF4ReA==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "^5.1.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, "node_modules/@rollup/plugin-node-resolve": { "version": "15.2.1", "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.2.1.tgz", @@ -7038,9 +7151,9 @@ } }, "node_modules/@rollup/pluginutils": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.4.tgz", - "integrity": "sha512-0KJnIoRI8A+a1dqOYLxH8vBf8bphDmty5QvIm2hqm7oFCFYKCAZWWd2hXgMibaPsNDhI0AtpYfQZJG47pt/k4g==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.0.tgz", + "integrity": "sha512-XTIWOPPcpvyKI6L1NHo0lFlCyznUEyPmPY1mc3KpPVDYulHSTvyeLNVW00QTLIAFNhR3kYnJTQHeGqU4M3n09g==", "dev": true, "dependencies": { "@types/estree": "^1.0.0", @@ -7051,7 +7164,7 @@ "node": ">=14.0.0" }, "peerDependencies": { - "rollup": "^1.20.0||^2.0.0||^3.0.0" + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" }, "peerDependenciesMeta": { "rollup": { @@ -25453,10 +25566,15 @@ "zod": "^3.22.4" }, "devDependencies": { + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.21.0", "@babel/preset-env": "^7.21.5", "@babel/preset-react": "^7.18.6", "@babel/preset-typescript": "^7.21.0", + "@rollup/plugin-alias": "^5.1.0", + "@rollup/plugin-babel": "^6.0.4", "@rollup/plugin-commonjs": "^25.0.2", + "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.1.0", "@rollup/plugin-replace": "^5.0.5", "@rollup/plugin-terser": "^0.4.4", diff --git a/packages/data-provider/.gitignore b/packages/data-provider/.gitignore index 40b878db5b1..7b961825b33 100644 --- a/packages/data-provider/.gitignore +++ b/packages/data-provider/.gitignore @@ -1 +1,2 @@ -node_modules/ \ No newline at end of file +node_modules/ +test_bundle/ diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index e0af1a0ced9..ab2f6af837d 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -21,6 +21,7 @@ "clean": "rimraf dist", "build": "npm run clean && rollup -c --silent --bundleConfigAsCjs", "build:watch": "rollup -c -w", + "rollup:api": "npx rollup -c server-rollup.config.js --bundleConfigAsCjs", "test": "jest --coverage --watch", "test:ci": "jest --coverage --ci", "verify": "npm run test:ci", @@ -43,10 +44,15 @@ "zod": "^3.22.4" }, "devDependencies": { + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.18.6", + "@babel/plugin-proposal-optional-chaining": "^7.21.0", "@babel/preset-env": "^7.21.5", "@babel/preset-react": "^7.18.6", "@babel/preset-typescript": "^7.21.0", + "@rollup/plugin-alias": "^5.1.0", + "@rollup/plugin-babel": "^6.0.4", "@rollup/plugin-commonjs": "^25.0.2", + "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.1.0", "@rollup/plugin-replace": "^5.0.5", "@rollup/plugin-terser": "^0.4.4", diff --git a/packages/data-provider/server-rollup.config.js b/packages/data-provider/server-rollup.config.js new file mode 100644 index 00000000000..e20b877fe13 --- /dev/null +++ b/packages/data-provider/server-rollup.config.js @@ -0,0 +1,54 @@ +import path from 'path'; +import resolve from '@rollup/plugin-node-resolve'; +import commonjs from '@rollup/plugin-commonjs'; +import alias from '@rollup/plugin-alias'; +import json from '@rollup/plugin-json'; +import babel from '@rollup/plugin-babel'; + +const rootPath = path.resolve(__dirname, '../../'); +const rootServerPath = path.resolve(__dirname, '../../api'); +const entryPath = path.resolve(rootPath, 'api/server/index.js'); + +console.log('entryPath', entryPath); + +// Define your custom aliases here +const customAliases = { + entries: [{ find: '~', replacement: rootServerPath }], +}; + +export default { + input: entryPath, + output: { + file: 'test_bundle/bundle.js', + format: 'cjs', + }, + plugins: [ + alias(customAliases), + resolve({ + preferBuiltins: true, + extensions: ['.js', '.json', '.node'], + }), + commonjs(), + json(), + babel({ + exclude: 'node_modules/**', + babelHelpers: 'bundled', + presets: [ + ['@babel/preset-env', { targets: { node: 'current' } }], + '@babel/preset-typescript', + ], + plugins: [ + '@babel/plugin-syntax-dynamic-import', + '@babel/plugin-proposal-nullish-coalescing-operator', + '@babel/plugin-proposal-optional-chaining', + ], + }), + ], + external: (id) => { + // More selective external function + if (/node_modules/.test(id)) { + return !id.startsWith('langchain/'); + } + return false; + }, +};