From aea07aa9a18f3a78077e9c97da250dfcf167c956 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 19 Feb 2024 15:05:56 -0500 Subject: [PATCH 01/35] wip: first pass for azure endpoint schema --- packages/data-provider/specs/azure.spec.ts | 155 +++++++++++++++++++++ packages/data-provider/src/azure.ts | 47 +++++++ packages/data-provider/src/config.ts | 27 ++++ 3 files changed, 229 insertions(+) create mode 100644 packages/data-provider/specs/azure.spec.ts create mode 100644 packages/data-provider/src/azure.ts diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts new file mode 100644 index 00000000000..c8ec8a2ab6a --- /dev/null +++ b/packages/data-provider/specs/azure.spec.ts @@ -0,0 +1,155 @@ +// import type { TAzureGroupConfigs } from '../src/config'; +import { validateAzureGroupConfigs } from '../src/azure'; + +describe('validateAzureGroupConfigs', () => { + it('should validate a correct configuration', () => { + const configs = [ + { + group: 'us-east', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-4-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', + }, + }, + }, + ]; + const { isValid, modelNames } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(true); + expect(modelNames).toEqual(['gpt-4-turbo']); + }); + + it('should return invalid for a configuration missing deploymentName at the model level where required', () => { + const configs = [ + { + group: 'us-west', + apiKey: 'us-west-key-5678', + instanceName: 'us-west-instance', + models: { + 'gpt-5': { + version: '2023-12-01', // Missing deploymentName + }, + }, + }, + ]; + const { isValid } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + }); + + it('should return invalid for a configuration with a boolean model where group lacks deploymentName and version', () => { + const configs = [ + { + group: 'sweden-central', + apiKey: 'sweden-central-9012', + instanceName: 'sweden-central-instance', + models: { + 'gpt-35-turbo': true, // The group lacks deploymentName and version + }, + }, + ]; + const { isValid } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + }); + + it('should allow a boolean model when group has both deploymentName and version', () => { + const configs = [ + { + group: 'japan-east', + apiKey: 'japan-east-3456', + instanceName: 'japan-east-instance', + deploymentName: 'default-deployment', + version: '2023-04-01', + models: { + 'gpt-5-turbo': true, + }, + }, + ]; + const { isValid, modelNames } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(true); + expect(modelNames).toContain('gpt-5-turbo'); + }); + + describe('validateAzureGroupConfigs additional cases', () => { + it('should validate correctly when optional fields are missing', () => { + const configs = [ + { + group: 'canada-central', + apiKey: 'canada-key', + instanceName: 'canada-instance', + models: { + 'gpt-6': { + deploymentName: 'gpt-6-deployment', + version: '2023-01-01', + }, + }, + }, + ]; + const { isValid, modelNames } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(true); + expect(modelNames).toEqual(['gpt-6']); + }); + + it('should return invalid for configurations with incorrect types', () => { + const configs = [ + { + group: 123, // incorrect type + apiKey: 'key123', + instanceName: 'instance123', + models: { + 'gpt-7': true, + }, + }, + ]; + // @ts-expect-error This error is expected because the 'group' property should be a string. + const { isValid } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + }); + + it('should correctly handle a mix of valid and invalid model configurations', () => { + const configs = [ + { + group: 'australia-southeast', + apiKey: 'australia-key', + instanceName: 'australia-instance', + models: { + 'valid-model': { + deploymentName: 'valid-deployment', + version: '2023-02-02', + }, + 'invalid-model': true, // Invalid because the group lacks deploymentName and version + }, + }, + ]; + const { isValid, modelNames } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + expect(modelNames).toEqual(expect.arrayContaining(['valid-model', 'invalid-model'])); + }); + + it('should return invalid for configuration missing required fields at the group level', () => { + const configs = [ + { + group: 'brazil-south', + apiKey: 'brazil-key', + // Missing instanceName + models: { + 'gpt-8': { + deploymentName: 'gpt-8-deployment', + version: '2023-03-03', + }, + }, + }, + ]; + // @ts-expect-error This error is expected because the 'instanceName' property is intentionally left out. + const { isValid } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + }); + }); +}); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts new file mode 100644 index 00000000000..f1499c6d2b4 --- /dev/null +++ b/packages/data-provider/src/azure.ts @@ -0,0 +1,47 @@ +import type { TAzureGroupConfigs } from '../src/config'; +import { azureGroupConfigsSchema } from '../src/config'; + +export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { + isValid: boolean; + modelNames: string[]; +} { + try { + const result = azureGroupConfigsSchema.safeParse(configs); + if (!result.success) { + // Basic structure is wrong, immediately return. + return { isValid: false, modelNames: [] }; + } + + const modelNames: string[] = []; + + for (const group of result.data) { + // Check if deploymentName and version are defined at the group level if a model is a boolean. + for (const modelName in group.models) { + // Collect model names + modelNames.push(modelName); + + const model = group.models[modelName]; + if (typeof model === 'boolean') { + // If model is boolean, check for deploymentName and version at group level. + if (!group.deploymentName || !group.version) { + return { isValid: false, modelNames }; + } + } else { + // If model is an object and does not define deploymentName or version, check group level. + if ( + (!model.deploymentName && !group.deploymentName) || + (!model.version && !group.version) + ) { + return { isValid: false, modelNames }; + } + } + } + } + + // If all checks are passed, the structure is valid. + return { isValid: true, modelNames }; + } catch (error) { + console.error(error); + return { isValid: false, modelNames: [] }; // In case of unexpected error, mark as invalid. + } +} diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 27ef91d6958..450cc2fca49 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -8,6 +8,32 @@ export const defaultSocialLogins = ['google', 'facebook', 'openid', 'github', 'd export const fileSourceSchema = z.nativeEnum(FileSources); +export const modelConfigSchema = z + .object({ + deploymentName: z.string().optional(), + version: z.string().optional(), + }) + .or(z.boolean()); + +export const azureGroupSchema = z.object({ + group: z.string(), + apiKey: z.string(), + instanceName: z.string(), + deploymentName: z.string().optional(), + version: z.string().optional(), + baseURL: z.string().optional(), + additionalHeaders: z.record(z.any()).optional(), + models: z.record(z.string(), modelConfigSchema), +}); + +export const azureGroupConfigsSchema = z.array(azureGroupSchema).min(1); + +export type TAzureGroupConfigs = z.infer; + +export const azureEndpointSchema = z.object({ + groupConfigs: azureGroupConfigsSchema, +}); + export const assistantEndpointSchema = z.object({ /* assistants specific */ disableBuilder: z.boolean().optional(), @@ -83,6 +109,7 @@ export const configSchema = z.object({ fileConfig: fileConfigSchema.optional(), endpoints: z .object({ + [EModelEndpoint.azureOpenAI]: azureEndpointSchema.optional(), [EModelEndpoint.assistants]: assistantEndpointSchema.optional(), custom: z.array(endpointSchema.partial()).optional(), }) From 83dc2f73bd53716f3505b09dd5f2422ceec2b86f Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 19 Feb 2024 18:52:40 -0500 Subject: [PATCH 02/35] refactor: azure config to return groupMap and modelConfigMap --- packages/data-provider/specs/azure.spec.ts | 173 +++++++++++++-------- packages/data-provider/src/azure.ts | 64 +++++--- packages/data-provider/src/config.ts | 20 ++- 3 files changed, 168 insertions(+), 89 deletions(-) diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index c8ec8a2ab6a..ac2308a762e 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -77,79 +77,126 @@ describe('validateAzureGroupConfigs', () => { expect(modelNames).toContain('gpt-5-turbo'); }); - describe('validateAzureGroupConfigs additional cases', () => { - it('should validate correctly when optional fields are missing', () => { - const configs = [ - { - group: 'canada-central', - apiKey: 'canada-key', - instanceName: 'canada-instance', - models: { - 'gpt-6': { - deploymentName: 'gpt-6-deployment', - version: '2023-01-01', - }, + it('should validate correctly when optional fields are missing', () => { + const configs = [ + { + group: 'canada-central', + apiKey: 'canada-key', + instanceName: 'canada-instance', + models: { + 'gpt-6': { + deploymentName: 'gpt-6-deployment', + version: '2023-01-01', }, }, - ]; - const { isValid, modelNames } = validateAzureGroupConfigs(configs); - expect(isValid).toBe(true); - expect(modelNames).toEqual(['gpt-6']); - }); + }, + ]; + const { isValid, modelNames } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(true); + expect(modelNames).toEqual(['gpt-6']); + }); + + it('should return invalid for configurations with incorrect types', () => { + const configs = [ + { + group: 123, // incorrect type + apiKey: 'key123', + instanceName: 'instance123', + models: { + 'gpt-7': true, + }, + }, + ]; + // @ts-expect-error This error is expected because the 'group' property should be a string. + const { isValid } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + }); - it('should return invalid for configurations with incorrect types', () => { - const configs = [ - { - group: 123, // incorrect type - apiKey: 'key123', - instanceName: 'instance123', - models: { - 'gpt-7': true, + it('should correctly handle a mix of valid and invalid model configurations', () => { + const configs = [ + { + group: 'australia-southeast', + apiKey: 'australia-key', + instanceName: 'australia-instance', + models: { + 'valid-model': { + deploymentName: 'valid-deployment', + version: '2023-02-02', }, + 'invalid-model': true, // Invalid because the group lacks deploymentName and version }, - ]; - // @ts-expect-error This error is expected because the 'group' property should be a string. - const { isValid } = validateAzureGroupConfigs(configs); - expect(isValid).toBe(false); - }); + }, + ]; + const { isValid, modelNames } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + expect(modelNames).toEqual(expect.arrayContaining(['valid-model', 'invalid-model'])); + }); - it('should correctly handle a mix of valid and invalid model configurations', () => { - const configs = [ - { - group: 'australia-southeast', - apiKey: 'australia-key', - instanceName: 'australia-instance', - models: { - 'valid-model': { - deploymentName: 'valid-deployment', - version: '2023-02-02', - }, - 'invalid-model': true, // Invalid because the group lacks deploymentName and version + it('should return invalid for configuration missing required fields at the group level', () => { + const configs = [ + { + group: 'brazil-south', + apiKey: 'brazil-key', + // Missing instanceName + models: { + 'gpt-8': { + deploymentName: 'gpt-8-deployment', + version: '2023-03-03', }, }, - ]; - const { isValid, modelNames } = validateAzureGroupConfigs(configs); - expect(isValid).toBe(false); - expect(modelNames).toEqual(expect.arrayContaining(['valid-model', 'invalid-model'])); - }); + }, + ]; + // @ts-expect-error This error is expected because the 'instanceName' property is intentionally left out. + const { isValid } = validateAzureGroupConfigs(configs); + expect(isValid).toBe(false); + }); +}); - it('should return invalid for configuration missing required fields at the group level', () => { - const configs = [ - { - group: 'brazil-south', - apiKey: 'brazil-key', - // Missing instanceName - models: { - 'gpt-8': { - deploymentName: 'gpt-8-deployment', - version: '2023-03-03', - }, +describe('validateAzureGroupConfigs with modelConfigMap and regionMap', () => { + it('should provide a valid modelConfigMap and regionMap for a correct configuration', () => { + const validConfigs = [ + { + group: 'us-east', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-4-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', + }, + }, + }, + { + group: 'us-east', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-4-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', }, }, - ]; - // @ts-expect-error This error is expected because the 'instanceName' property is intentionally left out. - const { isValid } = validateAzureGroupConfigs(configs); - expect(isValid).toBe(false); - }); + }, + ]; + const { isValid, modelConfigMap, groupMap } = validateAzureGroupConfigs(validConfigs); + expect(isValid).toBe(true); + expect(modelConfigMap['gpt-4-turbo']).toBeDefined(); + expect(modelConfigMap['gpt-4-turbo'].group).toBe('us-east'); + expect(groupMap['us-east']).toBeDefined(); + expect(groupMap['us-east'].apiKey).toBe('prod-1234'); + console.dir(modelConfigMap, { depth: null }); + console.dir(groupMap, { depth: null }); }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index f1499c6d2b4..575313e91e3 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -1,47 +1,67 @@ -import type { TAzureGroupConfigs } from '../src/config'; +import type { TAzureGroupConfigs, TAzureBaseSchema, TModelMapSchema } from '../src/config'; import { azureGroupConfigsSchema } from '../src/config'; export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { isValid: boolean; modelNames: string[]; + modelConfigMap: Record; + groupMap: Record; } { - try { - const result = azureGroupConfigsSchema.safeParse(configs); - if (!result.success) { - // Basic structure is wrong, immediately return. - return { isValid: false, modelNames: [] }; - } - - const modelNames: string[] = []; + let isValid = true; + const modelNames: string[] = []; + const modelConfigMap: Record = {}; + const groupMap: Record = {}; + const result = azureGroupConfigsSchema.safeParse(configs); + if (!result.success) { + isValid = false; + } else { for (const group of result.data) { - // Check if deploymentName and version are defined at the group level if a model is a boolean. + const { + group: groupName, + apiKey, + instanceName, + deploymentName, + version, + baseURL, + additionalHeaders, + } = group; + groupMap[groupName] = { + apiKey, + instanceName, + deploymentName, + version, + baseURL, + additionalHeaders, + }; + for (const modelName in group.models) { - // Collect model names modelNames.push(modelName); - const model = group.models[modelName]; + if (typeof model === 'boolean') { - // If model is boolean, check for deploymentName and version at group level. + // For boolean models, check if group-level deploymentName and version are present. if (!group.deploymentName || !group.version) { - return { isValid: false, modelNames }; + return { isValid: false, modelNames, modelConfigMap, groupMap }; } } else { - // If model is an object and does not define deploymentName or version, check group level. + // For object models, check if deploymentName and version are required but missing. if ( (!model.deploymentName && !group.deploymentName) || (!model.version && !group.version) ) { - return { isValid: false, modelNames }; + return { isValid: false, modelNames, modelConfigMap, groupMap }; } + + modelConfigMap[modelName] = { + group: groupName, + deploymentName: model.deploymentName || group.deploymentName, + version: model.version || group.version, + }; } } } - - // If all checks are passed, the structure is valid. - return { isValid: true, modelNames }; - } catch (error) { - console.error(error); - return { isValid: false, modelNames: [] }; // In case of unexpected error, mark as invalid. } + + return { isValid, modelNames, modelConfigMap, groupMap }; } diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 450cc2fca49..acfb0734d29 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -15,25 +15,37 @@ export const modelConfigSchema = z }) .or(z.boolean()); -export const azureGroupSchema = z.object({ - group: z.string(), +export const azureBaseSchema = z.object({ apiKey: z.string(), instanceName: z.string(), deploymentName: z.string().optional(), version: z.string().optional(), baseURL: z.string().optional(), additionalHeaders: z.record(z.any()).optional(), - models: z.record(z.string(), modelConfigSchema), }); -export const azureGroupConfigsSchema = z.array(azureGroupSchema).min(1); +export type TAzureBaseSchema = z.infer; + +export const azureGroupSchema = z + .object({ + group: z.string(), + models: z.record(z.string(), modelConfigSchema), + }) + .and(azureBaseSchema); +export const azureGroupConfigsSchema = z.array(azureGroupSchema).min(1); export type TAzureGroupConfigs = z.infer; export const azureEndpointSchema = z.object({ groupConfigs: azureGroupConfigsSchema, }); +export type TModelMapSchema = { + deploymentName?: string; + version?: string; + group: string; +}; + export const assistantEndpointSchema = z.object({ /* assistants specific */ disableBuilder: z.boolean().optional(), From d06dec919e1ce4188be3ad9c11614e039f41a5ea Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 20 Feb 2024 11:58:13 -0500 Subject: [PATCH 03/35] wip: naming and schema changes --- packages/data-provider/specs/azure.spec.ts | 13 ++++++------ packages/data-provider/src/azure.ts | 24 ++++++++++++---------- packages/data-provider/src/config.ts | 13 ++++++++---- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index ac2308a762e..55adf75accd 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -152,8 +152,8 @@ describe('validateAzureGroupConfigs', () => { }); }); -describe('validateAzureGroupConfigs with modelConfigMap and regionMap', () => { - it('should provide a valid modelConfigMap and regionMap for a correct configuration', () => { +describe('validateAzureGroupConfigs with modelGroupMap and regionMap', () => { + it('should provide a valid modelGroupMap and regionMap for a correct configuration', () => { const validConfigs = [ { group: 'us-east', @@ -190,13 +190,14 @@ describe('validateAzureGroupConfigs with modelConfigMap and regionMap', () => { }, }, ]; - const { isValid, modelConfigMap, groupMap } = validateAzureGroupConfigs(validConfigs); + const { isValid, modelGroupMap, groupMap } = validateAzureGroupConfigs(validConfigs); expect(isValid).toBe(true); - expect(modelConfigMap['gpt-4-turbo']).toBeDefined(); - expect(modelConfigMap['gpt-4-turbo'].group).toBe('us-east'); + expect(modelGroupMap['gpt-4-turbo']).toBeDefined(); + expect(modelGroupMap['gpt-4-turbo'].group).toBe('us-east'); expect(groupMap['us-east']).toBeDefined(); expect(groupMap['us-east'].apiKey).toBe('prod-1234'); - console.dir(modelConfigMap, { depth: null }); + expect(groupMap['us-east'].models['gpt-4-turbo']).toBeDefined(); + console.dir(modelGroupMap, { depth: null }); console.dir(groupMap, { depth: null }); }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index 575313e91e3..e810cf1ae04 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -1,16 +1,16 @@ -import type { TAzureGroupConfigs, TAzureBaseSchema, TModelMapSchema } from '../src/config'; +import type { TAzureGroupConfigs, TAzureGroupMap, TAzureModelMapSchema } from '../src/config'; import { azureGroupConfigsSchema } from '../src/config'; export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { isValid: boolean; modelNames: string[]; - modelConfigMap: Record; - groupMap: Record; + modelGroupMap: Record; + groupMap: Record; } { let isValid = true; const modelNames: string[] = []; - const modelConfigMap: Record = {}; - const groupMap: Record = {}; + const modelGroupMap: Record = {}; + const groupMap: Record = {}; const result = azureGroupConfigsSchema.safeParse(configs); if (!result.success) { @@ -25,6 +25,7 @@ export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { version, baseURL, additionalHeaders, + models, } = group; groupMap[groupName] = { apiKey, @@ -33,6 +34,7 @@ export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { version, baseURL, additionalHeaders, + models, }; for (const modelName in group.models) { @@ -42,7 +44,7 @@ export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { if (typeof model === 'boolean') { // For boolean models, check if group-level deploymentName and version are present. if (!group.deploymentName || !group.version) { - return { isValid: false, modelNames, modelConfigMap, groupMap }; + return { isValid: false, modelNames, modelGroupMap, groupMap }; } } else { // For object models, check if deploymentName and version are required but missing. @@ -50,18 +52,18 @@ export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { (!model.deploymentName && !group.deploymentName) || (!model.version && !group.version) ) { - return { isValid: false, modelNames, modelConfigMap, groupMap }; + return { isValid: false, modelNames, modelGroupMap, groupMap }; } - modelConfigMap[modelName] = { + modelGroupMap[modelName] = { group: groupName, - deploymentName: model.deploymentName || group.deploymentName, - version: model.version || group.version, + // deploymentName: model.deploymentName || group.deploymentName, + // version: model.version || group.version, }; } } } } - return { isValid, modelNames, modelConfigMap, groupMap }; + return { isValid, modelNames, modelGroupMap, groupMap }; } diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index acfb0734d29..37d77503b50 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -15,6 +15,8 @@ export const modelConfigSchema = z }) .or(z.boolean()); +export type TAzureModelConfig = z.infer; + export const azureBaseSchema = z.object({ apiKey: z.string(), instanceName: z.string(), @@ -31,21 +33,24 @@ export const azureGroupSchema = z group: z.string(), models: z.record(z.string(), modelConfigSchema), }) + .required() .and(azureBaseSchema); export const azureGroupConfigsSchema = z.array(azureGroupSchema).min(1); export type TAzureGroupConfigs = z.infer; export const azureEndpointSchema = z.object({ - groupConfigs: azureGroupConfigsSchema, + groups: azureGroupConfigsSchema, }); -export type TModelMapSchema = { - deploymentName?: string; - version?: string; +export type TAzureModelMapSchema = { + // deploymentName?: string; + // version?: string; group: string; }; +export type TAzureGroupMap = TAzureBaseSchema & { models: Record }; + export const assistantEndpointSchema = z.object({ /* assistants specific */ disableBuilder: z.boolean().optional(), From 79091a3659ecd3ae753d0641eb0c2a0d9b2a6fc6 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 20 Feb 2024 20:26:27 -0500 Subject: [PATCH 04/35] refactor(errorsToString): move to data-provider --- api/server/services/AuthService.js | 3 ++- api/strategies/localStrategy.js | 7 ++++--- api/strategies/validators.js | 12 ------------ api/strategies/validators.spec.js | 4 ++-- packages/data-provider/src/parsers.ts | 12 ++++++++++++ 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/api/server/services/AuthService.js b/api/server/services/AuthService.js index 0811b10fa25..5e8a3e55e39 100644 --- a/api/server/services/AuthService.js +++ b/api/server/services/AuthService.js @@ -1,6 +1,7 @@ const crypto = require('crypto'); const bcrypt = require('bcryptjs'); -const { registerSchema, errorsToString } = require('~/strategies/validators'); +const { errorsToString } = require('librechat-data-provider'); +const { registerSchema } = require('~/strategies/validators'); const getCustomConfig = require('~/server/services/Config/getCustomConfig'); const Token = require('~/models/schema/tokenSchema'); const { sendEmail } = require('~/server/utils'); diff --git a/api/strategies/localStrategy.js b/api/strategies/localStrategy.js index 916766e6287..4408382cc42 100644 --- a/api/strategies/localStrategy.js +++ b/api/strategies/localStrategy.js @@ -1,7 +1,8 @@ +const { errorsToString } = require('librechat-data-provider'); const { Strategy: PassportLocalStrategy } = require('passport-local'); -const User = require('../models/User'); -const { loginSchema, errorsToString } = require('./validators'); -const logger = require('../utils/logger'); +const { loginSchema } = require('./validators'); +const logger = require('~/utils/logger'); +const User = require('~/models/User'); async function validateLoginRequest(req) { const { error } = loginSchema.safeParse(req.body); diff --git a/api/strategies/validators.js b/api/strategies/validators.js index 22e4fa6ec5a..4cd43e5fcea 100644 --- a/api/strategies/validators.js +++ b/api/strategies/validators.js @@ -1,16 +1,5 @@ const { z } = require('zod'); -function errorsToString(errors) { - return errors - .map((error) => { - let field = error.path.join('.'); - let message = error.message; - - return `${field}: ${message}`; - }) - .join(' '); -} - const allowedCharactersRegex = /^[a-zA-Z0-9_.@#$%&*()\p{Script=Latin}\p{Script=Common}]+$/u; const injectionPatternsRegex = /('|--|\$ne|\$gt|\$lt|\$or|\{|\}|\*|;|<|>|\/|=)/i; @@ -72,5 +61,4 @@ const registerSchema = z module.exports = { loginSchema, registerSchema, - errorsToString, }; diff --git a/api/strategies/validators.spec.js b/api/strategies/validators.spec.js index 365818b65b8..7f4e02b60bf 100644 --- a/api/strategies/validators.spec.js +++ b/api/strategies/validators.spec.js @@ -1,6 +1,6 @@ // file deepcode ignore NoHardcodedPasswords: No hard-coded passwords in tests - -const { loginSchema, registerSchema, errorsToString } = require('./validators'); +const { errorsToString } = require('librechat-data-provider'); +const { loginSchema, registerSchema } = require('./validators'); describe('Zod Schemas', () => { describe('loginSchema', () => { diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.ts index 1a3e7d8340f..c8b93b7f018 100644 --- a/packages/data-provider/src/parsers.ts +++ b/packages/data-provider/src/parsers.ts @@ -1,3 +1,4 @@ +import type { ZodIssue } from 'zod'; import type { TConversation, TPreset } from './schemas'; import type { TEndpointOption } from './types'; import { @@ -42,6 +43,17 @@ const endpointSchemas: Record = { // [EModelEndpoint.google]: createGoogleSchema, // }; +export function errorsToString(errors: ZodIssue[]) { + return errors + .map((error) => { + const field = error.path.join('.'); + const message = error.message; + + return `${field}: ${message}`; + }) + .join(' '); +} + export function getFirstDefinedValue(possibleValues: string[]) { let returnValue; for (const value of possibleValues) { From 00d4372e6af9a2a920f72be92660a30fedc6b113 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 20 Feb 2024 20:27:24 -0500 Subject: [PATCH 05/35] feat: rename to azureGroups, add additional tests, tests all expected outcomes, return errors --- packages/data-provider/specs/azure.spec.ts | 122 ++++++++++++++++++--- packages/data-provider/src/azure.ts | 25 ++++- packages/data-provider/src/config.ts | 2 +- packages/data-provider/src/index.ts | 1 + 4 files changed, 129 insertions(+), 21 deletions(-) diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index 55adf75accd..1d95a891dde 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -1,7 +1,7 @@ -// import type { TAzureGroupConfigs } from '../src/config'; -import { validateAzureGroupConfigs } from '../src/azure'; +import type { TAzureGroups } from '../src/config'; +import { validateAzureGroups } from '../src/azure'; -describe('validateAzureGroupConfigs', () => { +describe('validateAzureGroups', () => { it('should validate a correct configuration', () => { const configs = [ { @@ -22,7 +22,7 @@ describe('validateAzureGroupConfigs', () => { }, }, ]; - const { isValid, modelNames } = validateAzureGroupConfigs(configs); + const { isValid, modelNames } = validateAzureGroups(configs); expect(isValid).toBe(true); expect(modelNames).toEqual(['gpt-4-turbo']); }); @@ -40,8 +40,9 @@ describe('validateAzureGroupConfigs', () => { }, }, ]; - const { isValid } = validateAzureGroupConfigs(configs); + const { isValid, errors } = validateAzureGroups(configs); expect(isValid).toBe(false); + expect(errors.length).toBe(1); }); it('should return invalid for a configuration with a boolean model where group lacks deploymentName and version', () => { @@ -55,8 +56,9 @@ describe('validateAzureGroupConfigs', () => { }, }, ]; - const { isValid } = validateAzureGroupConfigs(configs); + const { isValid, errors } = validateAzureGroups(configs); expect(isValid).toBe(false); + expect(errors.length).toBe(1); }); it('should allow a boolean model when group has both deploymentName and version', () => { @@ -72,8 +74,12 @@ describe('validateAzureGroupConfigs', () => { }, }, ]; - const { isValid, modelNames } = validateAzureGroupConfigs(configs); + const { isValid, modelNames, modelGroupMap, groupMap } = validateAzureGroups(configs); expect(isValid).toBe(true); + const modelGroup = modelGroupMap['gpt-5-turbo']; + expect(modelGroup).toBeDefined(); + expect(modelGroup.group).toBe('japan-east'); + expect(groupMap[modelGroup.group]).toBeDefined(); expect(modelNames).toContain('gpt-5-turbo'); }); @@ -91,7 +97,7 @@ describe('validateAzureGroupConfigs', () => { }, }, ]; - const { isValid, modelNames } = validateAzureGroupConfigs(configs); + const { isValid, modelNames } = validateAzureGroups(configs); expect(isValid).toBe(true); expect(modelNames).toEqual(['gpt-6']); }); @@ -108,8 +114,9 @@ describe('validateAzureGroupConfigs', () => { }, ]; // @ts-expect-error This error is expected because the 'group' property should be a string. - const { isValid } = validateAzureGroupConfigs(configs); + const { isValid, errors } = validateAzureGroups(configs); expect(isValid).toBe(false); + expect(errors.length).toBe(1); }); it('should correctly handle a mix of valid and invalid model configurations', () => { @@ -127,9 +134,10 @@ describe('validateAzureGroupConfigs', () => { }, }, ]; - const { isValid, modelNames } = validateAzureGroupConfigs(configs); + const { isValid, modelNames, errors } = validateAzureGroups(configs); expect(isValid).toBe(false); expect(modelNames).toEqual(expect.arrayContaining(['valid-model', 'invalid-model'])); + expect(errors.length).toBe(1); }); it('should return invalid for configuration missing required fields at the group level', () => { @@ -147,12 +155,13 @@ describe('validateAzureGroupConfigs', () => { }, ]; // @ts-expect-error This error is expected because the 'instanceName' property is intentionally left out. - const { isValid } = validateAzureGroupConfigs(configs); + const { isValid, errors } = validateAzureGroups(configs); expect(isValid).toBe(false); + expect(errors.length).toBe(1); }); }); -describe('validateAzureGroupConfigs with modelGroupMap and regionMap', () => { +describe('validateAzureGroups with modelGroupMap and regionMap', () => { it('should provide a valid modelGroupMap and regionMap for a correct configuration', () => { const validConfigs = [ { @@ -190,14 +199,97 @@ describe('validateAzureGroupConfigs with modelGroupMap and regionMap', () => { }, }, ]; - const { isValid, modelGroupMap, groupMap } = validateAzureGroupConfigs(validConfigs); + const { isValid, modelGroupMap, groupMap } = validateAzureGroups(validConfigs); expect(isValid).toBe(true); expect(modelGroupMap['gpt-4-turbo']).toBeDefined(); expect(modelGroupMap['gpt-4-turbo'].group).toBe('us-east'); expect(groupMap['us-east']).toBeDefined(); expect(groupMap['us-east'].apiKey).toBe('prod-1234'); expect(groupMap['us-east'].models['gpt-4-turbo']).toBeDefined(); - console.dir(modelGroupMap, { depth: null }); - console.dir(groupMap, { depth: null }); + }); + + it('should list all expected models in both modelGroupMap and groupMap', () => { + const validConfigs: TAzureGroups = [ + { + group: 'librechat-westus', + apiKey: '${WESTUS_API_KEY}', + instanceName: 'librechat-westus', + version: '2023-12-01-preview', + models: { + 'gpt-4-vision-preview': { + deploymentName: 'gpt-4-vision-preview', + version: '2024-02-15-preview', + }, + 'gpt-3.5-turbo': { + deploymentName: 'gpt-35-turbo', + }, + 'gpt-3.5-turbo-1106': { + deploymentName: 'gpt-35-turbo-1106', + }, + 'gpt-4': { + deploymentName: 'gpt-4', + }, + 'gpt-4-1106-preview': { + deploymentName: 'gpt-4-1106-preview', + }, + }, + }, + { + group: 'librechat-eastus', + apiKey: '${EASTUS_API_KEY}', + instanceName: 'librechat-eastus', + deploymentName: 'gpt-4-turbo', + version: '2024-02-15-preview', + models: { + 'gpt-4-turbo': true, + }, + }, + ]; + const { isValid, modelGroupMap, groupMap, modelNames } = validateAzureGroups(validConfigs); + expect(isValid).toBe(true); + expect(modelNames).toEqual([ + 'gpt-4-vision-preview', + 'gpt-3.5-turbo', + 'gpt-3.5-turbo-1106', + 'gpt-4', + 'gpt-4-1106-preview', + 'gpt-4-turbo', + ]); + + // Check modelGroupMap + modelNames.forEach((modelName) => { + expect(modelGroupMap[modelName]).toBeDefined(); + }); + + // Check groupMap for 'librechat-westus' + expect(groupMap).toHaveProperty('librechat-westus'); + expect(groupMap['librechat-westus']).toEqual( + expect.objectContaining({ + apiKey: '${WESTUS_API_KEY}', + instanceName: 'librechat-westus', + version: '2023-12-01-preview', + models: expect.objectContaining({ + 'gpt-4-vision-preview': expect.any(Object), + 'gpt-3.5-turbo': expect.any(Object), + 'gpt-3.5-turbo-1106': expect.any(Object), + 'gpt-4': expect.any(Object), + 'gpt-4-1106-preview': expect.any(Object), + }), + }), + ); + + // Check groupMap for 'librechat-eastus' + expect(groupMap).toHaveProperty('librechat-eastus'); + expect(groupMap['librechat-eastus']).toEqual( + expect.objectContaining({ + apiKey: '${EASTUS_API_KEY}', + instanceName: 'librechat-eastus', + deploymentName: 'gpt-4-turbo', + version: '2024-02-15-preview', + models: expect.objectContaining({ + 'gpt-4-turbo': true, + }), + }), + ); }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index e810cf1ae04..f699a11ed33 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -1,20 +1,25 @@ -import type { TAzureGroupConfigs, TAzureGroupMap, TAzureModelMapSchema } from '../src/config'; +import type { ZodError } from 'zod'; +import type { TAzureGroups, TAzureGroupMap, TAzureModelMapSchema } from '../src/config'; import { azureGroupConfigsSchema } from '../src/config'; +import { errorsToString } from '../src/parsers'; -export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { +export function validateAzureGroups(configs: TAzureGroups): { isValid: boolean; modelNames: string[]; modelGroupMap: Record; groupMap: Record; + errors: (ZodError | string)[]; } { let isValid = true; const modelNames: string[] = []; const modelGroupMap: Record = {}; const groupMap: Record = {}; + const errors: (ZodError | string)[] = []; const result = azureGroupConfigsSchema.safeParse(configs); if (!result.success) { isValid = false; + errors.push(errorsToString(result.error.errors)); } else { for (const group of result.data) { const { @@ -44,15 +49,25 @@ export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { if (typeof model === 'boolean') { // For boolean models, check if group-level deploymentName and version are present. if (!group.deploymentName || !group.version) { - return { isValid: false, modelNames, modelGroupMap, groupMap }; + errors.push( + `Model "${modelName}" in group "${groupName}" is missing a deploymentName or version.`, + ); + return { isValid: false, modelNames, modelGroupMap, groupMap, errors }; } + + modelGroupMap[modelName] = { + group: groupName, + }; } else { // For object models, check if deploymentName and version are required but missing. if ( (!model.deploymentName && !group.deploymentName) || (!model.version && !group.version) ) { - return { isValid: false, modelNames, modelGroupMap, groupMap }; + errors.push( + `Model "${modelName}" in group "${groupName}" is missing a required deploymentName or version.`, + ); + return { isValid: false, modelNames, modelGroupMap, groupMap, errors }; } modelGroupMap[modelName] = { @@ -65,5 +80,5 @@ export function validateAzureGroupConfigs(configs: TAzureGroupConfigs): { } } - return { isValid, modelNames, modelGroupMap, groupMap }; + return { isValid, modelNames, modelGroupMap, groupMap, errors }; } diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 37d77503b50..528b4e9acce 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -37,7 +37,7 @@ export const azureGroupSchema = z .and(azureBaseSchema); export const azureGroupConfigsSchema = z.array(azureGroupSchema).min(1); -export type TAzureGroupConfigs = z.infer; +export type TAzureGroups = z.infer; export const azureEndpointSchema = z.object({ groups: azureGroupConfigsSchema, diff --git a/packages/data-provider/src/index.ts b/packages/data-provider/src/index.ts index f44a4ffa52e..9e173eaec3e 100644 --- a/packages/data-provider/src/index.ts +++ b/packages/data-provider/src/index.ts @@ -1,4 +1,5 @@ /* config */ +export * from './azure'; export * from './config'; export * from './file-config'; /* schema helpers */ From dea6984e1edfc4e9054821d2852a6ea17b5788ed Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 20 Feb 2024 20:29:54 -0500 Subject: [PATCH 06/35] feat(AppService): load Azure groups --- api/server/services/AppService.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 69e92761200..7a98d25d726 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -1,8 +1,9 @@ const { + Constants, FileSources, EModelEndpoint, - Constants, defaultSocialLogins, + validateAzureGroups, } = require('librechat-data-provider'); const { initializeFirebase } = require('./Files/Firebase/initialize'); const loadCustomConfig = require('./Config/loadCustomConfig'); @@ -62,6 +63,21 @@ const AppService = async (app) => { handleRateLimits(config?.rateLimits); const endpointLocals = {}; + if (config?.endpoints?.[EModelEndpoint.azureOpenAI]) { + const { isValid, modelNames, modelGroupMap, groupMap, errors } = validateAzureGroups( + config.endpoints[EModelEndpoint.azureOpenAI].groups, + ); + if (!isValid) { + logger.error('Invalid Azure OpenAI configuration', errors); + throw new Error('Invalid Azure OpenAI configuration'); + } + + endpointLocals[EModelEndpoint.azureOpenAI] = { + modelNames, + modelGroupMap, + groupMap, + }; + } if (config?.endpoints?.[EModelEndpoint.assistants]) { const { disableBuilder, pollIntervalMs, timeoutMs, supportedIds, excludedIds } = config.endpoints[EModelEndpoint.assistants]; From d5cdf63b59e4d3a533110bafcfd82f26f77ec7ec Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 07:19:52 -0500 Subject: [PATCH 07/35] refactor(azure): use imported types, write `mapModelToAzureConfig` --- api/typedefs.js | 6 +++ packages/data-provider/src/azure.ts | 58 +++++++++++++++++++++++++--- packages/data-provider/src/config.ts | 6 ++- 3 files changed, 64 insertions(+), 6 deletions(-) diff --git a/api/typedefs.js b/api/typedefs.js index b8eef88a509..4941846d119 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -32,6 +32,12 @@ * @memberof typedefs */ +/** + * @exports TAzureGroups + * @typedef {import('librechat-data-provider').TAzureGroups} TAzureGroups + * @memberof typedefs + */ + /** * @exports TModelsConfig * @typedef {import('librechat-data-provider').TModelsConfig} TModelsConfig diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index f699a11ed33..8d7bbb88e57 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -1,19 +1,19 @@ import type { ZodError } from 'zod'; -import type { TAzureGroups, TAzureGroupMap, TAzureModelMapSchema } from '../src/config'; +import type { TAzureGroups, TAzureGroupMap, TAzureModelGroupMap } from '../src/config'; import { azureGroupConfigsSchema } from '../src/config'; import { errorsToString } from '../src/parsers'; export function validateAzureGroups(configs: TAzureGroups): { isValid: boolean; modelNames: string[]; - modelGroupMap: Record; - groupMap: Record; + modelGroupMap: TAzureModelGroupMap; + groupMap: TAzureGroupMap; errors: (ZodError | string)[]; } { let isValid = true; const modelNames: string[] = []; - const modelGroupMap: Record = {}; - const groupMap: Record = {}; + const modelGroupMap: TAzureModelGroupMap = {}; + const groupMap: TAzureGroupMap = {}; const errors: (ZodError | string)[] = []; const result = azureGroupConfigsSchema.safeParse(configs); @@ -82,3 +82,51 @@ export function validateAzureGroups(configs: TAzureGroups): { return { isValid, modelNames, modelGroupMap, groupMap, errors }; } + +export type AzureOptions = { + azureOpenAIApiKey: string; + azureOpenAIApiInstanceName: string; + azureOpenAIApiDeploymentName?: string; + azureOpenAIApiVersion?: string; +}; + +export function mapModelToAzureConfig({ + modelName, + modelGroupMap, + groupMap, +}: { + modelName: string; + modelGroupMap: TAzureModelGroupMap; + groupMap: TAzureGroupMap; +}): AzureOptions { + const modelConfig = modelGroupMap[modelName]; + if (!modelConfig) { + throw new Error(`Model named "${modelName}" not found in configuration.`); + } + + const groupConfig = groupMap[modelConfig.group]; + if (!groupConfig) { + throw new Error( + `Group "${modelConfig.group}" for model "${modelName}" not found in configuration.`, + ); + } + + const modelDetails = groupConfig.models[modelName]; + const deploymentName = + typeof modelDetails === 'object' + ? modelDetails.deploymentName || groupConfig.deploymentName + : groupConfig.deploymentName; + const version = + typeof modelDetails === 'object' + ? modelDetails.version || groupConfig.version + : groupConfig.version; + + const clientOptions: AzureOptions = { + azureOpenAIApiKey: groupConfig.apiKey, + azureOpenAIApiInstanceName: groupConfig.instanceName, + azureOpenAIApiDeploymentName: deploymentName, + azureOpenAIApiVersion: version, + }; + + return clientOptions; +} diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 528b4e9acce..1209dfbf994 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -49,7 +49,11 @@ export type TAzureModelMapSchema = { group: string; }; -export type TAzureGroupMap = TAzureBaseSchema & { models: Record }; +export type TAzureModelGroupMap = Record; +export type TAzureGroupMap = Record< + string, + TAzureBaseSchema & { models: Record } +>; export const assistantEndpointSchema = z.object({ /* assistants specific */ From a0345fd92ad2694c7803946d6bfb3a222aa653f0 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 07:27:17 -0500 Subject: [PATCH 08/35] refactor: move `extractEnvVariable` to data-provider --- .../services/Config/loadConfigEndpoints.js | 4 +- .../services/Config/loadConfigModels.js | 4 +- .../Endpoints/custom/initializeClient.js | 11 +++-- api/server/utils/handleText.js | 14 ------ api/server/utils/handleText.spec.js | 49 +------------------ packages/data-provider/specs/parsers.spec.ts | 48 ++++++++++++++++++ packages/data-provider/src/parsers.ts | 13 +++++ 7 files changed, 73 insertions(+), 70 deletions(-) create mode 100644 packages/data-provider/specs/parsers.spec.ts diff --git a/api/server/services/Config/loadConfigEndpoints.js b/api/server/services/Config/loadConfigEndpoints.js index 156688b059c..c80ba2e661d 100644 --- a/api/server/services/Config/loadConfigEndpoints.js +++ b/api/server/services/Config/loadConfigEndpoints.js @@ -1,5 +1,5 @@ -const { EModelEndpoint } = require('librechat-data-provider'); -const { isUserProvided, extractEnvVariable } = require('~/server/utils'); +const { EModelEndpoint, extractEnvVariable } = require('librechat-data-provider'); +const { isUserProvided } = require('~/server/utils'); const getCustomConfig = require('./getCustomConfig'); /** diff --git a/api/server/services/Config/loadConfigModels.js b/api/server/services/Config/loadConfigModels.js index d78ad5950b7..2b99b1f8e72 100644 --- a/api/server/services/Config/loadConfigModels.js +++ b/api/server/services/Config/loadConfigModels.js @@ -1,5 +1,5 @@ -const { EModelEndpoint } = require('librechat-data-provider'); -const { isUserProvided, extractEnvVariable } = require('~/server/utils'); +const { EModelEndpoint, extractEnvVariable } = require('librechat-data-provider'); +const { isUserProvided } = require('~/server/utils'); const { fetchModels } = require('~/server/services/ModelService'); const getCustomConfig = require('./getCustomConfig'); diff --git a/api/server/services/Endpoints/custom/initializeClient.js b/api/server/services/Endpoints/custom/initializeClient.js index da14b37dea9..9e0b0f666f3 100644 --- a/api/server/services/Endpoints/custom/initializeClient.js +++ b/api/server/services/Endpoints/custom/initializeClient.js @@ -1,13 +1,16 @@ -const { EModelEndpoint, CacheKeys } = require('librechat-data-provider'); +const { + EModelEndpoint, + CacheKeys, + extractEnvVariable, + envVarRegex, +} = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); const getCustomConfig = require('~/server/services/Config/getCustomConfig'); -const { isUserProvided, extractEnvVariable } = require('~/server/utils'); const { fetchModels } = require('~/server/services/ModelService'); const getLogStores = require('~/cache/getLogStores'); +const { isUserProvided } = require('~/server/utils'); const { OpenAIClient } = require('~/app'); -const envVarRegex = /^\${(.+)}$/; - const { PROXY } = process.env; const initializeClient = async ({ req, res, endpointOption }) => { diff --git a/api/server/utils/handleText.js b/api/server/utils/handleText.js index 3cb5bfa148d..8607d715198 100644 --- a/api/server/utils/handleText.js +++ b/api/server/utils/handleText.js @@ -172,19 +172,6 @@ function isEnabled(value) { */ const isUserProvided = (value) => value === 'user_provided'; -/** - * Extracts the value of an environment variable from a string. - * @param {string} value - The value to be processed, possibly containing an env variable placeholder. - * @returns {string} - The actual value from the environment variable or the original value. - */ -function extractEnvVariable(value) { - const envVarMatch = value.match(/^\${(.+)}$/); - if (envVarMatch) { - return process.env[envVarMatch[1]] || value; - } - return value; -} - module.exports = { createOnProgress, isEnabled, @@ -193,5 +180,4 @@ module.exports = { formatAction, addSpaceIfNeeded, isUserProvided, - extractEnvVariable, }; diff --git a/api/server/utils/handleText.spec.js b/api/server/utils/handleText.spec.js index a5566fb1b2b..ea440a89a57 100644 --- a/api/server/utils/handleText.spec.js +++ b/api/server/utils/handleText.spec.js @@ -1,4 +1,4 @@ -const { isEnabled, extractEnvVariable } = require('./handleText'); +const { isEnabled } = require('./handleText'); describe('isEnabled', () => { test('should return true when input is "true"', () => { @@ -48,51 +48,4 @@ describe('isEnabled', () => { test('should return false when input is an array', () => { expect(isEnabled([])).toBe(false); }); - - describe('extractEnvVariable', () => { - const originalEnv = process.env; - - beforeEach(() => { - jest.resetModules(); - process.env = { ...originalEnv }; - }); - - afterAll(() => { - process.env = originalEnv; - }); - - test('should return the value of the environment variable', () => { - process.env.TEST_VAR = 'test_value'; - expect(extractEnvVariable('${TEST_VAR}')).toBe('test_value'); - }); - - test('should return the original string if the envrionment variable is not defined correctly', () => { - process.env.TEST_VAR = 'test_value'; - expect(extractEnvVariable('${ TEST_VAR }')).toBe('${ TEST_VAR }'); - }); - - test('should return the original string if environment variable is not set', () => { - expect(extractEnvVariable('${NON_EXISTENT_VAR}')).toBe('${NON_EXISTENT_VAR}'); - }); - - test('should return the original string if it does not contain an environment variable', () => { - expect(extractEnvVariable('some_string')).toBe('some_string'); - }); - - test('should handle empty strings', () => { - expect(extractEnvVariable('')).toBe(''); - }); - - test('should handle strings without variable format', () => { - expect(extractEnvVariable('no_var_here')).toBe('no_var_here'); - }); - - test('should not process multiple variable formats', () => { - process.env.FIRST_VAR = 'first'; - process.env.SECOND_VAR = 'second'; - expect(extractEnvVariable('${FIRST_VAR} and ${SECOND_VAR}')).toBe( - '${FIRST_VAR} and ${SECOND_VAR}', - ); - }); - }); }); diff --git a/packages/data-provider/specs/parsers.spec.ts b/packages/data-provider/specs/parsers.spec.ts new file mode 100644 index 00000000000..e9ec9b20a4c --- /dev/null +++ b/packages/data-provider/specs/parsers.spec.ts @@ -0,0 +1,48 @@ +import { extractEnvVariable } from '../src/parsers'; + +describe('extractEnvVariable', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { ...originalEnv }; + }); + + afterAll(() => { + process.env = originalEnv; + }); + + test('should return the value of the environment variable', () => { + process.env.TEST_VAR = 'test_value'; + expect(extractEnvVariable('${TEST_VAR}')).toBe('test_value'); + }); + + test('should return the original string if the envrionment variable is not defined correctly', () => { + process.env.TEST_VAR = 'test_value'; + expect(extractEnvVariable('${ TEST_VAR }')).toBe('${ TEST_VAR }'); + }); + + test('should return the original string if environment variable is not set', () => { + expect(extractEnvVariable('${NON_EXISTENT_VAR}')).toBe('${NON_EXISTENT_VAR}'); + }); + + test('should return the original string if it does not contain an environment variable', () => { + expect(extractEnvVariable('some_string')).toBe('some_string'); + }); + + test('should handle empty strings', () => { + expect(extractEnvVariable('')).toBe(''); + }); + + test('should handle strings without variable format', () => { + expect(extractEnvVariable('no_var_here')).toBe('no_var_here'); + }); + + test('should not process multiple variable formats', () => { + process.env.FIRST_VAR = 'first'; + process.env.SECOND_VAR = 'second'; + expect(extractEnvVariable('${FIRST_VAR} and ${SECOND_VAR}')).toBe( + '${FIRST_VAR} and ${SECOND_VAR}', + ); + }); +}); diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.ts index c8b93b7f018..41d3fbef585 100644 --- a/packages/data-provider/src/parsers.ts +++ b/packages/data-provider/src/parsers.ts @@ -54,6 +54,19 @@ export function errorsToString(errors: ZodIssue[]) { .join(' '); } +export const envVarRegex = /^\${(.+)}$/; + +/** + * Extracts the value of an environment variable from a string. + */ +export function extractEnvVariable(value: string) { + const envVarMatch = value.match(envVarRegex); + if (envVarMatch) { + return process.env[envVarMatch[1]] || value; + } + return value; +} + export function getFirstDefinedValue(possibleValues: string[]) { let returnValue; for (const value of possibleValues) { From de913a377056c571b79bc76ff591ad2e65621d06 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 08:17:42 -0500 Subject: [PATCH 09/35] refactor(validateAzureGroups): throw on duplicate groups or models; feat(mapModelToAzureConfig): throw if env vars not present, add tests --- packages/data-provider/specs/azure.spec.ts | 264 ++++++++++++++++++++- packages/data-provider/src/azure.ts | 45 +++- 2 files changed, 291 insertions(+), 18 deletions(-) diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index 1d95a891dde..e5c772b33d5 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -1,5 +1,5 @@ import type { TAzureGroups } from '../src/config'; -import { validateAzureGroups } from '../src/azure'; +import { validateAzureGroups, mapModelToAzureConfig } from '../src/azure'; describe('validateAzureGroups', () => { it('should validate a correct configuration', () => { @@ -22,9 +22,21 @@ describe('validateAzureGroups', () => { }, }, ]; - const { isValid, modelNames } = validateAzureGroups(configs); + const { isValid, modelNames, modelGroupMap, groupMap } = validateAzureGroups(configs); expect(isValid).toBe(true); expect(modelNames).toEqual(['gpt-4-turbo']); + + const azureOptions = mapModelToAzureConfig({ + modelName: 'gpt-4-turbo', + modelGroupMap, + groupMap, + }); + expect(azureOptions).toEqual({ + azureOpenAIApiKey: 'prod-1234', + azureOpenAIApiInstanceName: 'prod-instance', + azureOpenAIApiDeploymentName: 'gpt-4-turbo-deployment', + azureOpenAIApiVersion: '2023-11-06', + }); }); it('should return invalid for a configuration missing deploymentName at the model level where required', () => { @@ -81,6 +93,17 @@ describe('validateAzureGroups', () => { expect(modelGroup.group).toBe('japan-east'); expect(groupMap[modelGroup.group]).toBeDefined(); expect(modelNames).toContain('gpt-5-turbo'); + const azureOptions = mapModelToAzureConfig({ + modelName: 'gpt-5-turbo', + modelGroupMap, + groupMap, + }); + expect(azureOptions).toEqual({ + azureOpenAIApiKey: 'japan-east-3456', + azureOpenAIApiInstanceName: 'japan-east-instance', + azureOpenAIApiDeploymentName: 'default-deployment', + azureOpenAIApiVersion: '2023-04-01', + }); }); it('should validate correctly when optional fields are missing', () => { @@ -97,9 +120,16 @@ describe('validateAzureGroups', () => { }, }, ]; - const { isValid, modelNames } = validateAzureGroups(configs); + const { isValid, modelNames, modelGroupMap, groupMap } = validateAzureGroups(configs); expect(isValid).toBe(true); expect(modelNames).toEqual(['gpt-6']); + const azureOptions = mapModelToAzureConfig({ modelName: 'gpt-6', modelGroupMap, groupMap }); + expect(azureOptions).toEqual({ + azureOpenAIApiKey: 'canada-key', + azureOpenAIApiInstanceName: 'canada-instance', + azureOpenAIApiDeploymentName: 'gpt-6-deployment', + azureOpenAIApiVersion: '2023-01-01', + }); }); it('should return invalid for configurations with incorrect types', () => { @@ -162,8 +192,19 @@ describe('validateAzureGroups', () => { }); describe('validateAzureGroups with modelGroupMap and regionMap', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.resetModules(); + process.env = { ...originalEnv }; + }); + + afterAll(() => { + process.env = originalEnv; + }); + it('should provide a valid modelGroupMap and regionMap for a correct configuration', () => { - const validConfigs = [ + const validConfigs: TAzureGroups = [ { group: 'us-east', apiKey: 'prod-1234', @@ -182,8 +223,8 @@ describe('validateAzureGroups with modelGroupMap and regionMap', () => { }, }, { - group: 'us-east', - apiKey: 'prod-1234', + group: 'us-west', + apiKey: 'prod-12345', instanceName: 'prod-instance', deploymentName: 'v1-deployment', version: '2023-12-31', @@ -192,8 +233,8 @@ describe('validateAzureGroups with modelGroupMap and regionMap', () => { 'X-Custom-Header': 'value', }, models: { - 'gpt-4-turbo': { - deploymentName: 'gpt-4-turbo-deployment', + 'gpt-5-turbo': { + deploymentName: 'gpt-5-turbo-deployment', version: '2023-11-06', }, }, @@ -206,9 +247,148 @@ describe('validateAzureGroups with modelGroupMap and regionMap', () => { expect(groupMap['us-east']).toBeDefined(); expect(groupMap['us-east'].apiKey).toBe('prod-1234'); expect(groupMap['us-east'].models['gpt-4-turbo']).toBeDefined(); + const azureOptions = mapModelToAzureConfig({ + modelName: 'gpt-4-turbo', + modelGroupMap, + groupMap, + }); + expect(azureOptions).toEqual({ + azureOpenAIApiKey: 'prod-1234', + azureOpenAIApiInstanceName: 'prod-instance', + azureOpenAIApiDeploymentName: 'gpt-4-turbo-deployment', + azureOpenAIApiVersion: '2023-11-06', + }); + }); + + it('should not allow duplicate group names', () => { + const duplicateGroups: TAzureGroups = [ + { + group: 'us-east', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-4-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', + }, + }, + }, + { + group: 'us-east', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-5-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', + }, + }, + }, + ]; + const { isValid } = validateAzureGroups(duplicateGroups); + expect(isValid).toBe(false); + }); + it('should not allow duplicate models across groups', () => { + const duplicateGroups: TAzureGroups = [ + { + group: 'us-east', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-4-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', + }, + }, + }, + { + group: 'us-west', + apiKey: 'prod-1234', + instanceName: 'prod-instance', + deploymentName: 'v1-deployment', + version: '2023-12-31', + baseURL: 'https://prod.example.com', + additionalHeaders: { + 'X-Custom-Header': 'value', + }, + models: { + 'gpt-4-turbo': { + deploymentName: 'gpt-4-turbo-deployment', + version: '2023-11-06', + }, + }, + }, + ]; + const { isValid } = validateAzureGroups(duplicateGroups); + expect(isValid).toBe(false); + }); + + it('should throw an error if environment variables are set but not configured', () => { + const validConfigs: TAzureGroups = [ + { + group: 'librechat-westus', + apiKey: '${WESTUS_API_KEY}', + instanceName: 'librechat-westus', + version: '2023-12-01-preview', + models: { + 'gpt-4-vision-preview': { + deploymentName: 'gpt-4-vision-preview', + version: '2024-02-15-preview', + }, + 'gpt-3.5-turbo': { + deploymentName: 'gpt-35-turbo', + }, + 'gpt-3.5-turbo-1106': { + deploymentName: 'gpt-35-turbo-1106', + }, + 'gpt-4': { + deploymentName: 'gpt-4', + }, + 'gpt-4-1106-preview': { + deploymentName: 'gpt-4-1106-preview', + }, + }, + }, + { + group: 'librechat-eastus', + apiKey: '${EASTUS_API_KEY}', + instanceName: 'librechat-eastus', + deploymentName: 'gpt-4-turbo', + version: '2024-02-15-preview', + models: { + 'gpt-4-turbo': true, + }, + }, + ]; + const { isValid, modelGroupMap, groupMap } = validateAzureGroups(validConfigs); + expect(isValid).toBe(true); + expect(() => + mapModelToAzureConfig({ modelName: 'gpt-4-turbo', modelGroupMap, groupMap }), + ).toThrow(); }); it('should list all expected models in both modelGroupMap and groupMap', () => { + process.env.WESTUS_API_KEY = 'westus-key'; + process.env.EASTUS_API_KEY = 'eastus-key'; + const validConfigs: TAzureGroups = [ { group: 'librechat-westus', @@ -291,5 +471,73 @@ describe('validateAzureGroups with modelGroupMap and regionMap', () => { }), }), ); + + const azureOptions1 = mapModelToAzureConfig({ + modelName: 'gpt-4-vision-preview', + modelGroupMap, + groupMap, + }); + expect(azureOptions1).toEqual({ + azureOpenAIApiKey: 'westus-key', + azureOpenAIApiInstanceName: 'librechat-westus', + azureOpenAIApiDeploymentName: 'gpt-4-vision-preview', + azureOpenAIApiVersion: '2024-02-15-preview', + }); + + const azureOptions2 = mapModelToAzureConfig({ + modelName: 'gpt-4-turbo', + modelGroupMap, + groupMap, + }); + expect(azureOptions2).toEqual({ + azureOpenAIApiKey: 'eastus-key', + azureOpenAIApiInstanceName: 'librechat-eastus', + azureOpenAIApiDeploymentName: 'gpt-4-turbo', + azureOpenAIApiVersion: '2024-02-15-preview', + }); + + const azureOptions3 = mapModelToAzureConfig({ modelName: 'gpt-4', modelGroupMap, groupMap }); + expect(azureOptions3).toEqual({ + azureOpenAIApiKey: 'westus-key', + azureOpenAIApiInstanceName: 'librechat-westus', + azureOpenAIApiDeploymentName: 'gpt-4', + azureOpenAIApiVersion: '2023-12-01-preview', + }); + + const azureOptions4 = mapModelToAzureConfig({ + modelName: 'gpt-3.5-turbo', + modelGroupMap, + groupMap, + }); + expect(azureOptions4).toEqual({ + azureOpenAIApiKey: 'westus-key', + azureOpenAIApiInstanceName: 'librechat-westus', + azureOpenAIApiDeploymentName: 'gpt-35-turbo', + azureOpenAIApiVersion: '2023-12-01-preview', + }); + + const azureOptions5 = mapModelToAzureConfig({ + modelName: 'gpt-3.5-turbo-1106', + modelGroupMap, + groupMap, + }); + expect(azureOptions5).toEqual({ + azureOpenAIApiKey: 'westus-key', + azureOpenAIApiInstanceName: 'librechat-westus', + azureOpenAIApiDeploymentName: 'gpt-35-turbo-1106', + azureOpenAIApiVersion: '2023-12-01-preview', + }); + + const azureOptions6 = mapModelToAzureConfig({ + modelName: 'gpt-4-1106-preview', + modelGroupMap, + groupMap, + }); + expect(azureOptions6).toEqual({ + azureOpenAIApiKey: 'westus-key', + azureOpenAIApiInstanceName: 'librechat-westus', + azureOpenAIApiDeploymentName: 'gpt-4-1106-preview', + azureOpenAIApiVersion: '2023-12-01-preview', + }); }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index 8d7bbb88e57..cc49913e0b8 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -1,7 +1,7 @@ import type { ZodError } from 'zod'; import type { TAzureGroups, TAzureGroupMap, TAzureModelGroupMap } from '../src/config'; +import { errorsToString, extractEnvVariable, envVarRegex } from '../src/parsers'; import { azureGroupConfigsSchema } from '../src/config'; -import { errorsToString } from '../src/parsers'; export function validateAzureGroups(configs: TAzureGroups): { isValid: boolean; @@ -32,6 +32,12 @@ export function validateAzureGroups(configs: TAzureGroups): { additionalHeaders, models, } = group; + + if (groupMap[groupName]) { + errors.push(`Duplicate group name detected: "${groupName}". Group names must be unique.`); + return { isValid: false, modelNames, modelGroupMap, groupMap, errors }; + } + groupMap[groupName] = { apiKey, instanceName, @@ -46,6 +52,13 @@ export function validateAzureGroups(configs: TAzureGroups): { modelNames.push(modelName); const model = group.models[modelName]; + if (modelGroupMap[modelName]) { + errors.push( + `Duplicate model name detected: "${modelName}". Model names must be unique across groups.`, + ); + return { isValid: false, modelNames, modelGroupMap, groupMap, errors }; + } + if (typeof model === 'boolean') { // For boolean models, check if group-level deploymentName and version are present. if (!group.deploymentName || !group.version) { @@ -83,11 +96,11 @@ export function validateAzureGroups(configs: TAzureGroups): { return { isValid, modelNames, modelGroupMap, groupMap, errors }; } -export type AzureOptions = { +type AzureOptions = { azureOpenAIApiKey: string; azureOpenAIApiInstanceName: string; - azureOpenAIApiDeploymentName?: string; - azureOpenAIApiVersion?: string; + azureOpenAIApiDeploymentName: string; + azureOpenAIApiVersion: string; }; export function mapModelToAzureConfig({ @@ -121,12 +134,24 @@ export function mapModelToAzureConfig({ ? modelDetails.version || groupConfig.version : groupConfig.version; - const clientOptions: AzureOptions = { - azureOpenAIApiKey: groupConfig.apiKey, - azureOpenAIApiInstanceName: groupConfig.instanceName, - azureOpenAIApiDeploymentName: deploymentName, - azureOpenAIApiVersion: version, + if (!deploymentName || !version) { + throw new Error( + `Model "${modelName}" in group "${modelConfig.group}" is missing a deploymentName ("${deploymentName}") or version ("${version}").`, + ); + } + + const azureOptions: AzureOptions = { + azureOpenAIApiKey: extractEnvVariable(groupConfig.apiKey), + azureOpenAIApiInstanceName: extractEnvVariable(groupConfig.instanceName), + azureOpenAIApiDeploymentName: extractEnvVariable(deploymentName), + azureOpenAIApiVersion: extractEnvVariable(version), }; - return clientOptions; + for (const value of Object.values(azureOptions)) { + if (typeof value === 'string' && envVarRegex.test(value)) { + throw new Error(`Azure configuration environment variable "${value}" was not found.`); + } + } + + return azureOptions; } From b19300a4513ce27d8f8a467b4ae81d3fe2f4f8ae Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 08:47:27 -0500 Subject: [PATCH 10/35] refactor(AppService): ensure each model is properly configured on startup --- api/server/services/AppService.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 7a98d25d726..5abbe0f1ebe 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -4,6 +4,7 @@ const { EModelEndpoint, defaultSocialLogins, validateAzureGroups, + mapModelToAzureConfig, } = require('librechat-data-provider'); const { initializeFirebase } = require('./Files/Firebase/initialize'); const loadCustomConfig = require('./Config/loadCustomConfig'); @@ -63,13 +64,21 @@ const AppService = async (app) => { handleRateLimits(config?.rateLimits); const endpointLocals = {}; + if (config?.endpoints?.[EModelEndpoint.azureOpenAI]) { const { isValid, modelNames, modelGroupMap, groupMap, errors } = validateAzureGroups( config.endpoints[EModelEndpoint.azureOpenAI].groups, ); + if (!isValid) { - logger.error('Invalid Azure OpenAI configuration', errors); - throw new Error('Invalid Azure OpenAI configuration'); + const errorString = errors.join('\n'); + const errorMessage = 'Invalid Azure OpenAI configuration:\n' + errorString; + logger.error(errorMessage); + throw new Error(errorMessage); + } + + for (const modelName of modelNames) { + mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); } endpointLocals[EModelEndpoint.azureOpenAI] = { @@ -78,6 +87,7 @@ const AppService = async (app) => { groupMap, }; } + if (config?.endpoints?.[EModelEndpoint.assistants]) { const { disableBuilder, pollIntervalMs, timeoutMs, supportedIds, excludedIds } = config.endpoints[EModelEndpoint.assistants]; From a7b9d6a38b11daace30a82ffbbb0f1f77cab5561 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 09:43:57 -0500 Subject: [PATCH 11/35] refactor: deprecate azureOpenAI environment variables in favor of librechat.yaml config --- .env.example | 23 +++--- api/server/services/AppService.js | 9 ++ api/server/services/AppService.spec.js | 109 +++++++++++++++++++++++-- packages/data-provider/src/azure.ts | 25 ++++++ 4 files changed, 150 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index ab9b57756a8..2d5bfca62ec 100644 --- a/.env.example +++ b/.env.example @@ -66,18 +66,21 @@ ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2 # Azure # #============# -# AZURE_API_KEY= -AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 -# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # PLUGINS_USE_AZURE="true" -AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE - -# AZURE_OPENAI_API_INSTANCE_NAME= -# AZURE_OPENAI_API_DEPLOYMENT_NAME= -# AZURE_OPENAI_API_VERSION= -# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= -# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= +# Note: these variables are DEPRECATED +# Use the `librechat.yaml` configuration for `azureOpenAI` instead +# You may also continue to use them if you opt out of using the `librechat.yaml` configuration + +# AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # Deprecated +# AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4 # Deprecated +# AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE # Deprecated +# AZURE_API_KEY= # Deprecated +# AZURE_OPENAI_API_INSTANCE_NAME= # Deprecated +# AZURE_OPENAI_API_DEPLOYMENT_NAME= # Deprecated +# AZURE_OPENAI_API_VERSION= # Deprecated +# AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated +# AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated #============# # BingAI # diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 5abbe0f1ebe..74d80f4db37 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -5,6 +5,7 @@ const { defaultSocialLogins, validateAzureGroups, mapModelToAzureConfig, + deprecatedAzureVariables, } = require('librechat-data-provider'); const { initializeFirebase } = require('./Files/Firebase/initialize'); const loadCustomConfig = require('./Config/loadCustomConfig'); @@ -86,6 +87,14 @@ const AppService = async (app) => { modelGroupMap, groupMap, }; + + deprecatedAzureVariables.forEach(({ key, description }) => { + if (process.env[key]) { + logger.warn( + `The \`${key}\` environment variable (related to ${description}) should not be used in combination with the \`azureOpenAI\` endpoint configuration, as you will experience conflicts and errors.`, + ); + } + }); } if (config?.endpoints?.[EModelEndpoint.assistants]) { diff --git a/api/server/services/AppService.spec.js b/api/server/services/AppService.spec.js index bedda9e3fd4..39d1c12673b 100644 --- a/api/server/services/AppService.spec.js +++ b/api/server/services/AppService.spec.js @@ -1,4 +1,10 @@ -const { FileSources, defaultSocialLogins } = require('librechat-data-provider'); +const { + FileSources, + EModelEndpoint, + defaultSocialLogins, + validateAzureGroups, + deprecatedAzureVariables, +} = require('librechat-data-provider'); const AppService = require('./AppService'); @@ -32,6 +38,43 @@ jest.mock('./ToolService', () => ({ }), })); +const azureGroups = [ + { + group: 'librechat-westus', + apiKey: '${WESTUS_API_KEY}', + instanceName: 'librechat-westus', + version: '2023-12-01-preview', + models: { + 'gpt-4-vision-preview': { + deploymentName: 'gpt-4-vision-preview', + version: '2024-02-15-preview', + }, + 'gpt-3.5-turbo': { + deploymentName: 'gpt-35-turbo', + }, + 'gpt-3.5-turbo-1106': { + deploymentName: 'gpt-35-turbo-1106', + }, + 'gpt-4': { + deploymentName: 'gpt-4', + }, + 'gpt-4-1106-preview': { + deploymentName: 'gpt-4-1106-preview', + }, + }, + }, + { + group: 'librechat-eastus', + apiKey: '${EASTUS_API_KEY}', + instanceName: 'librechat-eastus', + deploymentName: 'gpt-4-turbo', + version: '2024-02-15-preview', + models: { + 'gpt-4-turbo': true, + }, + }, +]; + describe('AppService', () => { let app; @@ -122,11 +165,11 @@ describe('AppService', () => { }); }); - it('should correctly configure endpoints based on custom config', async () => { + it('should correctly configure Assistants endpoint based on custom config', async () => { require('./Config/loadCustomConfig').mockImplementationOnce(() => Promise.resolve({ endpoints: { - assistants: { + [EModelEndpoint.assistants]: { disableBuilder: true, pollIntervalMs: 5000, timeoutMs: 30000, @@ -138,8 +181,8 @@ describe('AppService', () => { await AppService(app); - expect(app.locals).toHaveProperty('assistants'); - expect(app.locals.assistants).toEqual( + expect(app.locals).toHaveProperty(EModelEndpoint.assistants); + expect(app.locals[EModelEndpoint.assistants]).toEqual( expect.objectContaining({ disableBuilder: true, pollIntervalMs: 5000, @@ -149,6 +192,34 @@ describe('AppService', () => { ); }); + it('should correctly configure Azure OpenAI endpoint based on custom config', async () => { + require('./Config/loadCustomConfig').mockImplementationOnce(() => + Promise.resolve({ + endpoints: { + [EModelEndpoint.azureOpenAI]: { + groups: azureGroups, + }, + }, + }), + ); + + process.env.WESTUS_API_KEY = 'westus-key'; + process.env.EASTUS_API_KEY = 'eastus-key'; + + await AppService(app); + + expect(app.locals).toHaveProperty(EModelEndpoint.azureOpenAI); + const azureConfig = app.locals[EModelEndpoint.azureOpenAI]; + expect(azureConfig).toHaveProperty('modelNames'); + expect(azureConfig).toHaveProperty('modelGroupMap'); + expect(azureConfig).toHaveProperty('groupMap'); + + const { modelNames, modelGroupMap, groupMap } = validateAzureGroups(azureGroups); + expect(azureConfig.modelNames).toEqual(modelNames); + expect(azureConfig.modelGroupMap).toEqual(modelGroupMap); + expect(azureConfig.groupMap).toEqual(groupMap); + }); + it('should not modify FILE_UPLOAD environment variables without rate limits', async () => { // Setup initial environment variables process.env.FILE_UPLOAD_IP_MAX = '10'; @@ -213,7 +284,7 @@ describe('AppService', () => { }); }); -describe('AppService updating app.locals', () => { +describe('AppService updating app.locals and issuing warnings', () => { let app; let initialEnv; @@ -309,4 +380,30 @@ describe('AppService updating app.locals', () => { expect.stringContaining('Both `supportedIds` and `excludedIds` are defined'), ); }); + + it('should issue expected warnings when loading Azure Groups', async () => { + require('./Config/loadCustomConfig').mockImplementationOnce(() => + Promise.resolve({ + endpoints: { + [EModelEndpoint.azureOpenAI]: { + groups: azureGroups, + }, + }, + }), + ); + + deprecatedAzureVariables.forEach((varInfo) => { + process.env[varInfo.key] = 'test'; + }); + + const app = { locals: {} }; + await require('./AppService')(app); + + const { logger } = require('~/config'); + deprecatedAzureVariables.forEach(({ key, description }) => { + expect(logger.warn).toHaveBeenCalledWith( + `The \`${key}\` environment variable (related to ${description}) should not be used in combination with the \`azureOpenAI\` endpoint configuration, as you will experience conflicts and errors.`, + ); + }); + }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index cc49913e0b8..fcab4748518 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -3,6 +3,31 @@ import type { TAzureGroups, TAzureGroupMap, TAzureModelGroupMap } from '../src/c import { errorsToString, extractEnvVariable, envVarRegex } from '../src/parsers'; import { azureGroupConfigsSchema } from '../src/config'; +export const deprecatedAzureVariables = [ + /* "related to" precedes description text */ + { key: 'AZURE_OPENAI_DEFAULT_MODEL', description: 'setting a default model' }, + { key: 'AZURE_OPENAI_MODELS', description: 'setting models' }, + { + key: 'AZURE_USE_MODEL_AS_DEPLOYMENT_NAME', + description: 'using model names as deployment names', + }, + { key: 'AZURE_API_KEY', description: 'setting a single Azure API key' }, + { key: 'AZURE_OPENAI_API_INSTANCE_NAME', description: 'setting a single Azure instance name' }, + { + key: 'AZURE_OPENAI_API_DEPLOYMENT_NAME', + description: 'setting a single Azure deployment name', + }, + { key: 'AZURE_OPENAI_API_VERSION', description: 'setting a single Azure API version' }, + { + key: 'AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME', + description: 'setting a single Azure completions deployment name', + }, + { + key: 'AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME', + description: 'setting a single Azure embeddings deployment name', + }, +]; + export function validateAzureGroups(configs: TAzureGroups): { isValid: boolean; modelNames: string[]; From 6d9f393a54a7b8862d1bd595b898cc8eda4346a1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 12:18:33 -0500 Subject: [PATCH 12/35] feat: use helper functions to handle and order enabled/default endpoints; initialize azureOpenAI from config file --- api/server/controllers/EndpointController.js | 13 ++-- .../services/Config/loadConfigEndpoints.js | 13 +++- .../services/Config/loadDefaultEConfig.js | 21 +----- api/typedefs.js | 6 ++ packages/data-provider/src/parsers.ts | 66 +++++++++++++++++-- 5 files changed, 89 insertions(+), 30 deletions(-) diff --git a/api/server/controllers/EndpointController.js b/api/server/controllers/EndpointController.js index 3a0db022288..727a5a61c5f 100644 --- a/api/server/controllers/EndpointController.js +++ b/api/server/controllers/EndpointController.js @@ -1,4 +1,4 @@ -const { CacheKeys, EModelEndpoint } = require('librechat-data-provider'); +const { CacheKeys, EModelEndpoint, orderEndpointsConfig } = require('librechat-data-provider'); const { loadDefaultEndpointsConfig, loadConfigEndpoints } = require('~/server/services/Config'); const { getLogStores } = require('~/cache'); @@ -11,14 +11,17 @@ async function endpointController(req, res) { } const defaultEndpointsConfig = await loadDefaultEndpointsConfig(); - const customConfigEndpoints = await loadConfigEndpoints(); + const customConfigEndpoints = await loadConfigEndpoints(req); - const endpointsConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints }; - if (endpointsConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) { - endpointsConfig[EModelEndpoint.assistants].disableBuilder = + /** @type {TEndpointsConfig} */ + const mergedConfig = { ...defaultEndpointsConfig, ...customConfigEndpoints }; + if (mergedConfig[EModelEndpoint.assistants] && req.app.locals?.[EModelEndpoint.assistants]) { + mergedConfig[EModelEndpoint.assistants].disableBuilder = req.app.locals[EModelEndpoint.assistants].disableBuilder; } + const endpointsConfig = orderEndpointsConfig(mergedConfig); + await cache.set(CacheKeys.ENDPOINT_CONFIG, endpointsConfig); res.send(JSON.stringify(endpointsConfig)); } diff --git a/api/server/services/Config/loadConfigEndpoints.js b/api/server/services/Config/loadConfigEndpoints.js index c80ba2e661d..84d36e43333 100644 --- a/api/server/services/Config/loadConfigEndpoints.js +++ b/api/server/services/Config/loadConfigEndpoints.js @@ -4,8 +4,10 @@ const getCustomConfig = require('./getCustomConfig'); /** * Load config endpoints from the cached configuration object - * @function loadConfigEndpoints */ -async function loadConfigEndpoints() { + * @param {Express.Request} req - The request object + * @returns {Promise} A promise that resolves to an object containing the endpoints configuration + */ +async function loadConfigEndpoints(req) { const customConfig = await getCustomConfig(); if (!customConfig) { @@ -42,6 +44,13 @@ async function loadConfigEndpoints() { } } + if (req.app.locals[EModelEndpoint.azureOpenAI]) { + /** @type {Omit} */ + endpointsConfig[EModelEndpoint.azureOpenAI] = { + userProvide: false, + }; + } + return endpointsConfig; } diff --git a/api/server/services/Config/loadDefaultEConfig.js b/api/server/services/Config/loadDefaultEConfig.js index 0f1c7dcbb06..7e18203202e 100644 --- a/api/server/services/Config/loadDefaultEConfig.js +++ b/api/server/services/Config/loadDefaultEConfig.js @@ -1,4 +1,4 @@ -const { EModelEndpoint } = require('librechat-data-provider'); +const { EModelEndpoint, getEnabledEndpoints } = require('librechat-data-provider'); const loadAsyncEndpoints = require('./loadAsyncEndpoints'); const { config } = require('./EndpointService'); @@ -11,24 +11,7 @@ async function loadDefaultEndpointsConfig() { const { google, gptPlugins } = await loadAsyncEndpoints(); const { openAI, assistants, bingAI, anthropic, azureOpenAI, chatGPTBrowser } = config; - let enabledEndpoints = [ - EModelEndpoint.openAI, - EModelEndpoint.assistants, - EModelEndpoint.azureOpenAI, - EModelEndpoint.google, - EModelEndpoint.bingAI, - EModelEndpoint.chatGPTBrowser, - EModelEndpoint.gptPlugins, - EModelEndpoint.anthropic, - ]; - - const endpointsEnv = process.env.ENDPOINTS || ''; - if (endpointsEnv) { - enabledEndpoints = endpointsEnv - .split(',') - .filter((endpoint) => endpoint?.trim()) - .map((endpoint) => endpoint.trim()); - } + const enabledEndpoints = getEnabledEndpoints(); const endpointConfig = { [EModelEndpoint.openAI]: openAI, diff --git a/api/typedefs.js b/api/typedefs.js index 4941846d119..aacf875860e 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -56,6 +56,12 @@ * @memberof typedefs */ +/** + * @exports TEndpointsConfig + * @typedef {import('librechat-data-provider').TEndpointsConfig} TEndpointsConfig + * @memberof typedefs + */ + /** * @exports TMessage * @typedef {import('librechat-data-provider').TMessage} TMessage diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.ts index 41d3fbef585..216afe8a99c 100644 --- a/packages/data-provider/src/parsers.ts +++ b/packages/data-provider/src/parsers.ts @@ -1,6 +1,6 @@ import type { ZodIssue } from 'zod'; import type { TConversation, TPreset } from './schemas'; -import type { TEndpointOption } from './types'; +import type { TConfig, TEndpointOption, TEndpointsConfig } from './types'; import { EModelEndpoint, openAISchema, @@ -43,6 +43,66 @@ const endpointSchemas: Record = { // [EModelEndpoint.google]: createGoogleSchema, // }; +/** Get the enabled endpoints from the `ENDPOINTS` environment variable */ +export function getEnabledEndpoints() { + const defaultEndpoints: string[] = [ + EModelEndpoint.openAI, + EModelEndpoint.assistants, + EModelEndpoint.azureOpenAI, + EModelEndpoint.google, + EModelEndpoint.bingAI, + EModelEndpoint.chatGPTBrowser, + EModelEndpoint.gptPlugins, + EModelEndpoint.anthropic, + ]; + + const endpointsEnv = process.env.ENDPOINTS || ''; + let enabledEndpoints = defaultEndpoints; + if (endpointsEnv) { + enabledEndpoints = endpointsEnv + .split(',') + .filter((endpoint) => endpoint?.trim()) + .map((endpoint) => endpoint.trim()); + } + return enabledEndpoints; +} + +/** Orders an existing EndpointsConfig object based on enabled endpoint/custom ordering */ +export function orderEndpointsConfig(endpointsConfig: TEndpointsConfig) { + if (!endpointsConfig) { + return {}; + } + const enabledEndpoints = getEnabledEndpoints(); + const endpointKeys = Object.keys(endpointsConfig); + const defaultCustomIndex = enabledEndpoints.indexOf(EModelEndpoint.custom); + return endpointKeys.reduce( + (accumulatedConfig: Record, currentEndpointKey) => { + const isCustom = !(currentEndpointKey in EModelEndpoint); + const isEnabled = enabledEndpoints.includes(currentEndpointKey); + if (!isEnabled && !isCustom) { + return accumulatedConfig; + } + + const index = enabledEndpoints.indexOf(currentEndpointKey); + + if (isCustom) { + accumulatedConfig[currentEndpointKey] = { + order: defaultCustomIndex >= 0 ? defaultCustomIndex : 9999, + ...(endpointsConfig[currentEndpointKey] as Omit & { order?: number }), + }; + } else if (endpointsConfig[currentEndpointKey]) { + accumulatedConfig[currentEndpointKey] = { + ...endpointsConfig[currentEndpointKey], + order: index, + }; + } + return accumulatedConfig; + }, + {}, + ); +} + +/** Converts an array of Zod issues into a string. */ export function errorsToString(errors: ZodIssue[]) { return errors .map((error) => { @@ -56,9 +116,7 @@ export function errorsToString(errors: ZodIssue[]) { export const envVarRegex = /^\${(.+)}$/; -/** - * Extracts the value of an environment variable from a string. - */ +/** Extracts the value of an environment variable from a string. */ export function extractEnvVariable(value: string) { const envVarMatch = value.match(envVarRegex); if (envVarMatch) { From 9bf2ef03b7c80341799a9db3b5207b4ffd81381e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 12:30:20 -0500 Subject: [PATCH 13/35] refactor: redefine types as well as load azureOpenAI models from config file --- api/server/services/Config/loadConfigModels.js | 10 ++++++++++ api/typedefs.js | 6 ++++++ packages/data-provider/src/azure.ts | 16 ++++++++-------- packages/data-provider/src/config.ts | 6 ++++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/api/server/services/Config/loadConfigModels.js b/api/server/services/Config/loadConfigModels.js index 2b99b1f8e72..75c2ee70be1 100644 --- a/api/server/services/Config/loadConfigModels.js +++ b/api/server/services/Config/loadConfigModels.js @@ -18,6 +18,16 @@ async function loadConfigModels(req) { const { endpoints = {} } = customConfig ?? {}; const modelsConfig = {}; + if ( + endpoints[EModelEndpoint.azureOpenAI] && + req.app.locals[EModelEndpoint.azureOpenAI]?.modelNames + ) { + /** @type {TValidatedAzureConfig} */ + const validatedAzureConfig = req.app.locals[EModelEndpoint.azureOpenAI]; + const { modelNames } = validatedAzureConfig; + modelsConfig[EModelEndpoint.azureOpenAI] = modelNames; + } + if (!Array.isArray(endpoints[EModelEndpoint.custom])) { return modelsConfig; } diff --git a/api/typedefs.js b/api/typedefs.js index aacf875860e..4e93551b13c 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -38,6 +38,12 @@ * @memberof typedefs */ +/** + * @exports TValidatedAzureConfig + * @typedef {import('librechat-data-provider').TValidatedAzureConfig} TValidatedAzureConfig + * @memberof typedefs + */ + /** * @exports TModelsConfig * @typedef {import('librechat-data-provider').TModelsConfig} TModelsConfig diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index fcab4748518..0ca63134ab1 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -1,5 +1,10 @@ import type { ZodError } from 'zod'; -import type { TAzureGroups, TAzureGroupMap, TAzureModelGroupMap } from '../src/config'; +import type { + TAzureGroups, + TAzureGroupMap, + TAzureModelGroupMap, + TValidatedAzureConfig, +} from '../src/config'; import { errorsToString, extractEnvVariable, envVarRegex } from '../src/parsers'; import { azureGroupConfigsSchema } from '../src/config'; @@ -28,11 +33,8 @@ export const deprecatedAzureVariables = [ }, ]; -export function validateAzureGroups(configs: TAzureGroups): { +export function validateAzureGroups(configs: TAzureGroups): TValidatedAzureConfig & { isValid: boolean; - modelNames: string[]; - modelGroupMap: TAzureModelGroupMap; - groupMap: TAzureGroupMap; errors: (ZodError | string)[]; } { let isValid = true; @@ -132,10 +134,8 @@ export function mapModelToAzureConfig({ modelName, modelGroupMap, groupMap, -}: { +}: Omit & { modelName: string; - modelGroupMap: TAzureModelGroupMap; - groupMap: TAzureGroupMap; }): AzureOptions { const modelConfig = modelGroupMap[modelName]; if (!modelConfig) { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 1209dfbf994..9e13f4586bf 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -55,6 +55,12 @@ export type TAzureGroupMap = Record< TAzureBaseSchema & { models: Record } >; +export type TValidatedAzureConfig = { + modelNames: string[]; + modelGroupMap: TAzureModelGroupMap; + groupMap: TAzureGroupMap; +}; + export const assistantEndpointSchema = z.object({ /* assistants specific */ disableBuilder: z.boolean().optional(), From 0d2f7b40dcea46a051cb02b8e061f43de0c8549d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 13:14:20 -0500 Subject: [PATCH 14/35] chore(ci): fix test description naming --- packages/data-provider/specs/azure.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index e5c772b33d5..0fea6e846d3 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -191,7 +191,7 @@ describe('validateAzureGroups', () => { }); }); -describe('validateAzureGroups with modelGroupMap and regionMap', () => { +describe('validateAzureGroups with modelGroupMap and groupMap', () => { const originalEnv = process.env; beforeEach(() => { @@ -203,7 +203,7 @@ describe('validateAzureGroups with modelGroupMap and regionMap', () => { process.env = originalEnv; }); - it('should provide a valid modelGroupMap and regionMap for a correct configuration', () => { + it('should provide a valid modelGroupMap and groupMap for a correct configuration', () => { const validConfigs: TAzureGroups = [ { group: 'us-east', From 03ab6981518d1c2fdaf36717c1ea6f0c2bbe0155 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 13:15:03 -0500 Subject: [PATCH 15/35] feat(azureOpenAI): use validated model grouping for request authentication --- .../services/Endpoints/openAI/initializeClient.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initializeClient.js index 329749bdd60..f911d665963 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initializeClient.js @@ -1,4 +1,4 @@ -const { EModelEndpoint } = require('librechat-data-provider'); +const { EModelEndpoint, mapModelToAzureConfig } = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); const { getAzureCredentials } = require('~/utils'); const { isEnabled } = require('~/server/utils'); @@ -14,7 +14,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { OPENAI_SUMMARIZE, DEBUG_OPENAI, } = process.env; - const { key: expiresAt, endpoint } = req.body; + const { key: expiresAt, endpoint, model: modelName } = req.body; const contextStrategy = isEnabled(OPENAI_SUMMARIZE) ? 'summarize' : null; const baseURLOptions = { @@ -51,8 +51,16 @@ const initializeClient = async ({ req, res, endpointOption }) => { } let apiKey = isUserProvided ? userKey : credentials[endpoint]; + const isAzureOpenAI = endpoint === EModelEndpoint.azureOpenAI; + /** @type {false | TValidatedAzureConfig} */ + const azureConfig = isAzureOpenAI && req.app.locals[EModelEndpoint.azureOpenAI]; - if (endpoint === EModelEndpoint.azureOpenAI) { + if (isAzureOpenAI && azureConfig) { + /** @type {{ groups: TAzureGroups}} */ + const { modelGroupMap, groupMap } = azureConfig; + clientOptions.azure = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); + apiKey = clientOptions.azure.azureOpenAIApiKey; + } else if (isAzureOpenAI) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; } From a00558b70769fcd416f824bbbbb475fbf61ee9fb Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 13:30:20 -0500 Subject: [PATCH 16/35] chore: bump data-provider following rebase --- packages/data-provider/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data-provider/package.json b/packages/data-provider/package.json index 7628eed4dce..e0105fc49cf 100644 --- a/packages/data-provider/package.json +++ b/packages/data-provider/package.json @@ -1,6 +1,6 @@ { "name": "librechat-data-provider", - "version": "0.4.3", + "version": "0.4.4", "description": "data services for librechat apps", "main": "dist/index.js", "module": "dist/index.es.js", From 7aea82a5370e947ef45ffa29fab346052d809978 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 13:33:56 -0500 Subject: [PATCH 17/35] chore: bump config file version noting significant changes --- packages/data-provider/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 9e13f4586bf..e5cb9197162 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -425,7 +425,7 @@ export enum Constants { /** * Key for the Custom Config's version (librechat.yaml). */ - CONFIG_VERSION = '1.0.3', + CONFIG_VERSION = '1.0.4', /** * Standard value for the first message's `parentMessageId` value, to indicate no parent exists. */ From 2bec71a385ebf11f232b811c84346d94830840aa Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 21 Feb 2024 17:34:54 -0500 Subject: [PATCH 18/35] feat: add title options and switch azure configs for titling and vision requests --- api/app/clients/OpenAIClient.js | 31 +++++++++++++++++-- api/server/services/AppService.js | 9 ++++-- .../Endpoints/openAI/initializeClient.js | 5 ++- api/typedefs.js | 10 ++++++ packages/data-provider/src/config.ts | 22 ++++++++++--- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 4cd3df378f9..22d3e46872a 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -1,10 +1,12 @@ const OpenAI = require('openai'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { + ImageDetail, + EModelEndpoint, + ImageDetailCost, getResponseSender, validateVisionModel, - ImageDetailCost, - ImageDetail, + mapModelToAzureConfig, } = require('librechat-data-provider'); const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); const { @@ -725,6 +727,16 @@ class OpenAIClient extends BaseClient { max_tokens: 16, }; + if (this.azure && this.options.req.app.locals[EModelEndpoint.azureOpenAI]) { + /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ + const { modelGroupMap, groupMap } = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + this.azure = mapModelToAzureConfig({ + modelName: modelOptions.model, + modelGroupMap, + groupMap, + }); + } + const titleChatCompletion = async () => { modelOptions.model = model; @@ -975,6 +987,21 @@ ${convo} modelOptions.max_tokens = 4000; } + if ( + this.azure && + this.isVisionModel && + this.options.req.app.locals[EModelEndpoint.azureOpenAI] + ) { + /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ + const { modelGroupMap, groupMap } = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + this.azure = mapModelToAzureConfig({ + modelName: modelOptions.model, + modelGroupMap, + groupMap, + }); + this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); + } + if (this.azure || this.options.azure) { // Azure does not accept `model` in the body, so we need to remove it. delete modelOptions.model; diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 74d80f4db37..7562aad363b 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -67,9 +67,9 @@ const AppService = async (app) => { const endpointLocals = {}; if (config?.endpoints?.[EModelEndpoint.azureOpenAI]) { - const { isValid, modelNames, modelGroupMap, groupMap, errors } = validateAzureGroups( - config.endpoints[EModelEndpoint.azureOpenAI].groups, - ); + const { groups, titleModel, titleConvo, titleMethod } = + config.endpoints[EModelEndpoint.azureOpenAI]; + const { isValid, modelNames, modelGroupMap, groupMap, errors } = validateAzureGroups(groups); if (!isValid) { const errorString = errors.join('\n'); @@ -86,6 +86,9 @@ const AppService = async (app) => { modelNames, modelGroupMap, groupMap, + titleConvo, + titleMethod, + titleModel, }; deprecatedAzureVariables.forEach(({ key, description }) => { diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initializeClient.js index f911d665963..8147fe3b072 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initializeClient.js @@ -56,10 +56,13 @@ const initializeClient = async ({ req, res, endpointOption }) => { const azureConfig = isAzureOpenAI && req.app.locals[EModelEndpoint.azureOpenAI]; if (isAzureOpenAI && azureConfig) { - /** @type {{ groups: TAzureGroups}} */ + /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ const { modelGroupMap, groupMap } = azureConfig; clientOptions.azure = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); apiKey = clientOptions.azure.azureOpenAIApiKey; + clientOptions.titleConvo = azureConfig.titleConvo; + clientOptions.titleModel = azureConfig.titleModel; + clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion'; } else if (isAzureOpenAI) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; diff --git a/api/typedefs.js b/api/typedefs.js index 4e93551b13c..0862857e2e0 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -37,6 +37,16 @@ * @typedef {import('librechat-data-provider').TAzureGroups} TAzureGroups * @memberof typedefs */ +/** + * @exports TAzureModelGroupMap + * @typedef {import('librechat-data-provider').TAzureModelGroupMap} TAzureModelGroupMap + * @memberof typedefs + */ +/** + * @exports TAzureGroupMap + * @typedef {import('librechat-data-provider').TAzureGroupMap} TAzureGroupMap + * @memberof typedefs + */ /** * @exports TValidatedAzureConfig diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index e5cb9197162..9e1ca52459d 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -39,10 +39,6 @@ export const azureGroupSchema = z export const azureGroupConfigsSchema = z.array(azureGroupSchema).min(1); export type TAzureGroups = z.infer; -export const azureEndpointSchema = z.object({ - groups: azureGroupConfigsSchema, -}); - export type TAzureModelMapSchema = { // deploymentName?: string; // version?: string; @@ -109,8 +105,26 @@ export const endpointSchema = z.object({ headers: z.record(z.any()).optional(), addParams: z.record(z.any()).optional(), dropParams: z.array(z.string()).optional(), + customOrder: z.number().optional(), }); +export const azureEndpointSchema = z + .object({ + groups: azureGroupConfigsSchema, + }) + .and( + endpointSchema + .pick({ + titleConvo: true, + titleMethod: true, + titleModel: true, + summarize: true, + summaryModel: true, + customOrder: true, + }) + .partial(), + ); + export const rateLimitSchema = z.object({ fileUploads: z .object({ From 958dfcf7820f84951788e5243e82d70a1c4f8ac2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 22 Feb 2024 19:04:30 -0500 Subject: [PATCH 19/35] feat: enable azure plugins from config file --- api/server/controllers/EndpointController.js | 2 +- api/server/services/AppService.js | 3 ++- .../services/Config/loadAsyncEndpoints.js | 13 +++++---- .../services/Config/loadConfigModels.js | 16 +++++------ .../services/Config/loadDefaultEConfig.js | 6 ++--- .../services/Config/loadDefaultModels.js | 18 ++++++++----- .../Endpoints/gptPlugins/initializeClient.js | 27 ++++++++++++++----- api/typedefs.js | 7 +++++ packages/data-provider/src/azure.ts | 4 +++ packages/data-provider/src/config.ts | 3 +++ 10 files changed, 69 insertions(+), 30 deletions(-) diff --git a/api/server/controllers/EndpointController.js b/api/server/controllers/EndpointController.js index 727a5a61c5f..468dc21e799 100644 --- a/api/server/controllers/EndpointController.js +++ b/api/server/controllers/EndpointController.js @@ -10,7 +10,7 @@ async function endpointController(req, res) { return; } - const defaultEndpointsConfig = await loadDefaultEndpointsConfig(); + const defaultEndpointsConfig = await loadDefaultEndpointsConfig(req); const customConfigEndpoints = await loadConfigEndpoints(req); /** @type {TEndpointsConfig} */ diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 7562aad363b..d03cf959055 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -67,7 +67,7 @@ const AppService = async (app) => { const endpointLocals = {}; if (config?.endpoints?.[EModelEndpoint.azureOpenAI]) { - const { groups, titleModel, titleConvo, titleMethod } = + const { groups, titleModel, titleConvo, titleMethod, plugins } = config.endpoints[EModelEndpoint.azureOpenAI]; const { isValid, modelNames, modelGroupMap, groupMap, errors } = validateAzureGroups(groups); @@ -89,6 +89,7 @@ const AppService = async (app) => { titleConvo, titleMethod, titleModel, + plugins, }; deprecatedAzureVariables.forEach(({ key, description }) => { diff --git a/api/server/services/Config/loadAsyncEndpoints.js b/api/server/services/Config/loadAsyncEndpoints.js index 9e92f487fad..dcc3f624135 100644 --- a/api/server/services/Config/loadAsyncEndpoints.js +++ b/api/server/services/Config/loadAsyncEndpoints.js @@ -1,12 +1,14 @@ -const { availableTools } = require('~/app/clients/tools'); +const { EModelEndpoint } = require('librechat-data-provider'); const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs'); +const { availableTools } = require('~/app/clients/tools'); const { openAIApiKey, azureOpenAIApiKey, useAzurePlugins, userProvidedOpenAI, googleKey } = require('./EndpointService').config; /** * Load async endpoints and return a configuration object + * @param {Express.Request} req - The request object */ -async function loadAsyncEndpoints() { +async function loadAsyncEndpoints(req) { let i = 0; let serviceKey, googleUserProvides; try { @@ -35,13 +37,14 @@ async function loadAsyncEndpoints() { const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false; + const useAzure = req.app.locals[EModelEndpoint.azureOpenAI]?.plugins; const gptPlugins = - openAIApiKey || azureOpenAIApiKey + useAzure || openAIApiKey || azureOpenAIApiKey ? { plugins, availableAgents: ['classic', 'functions'], - userProvide: userProvidedOpenAI, - azure: useAzurePlugins, + userProvide: useAzure ? false : userProvidedOpenAI, + azure: useAzurePlugins || useAzure, } : false; diff --git a/api/server/services/Config/loadConfigModels.js b/api/server/services/Config/loadConfigModels.js index 75c2ee70be1..26920459a4a 100644 --- a/api/server/services/Config/loadConfigModels.js +++ b/api/server/services/Config/loadConfigModels.js @@ -17,15 +17,15 @@ async function loadConfigModels(req) { const { endpoints = {} } = customConfig ?? {}; const modelsConfig = {}; + const azureModels = req.app.locals[EModelEndpoint.azureOpenAI]?.modelNames; + const azureEndpoint = endpoints[EModelEndpoint.azureOpenAI]; - if ( - endpoints[EModelEndpoint.azureOpenAI] && - req.app.locals[EModelEndpoint.azureOpenAI]?.modelNames - ) { - /** @type {TValidatedAzureConfig} */ - const validatedAzureConfig = req.app.locals[EModelEndpoint.azureOpenAI]; - const { modelNames } = validatedAzureConfig; - modelsConfig[EModelEndpoint.azureOpenAI] = modelNames; + if (azureModels && azureEndpoint) { + modelsConfig[EModelEndpoint.azureOpenAI] = azureModels; + } + + if (azureModels && azureEndpoint && azureEndpoint.plugins) { + modelsConfig[EModelEndpoint.gptPlugins] = azureModels; } if (!Array.isArray(endpoints[EModelEndpoint.custom])) { diff --git a/api/server/services/Config/loadDefaultEConfig.js b/api/server/services/Config/loadDefaultEConfig.js index 7e18203202e..960dfb4c77a 100644 --- a/api/server/services/Config/loadDefaultEConfig.js +++ b/api/server/services/Config/loadDefaultEConfig.js @@ -4,11 +4,11 @@ const { config } = require('./EndpointService'); /** * Load async endpoints and return a configuration object - * @function loadDefaultEndpointsConfig + * @param {Express.Request} req - The request object * @returns {Promise>} An object whose keys are endpoint names and values are objects that contain the endpoint configuration and an order. */ -async function loadDefaultEndpointsConfig() { - const { google, gptPlugins } = await loadAsyncEndpoints(); +async function loadDefaultEndpointsConfig(req) { + const { google, gptPlugins } = await loadAsyncEndpoints(req); const { openAI, assistants, bingAI, anthropic, azureOpenAI, chatGPTBrowser } = config; const enabledEndpoints = getEnabledEndpoints(); diff --git a/api/server/services/Config/loadDefaultModels.js b/api/server/services/Config/loadDefaultModels.js index 29be4782210..2aa80d2ec99 100644 --- a/api/server/services/Config/loadDefaultModels.js +++ b/api/server/services/Config/loadDefaultModels.js @@ -18,12 +18,18 @@ async function loadDefaultModels(req) { const openAI = await getOpenAIModels({ user: req.user.id }); const anthropic = getAnthropicModels(); const chatGPTBrowser = getChatGPTBrowserModels(); - const azureOpenAI = await getOpenAIModels({ user: req.user.id, azure: true }); - const gptPlugins = await getOpenAIModels({ - user: req.user.id, - azure: useAzurePlugins, - plugins: true, - }); + let azureOpenAI; + if (!req.app.locals[EModelEndpoint.azureOpenAI]?.modelNames) { + azureOpenAI = await getOpenAIModels({ user: req.user.id, azure: true }); + } + let gptPlugins; + if (!req.app.locals[EModelEndpoint.azureOpenAI]?.plugins) { + gptPlugins = await getOpenAIModels({ + user: req.user.id, + azure: useAzurePlugins, + plugins: true, + }); + } const assistant = await getOpenAIModels({ assistants: true }); return { diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initializeClient.js index b667b2c0be5..5fb0123a158 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.js @@ -1,4 +1,4 @@ -const { EModelEndpoint } = require('librechat-data-provider'); +const { EModelEndpoint, mapModelToAzureConfig } = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); const { getAzureCredentials } = require('~/utils'); const { isEnabled } = require('~/server/utils'); @@ -16,11 +16,19 @@ const initializeClient = async ({ req, res, endpointOption }) => { DEBUG_PLUGINS, } = process.env; - const { key: expiresAt } = req.body; + const { key: expiresAt, model: modelName } = req.body; const contextStrategy = isEnabled(OPENAI_SUMMARIZE) ? 'summarize' : null; - const useAzure = isEnabled(PLUGINS_USE_AZURE); - const endpoint = useAzure ? EModelEndpoint.azureOpenAI : EModelEndpoint.openAI; + let useAzure = isEnabled(PLUGINS_USE_AZURE); + let endpoint = useAzure ? EModelEndpoint.azureOpenAI : EModelEndpoint.openAI; + + /** @type {false | TValidatedAzureConfig} */ + const azureConfig = req.app.locals[EModelEndpoint.azureOpenAI]; + useAzure = useAzure || azureConfig.plugins; + + if (useAzure && endpoint !== EModelEndpoint.azureOpenAI) { + endpoint = EModelEndpoint.azureOpenAI; + } const baseURLOptions = { [EModelEndpoint.openAI]: OPENAI_REVERSE_PROXY, @@ -59,8 +67,15 @@ const initializeClient = async ({ req, res, endpointOption }) => { } let apiKey = isUserProvided ? userKey : credentials[endpoint]; - - if (useAzure || (apiKey && apiKey.includes('{"azure') && !clientOptions.azure)) { + if (useAzure && azureConfig) { + /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ + const { modelGroupMap, groupMap } = azureConfig; + clientOptions.azure = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); + apiKey = clientOptions.azure.azureOpenAIApiKey; + clientOptions.titleConvo = azureConfig.titleConvo; + clientOptions.titleModel = azureConfig.titleModel; + clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion'; + } else if (useAzure || (apiKey && apiKey.includes('{"azure') && !clientOptions.azure)) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; } diff --git a/api/typedefs.js b/api/typedefs.js index 0862857e2e0..f64a3bdaf9c 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -37,6 +37,7 @@ * @typedef {import('librechat-data-provider').TAzureGroups} TAzureGroups * @memberof typedefs */ + /** * @exports TAzureModelGroupMap * @typedef {import('librechat-data-provider').TAzureModelGroupMap} TAzureModelGroupMap @@ -54,6 +55,12 @@ * @memberof typedefs */ +/** + * @exports TAzureEndpoint + * @typedef {import('librechat-data-provider').TAzureEndpoint} TAzureEndpoint + * @memberof typedefs + */ + /** * @exports TModelsConfig * @typedef {import('librechat-data-provider').TModelsConfig} TModelsConfig diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index 0ca63134ab1..b296336a1b8 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -31,6 +31,10 @@ export const deprecatedAzureVariables = [ key: 'AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME', description: 'setting a single Azure embeddings deployment name', }, + { + key: 'PLUGINS_USE_AZURE', + description: 'using Azure for Plugins', + }, ]; export function validateAzureGroups(configs: TAzureGroups): TValidatedAzureConfig & { diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 9e1ca52459d..997487dac07 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -111,6 +111,7 @@ export const endpointSchema = z.object({ export const azureEndpointSchema = z .object({ groups: azureGroupConfigsSchema, + plugins: z.boolean().optional(), }) .and( endpointSchema @@ -125,6 +126,8 @@ export const azureEndpointSchema = z .partial(), ); +export type TAzureEndpoint = z.infer; + export const rateLimitSchema = z.object({ fileUploads: z .object({ From 425c76ce9af5876efcd197aff1de8268f923fc43 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 22 Feb 2024 19:25:22 -0500 Subject: [PATCH 20/35] fix(ci): pass tests --- .../Endpoints/gptPlugins/initializeClient.js | 2 +- .../Endpoints/gptPlugins/initializeClient.spec.js | 13 +++++++++++++ .../Endpoints/openAI/initializeClient.spec.js | 12 ++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initializeClient.js index 5fb0123a158..3d6bca6715e 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.js @@ -24,7 +24,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { /** @type {false | TValidatedAzureConfig} */ const azureConfig = req.app.locals[EModelEndpoint.azureOpenAI]; - useAzure = useAzure || azureConfig.plugins; + useAzure = useAzure || azureConfig?.plugins; if (useAzure && endpoint !== EModelEndpoint.azureOpenAI) { endpoint = EModelEndpoint.azureOpenAI; diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.spec.js b/api/server/services/Endpoints/gptPlugins/initializeClient.spec.js index f3e7c89e304..f4539462f3d 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.spec.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.spec.js @@ -13,6 +13,9 @@ jest.mock('~/server/services/UserService', () => ({ describe('gptPlugins/initializeClient', () => { // Set up environment variables const originalEnvironment = process.env; + const app = { + locals: {}, + }; beforeEach(() => { jest.resetModules(); // Clears the cache @@ -32,6 +35,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: null }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -56,6 +60,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: null }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'test-model' } }; @@ -73,6 +78,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: null }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -89,6 +95,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: null }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -108,6 +115,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: null }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -129,6 +137,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: futureDate }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -148,6 +157,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: futureDate }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'test-model' } }; @@ -171,6 +181,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: expiresAt }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -187,6 +198,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: new Date(Date.now() + 10000).toISOString() }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; @@ -207,6 +219,7 @@ describe('gptPlugins/initializeClient', () => { const req = { body: { key: null }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'default-model' } }; diff --git a/api/server/services/Endpoints/openAI/initializeClient.spec.js b/api/server/services/Endpoints/openAI/initializeClient.spec.js index 792b73c664b..9be110c40bd 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.spec.js +++ b/api/server/services/Endpoints/openAI/initializeClient.spec.js @@ -12,6 +12,9 @@ jest.mock('~/server/services/UserService', () => ({ describe('initializeClient', () => { // Set up environment variables const originalEnvironment = process.env; + const app = { + locals: {}, + }; beforeEach(() => { jest.resetModules(); // Clears the cache @@ -30,6 +33,7 @@ describe('initializeClient', () => { const req = { body: { key: null, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; @@ -54,6 +58,7 @@ describe('initializeClient', () => { const req = { body: { key: null, endpoint: 'azureOpenAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = { modelOptions: { model: 'test-model' } }; @@ -71,6 +76,7 @@ describe('initializeClient', () => { const req = { body: { key: null, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; @@ -87,6 +93,7 @@ describe('initializeClient', () => { const req = { body: { key: null, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; @@ -104,6 +111,7 @@ describe('initializeClient', () => { const req = { body: { key: null, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; @@ -124,6 +132,7 @@ describe('initializeClient', () => { const req = { body: { key: expiresAt, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; @@ -141,6 +150,7 @@ describe('initializeClient', () => { const req = { body: { key: null, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; @@ -160,6 +170,7 @@ describe('initializeClient', () => { user: { id: '123', }, + app, }; const res = {}; @@ -183,6 +194,7 @@ describe('initializeClient', () => { const req = { body: { key: invalidKey, endpoint: 'openAI' }, user: { id: '123' }, + app, }; const res = {}; const endpointOption = {}; From 8d22017b0d9ba50f37896cbc4af6c3709a24b265 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 12:19:05 -0500 Subject: [PATCH 21/35] chore(.env.example): mark `PLUGINS_USE_AZURE` as deprecated --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 2d5bfca62ec..4dd251e29b3 100644 --- a/.env.example +++ b/.env.example @@ -66,7 +66,6 @@ ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2 # Azure # #============# -# PLUGINS_USE_AZURE="true" # Note: these variables are DEPRECATED # Use the `librechat.yaml` configuration for `azureOpenAI` instead @@ -81,6 +80,7 @@ ANTHROPIC_MODELS=claude-1,claude-instant-1,claude-2 # AZURE_OPENAI_API_VERSION= # Deprecated # AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME= # Deprecated # AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME= # Deprecated +# PLUGINS_USE_AZURE="true" # Deprecated #============# # BingAI # From c2f37b2a6a94e079aa280d65f1e16caf4980d5dd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 12:29:37 -0500 Subject: [PATCH 22/35] fix(fetchModels): early return if apiKey not passed --- .../services/Config/loadDefaultModels.js | 18 ++++++------------ api/server/services/ModelService.js | 8 +++++--- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/api/server/services/Config/loadDefaultModels.js b/api/server/services/Config/loadDefaultModels.js index 2aa80d2ec99..29be4782210 100644 --- a/api/server/services/Config/loadDefaultModels.js +++ b/api/server/services/Config/loadDefaultModels.js @@ -18,18 +18,12 @@ async function loadDefaultModels(req) { const openAI = await getOpenAIModels({ user: req.user.id }); const anthropic = getAnthropicModels(); const chatGPTBrowser = getChatGPTBrowserModels(); - let azureOpenAI; - if (!req.app.locals[EModelEndpoint.azureOpenAI]?.modelNames) { - azureOpenAI = await getOpenAIModels({ user: req.user.id, azure: true }); - } - let gptPlugins; - if (!req.app.locals[EModelEndpoint.azureOpenAI]?.plugins) { - gptPlugins = await getOpenAIModels({ - user: req.user.id, - azure: useAzurePlugins, - plugins: true, - }); - } + const azureOpenAI = await getOpenAIModels({ user: req.user.id, azure: true }); + const gptPlugins = await getOpenAIModels({ + user: req.user.id, + azure: useAzurePlugins, + plugins: true, + }); const assistant = await getOpenAIModels({ assistants: true }); return { diff --git a/api/server/services/ModelService.js b/api/server/services/ModelService.js index 107411e4cb5..96091e7b54d 100644 --- a/api/server/services/ModelService.js +++ b/api/server/services/ModelService.js @@ -38,6 +38,10 @@ const fetchModels = async ({ return models; } + if (!apiKey) { + return models; + } + try { const options = { headers: { @@ -92,9 +96,7 @@ const fetchModels = async ({ }, ); } else { - logger.error(`${logMessage} Something happened in setting up the request`, { - message: error.message ? error.message : '', - }); + logger.error(`${logMessage} Something happened in setting up the request`, error); } } From 4afed053e9f7909a7e13f49ca4eb0b35565b1fdd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 12:56:50 -0500 Subject: [PATCH 23/35] chore: fix azure config typing --- api/app/clients/OpenAIClient.js | 19 +++++++++---------- .../Endpoints/gptPlugins/initializeClient.js | 3 +-- .../Endpoints/openAI/initializeClient.js | 2 +- api/typedefs.js | 10 ++-------- packages/data-provider/src/config.ts | 3 ++- 5 files changed, 15 insertions(+), 22 deletions(-) diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 22d3e46872a..a3a112dda15 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -727,9 +727,10 @@ class OpenAIClient extends BaseClient { max_tokens: 16, }; - if (this.azure && this.options.req.app.locals[EModelEndpoint.azureOpenAI]) { - /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ - const { modelGroupMap, groupMap } = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + /** @type {TAzureConfig} */ + const azureConfig = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + if (this.azure && azureConfig) { + const { modelGroupMap, groupMap } = azureConfig; this.azure = mapModelToAzureConfig({ modelName: modelOptions.model, modelGroupMap, @@ -987,13 +988,11 @@ ${convo} modelOptions.max_tokens = 4000; } - if ( - this.azure && - this.isVisionModel && - this.options.req.app.locals[EModelEndpoint.azureOpenAI] - ) { - /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ - const { modelGroupMap, groupMap } = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + /** @type {TAzureConfig} */ + const azureConfig = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + + if (this.azure && this.isVisionModel && azureConfig) { + const { modelGroupMap, groupMap } = azureConfig; this.azure = mapModelToAzureConfig({ modelName: modelOptions.model, modelGroupMap, diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initializeClient.js index 3d6bca6715e..5804c14e70e 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.js @@ -22,7 +22,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { let useAzure = isEnabled(PLUGINS_USE_AZURE); let endpoint = useAzure ? EModelEndpoint.azureOpenAI : EModelEndpoint.openAI; - /** @type {false | TValidatedAzureConfig} */ + /** @type {false | TAzureConfig} */ const azureConfig = req.app.locals[EModelEndpoint.azureOpenAI]; useAzure = useAzure || azureConfig?.plugins; @@ -68,7 +68,6 @@ const initializeClient = async ({ req, res, endpointOption }) => { let apiKey = isUserProvided ? userKey : credentials[endpoint]; if (useAzure && azureConfig) { - /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ const { modelGroupMap, groupMap } = azureConfig; clientOptions.azure = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); apiKey = clientOptions.azure.azureOpenAIApiKey; diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initializeClient.js index 8147fe3b072..12fb34dab70 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initializeClient.js @@ -52,7 +52,7 @@ const initializeClient = async ({ req, res, endpointOption }) => { let apiKey = isUserProvided ? userKey : credentials[endpoint]; const isAzureOpenAI = endpoint === EModelEndpoint.azureOpenAI; - /** @type {false | TValidatedAzureConfig} */ + /** @type {false | TAzureConfig} */ const azureConfig = isAzureOpenAI && req.app.locals[EModelEndpoint.azureOpenAI]; if (isAzureOpenAI && azureConfig) { diff --git a/api/typedefs.js b/api/typedefs.js index f64a3bdaf9c..bd99cabb6b2 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -50,14 +50,8 @@ */ /** - * @exports TValidatedAzureConfig - * @typedef {import('librechat-data-provider').TValidatedAzureConfig} TValidatedAzureConfig - * @memberof typedefs - */ - -/** - * @exports TAzureEndpoint - * @typedef {import('librechat-data-provider').TAzureEndpoint} TAzureEndpoint + * @exports TAzureConfig + * @typedef {import('librechat-data-provider').TAzureConfig} TAzureConfig * @memberof typedefs */ diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 997487dac07..acba3b1bdc8 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -126,7 +126,8 @@ export const azureEndpointSchema = z .partial(), ); -export type TAzureEndpoint = z.infer; +export type TAzureConfig = Omit, 'groups'> & + TValidatedAzureConfig; export const rateLimitSchema = z.object({ fileUploads: z From d94e5a1c7dedfd5cfb55a8c477fd311650c97e57 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 13:20:47 -0500 Subject: [PATCH 24/35] refactor(mapModelToAzureConfig): return baseURL and headers as well as azureOptions --- api/app/clients/OpenAIClient.js | 6 ++- .../Endpoints/gptPlugins/initializeClient.js | 10 ++++- .../Endpoints/openAI/initializeClient.js | 11 +++-- packages/data-provider/specs/azure.spec.ts | 44 ++++++++++++++----- packages/data-provider/src/azure.ts | 20 ++++++++- 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index a3a112dda15..503b04c07e5 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -731,11 +731,12 @@ class OpenAIClient extends BaseClient { const azureConfig = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; if (this.azure && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - this.azure = mapModelToAzureConfig({ + const { azureOptions } = mapModelToAzureConfig({ modelName: modelOptions.model, modelGroupMap, groupMap, }); + this.azure = azureOptions; } const titleChatCompletion = async () => { @@ -993,11 +994,12 @@ ${convo} if (this.azure && this.isVisionModel && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - this.azure = mapModelToAzureConfig({ + const { azureOptions } = mapModelToAzureConfig({ modelName: modelOptions.model, modelGroupMap, groupMap, }); + this.azure = azureOptions; this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); } diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initializeClient.js index 5804c14e70e..31bb9832f0d 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.js @@ -69,11 +69,17 @@ const initializeClient = async ({ req, res, endpointOption }) => { let apiKey = isUserProvided ? userKey : credentials[endpoint]; if (useAzure && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - clientOptions.azure = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); - apiKey = clientOptions.azure.azureOpenAIApiKey; + const { azureOptions } = mapModelToAzureConfig({ + modelName, + modelGroupMap, + groupMap, + }); + clientOptions.azure = azureOptions; clientOptions.titleConvo = azureConfig.titleConvo; clientOptions.titleModel = azureConfig.titleModel; clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion'; + + apiKey = clientOptions.azure.azureOpenAIApiKey; } else if (useAzure || (apiKey && apiKey.includes('{"azure') && !clientOptions.azure)) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initializeClient.js index 12fb34dab70..52bc65855ad 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initializeClient.js @@ -56,13 +56,18 @@ const initializeClient = async ({ req, res, endpointOption }) => { const azureConfig = isAzureOpenAI && req.app.locals[EModelEndpoint.azureOpenAI]; if (isAzureOpenAI && azureConfig) { - /** @type {{ modelGroupMap: TAzureModelGroupMap, groupMap: TAzureGroupMap }} */ const { modelGroupMap, groupMap } = azureConfig; - clientOptions.azure = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap }); - apiKey = clientOptions.azure.azureOpenAIApiKey; + const { azureOptions } = mapModelToAzureConfig({ + modelName, + modelGroupMap, + groupMap, + }); + clientOptions.azure = azureOptions; clientOptions.titleConvo = azureConfig.titleConvo; clientOptions.titleModel = azureConfig.titleModel; clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion'; + + apiKey = clientOptions.azure.azureOpenAIApiKey; } else if (isAzureOpenAI) { clientOptions.azure = isUserProvided ? JSON.parse(userKey) : getAzureCredentials(); apiKey = clientOptions.azure.azureOpenAIApiKey; diff --git a/packages/data-provider/specs/azure.spec.ts b/packages/data-provider/specs/azure.spec.ts index 0fea6e846d3..156d7835f5b 100644 --- a/packages/data-provider/specs/azure.spec.ts +++ b/packages/data-provider/specs/azure.spec.ts @@ -26,7 +26,7 @@ describe('validateAzureGroups', () => { expect(isValid).toBe(true); expect(modelNames).toEqual(['gpt-4-turbo']); - const azureOptions = mapModelToAzureConfig({ + const { azureOptions, baseURL, headers } = mapModelToAzureConfig({ modelName: 'gpt-4-turbo', modelGroupMap, groupMap, @@ -37,6 +37,10 @@ describe('validateAzureGroups', () => { azureOpenAIApiDeploymentName: 'gpt-4-turbo-deployment', azureOpenAIApiVersion: '2023-11-06', }); + expect(baseURL).toBe('https://prod.example.com'); + expect(headers).toEqual({ + 'X-Custom-Header': 'value', + }); }); it('should return invalid for a configuration missing deploymentName at the model level where required', () => { @@ -93,7 +97,7 @@ describe('validateAzureGroups', () => { expect(modelGroup.group).toBe('japan-east'); expect(groupMap[modelGroup.group]).toBeDefined(); expect(modelNames).toContain('gpt-5-turbo'); - const azureOptions = mapModelToAzureConfig({ + const { azureOptions } = mapModelToAzureConfig({ modelName: 'gpt-5-turbo', modelGroupMap, groupMap, @@ -123,7 +127,7 @@ describe('validateAzureGroups', () => { const { isValid, modelNames, modelGroupMap, groupMap } = validateAzureGroups(configs); expect(isValid).toBe(true); expect(modelNames).toEqual(['gpt-6']); - const azureOptions = mapModelToAzureConfig({ modelName: 'gpt-6', modelGroupMap, groupMap }); + const { azureOptions } = mapModelToAzureConfig({ modelName: 'gpt-6', modelGroupMap, groupMap }); expect(azureOptions).toEqual({ azureOpenAIApiKey: 'canada-key', azureOpenAIApiInstanceName: 'canada-instance', @@ -247,7 +251,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { expect(groupMap['us-east']).toBeDefined(); expect(groupMap['us-east'].apiKey).toBe('prod-1234'); expect(groupMap['us-east'].models['gpt-4-turbo']).toBeDefined(); - const azureOptions = mapModelToAzureConfig({ + const { azureOptions, baseURL, headers } = mapModelToAzureConfig({ modelName: 'gpt-4-turbo', modelGroupMap, groupMap, @@ -258,6 +262,10 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { azureOpenAIApiDeploymentName: 'gpt-4-turbo-deployment', azureOpenAIApiVersion: '2023-11-06', }); + expect(baseURL).toBe('https://prod.example.com'); + expect(headers).toEqual({ + 'X-Custom-Header': 'value', + }); }); it('should not allow duplicate group names', () => { @@ -423,6 +431,10 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { models: { 'gpt-4-turbo': true, }, + baseURL: 'https://eastus.example.com', + additionalHeaders: { + 'x-api-key': 'x-api-key-value', + }, }, ]; const { isValid, modelGroupMap, groupMap, modelNames } = validateAzureGroups(validConfigs); @@ -472,7 +484,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { }), ); - const azureOptions1 = mapModelToAzureConfig({ + const { azureOptions: azureOptions1 } = mapModelToAzureConfig({ modelName: 'gpt-4-vision-preview', modelGroupMap, groupMap, @@ -484,7 +496,11 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { azureOpenAIApiVersion: '2024-02-15-preview', }); - const azureOptions2 = mapModelToAzureConfig({ + const { + azureOptions: azureOptions2, + baseURL, + headers, + } = mapModelToAzureConfig({ modelName: 'gpt-4-turbo', modelGroupMap, groupMap, @@ -495,8 +511,16 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { azureOpenAIApiDeploymentName: 'gpt-4-turbo', azureOpenAIApiVersion: '2024-02-15-preview', }); + expect(baseURL).toBe('https://eastus.example.com'); + expect(headers).toEqual({ + 'x-api-key': 'x-api-key-value', + }); - const azureOptions3 = mapModelToAzureConfig({ modelName: 'gpt-4', modelGroupMap, groupMap }); + const { azureOptions: azureOptions3 } = mapModelToAzureConfig({ + modelName: 'gpt-4', + modelGroupMap, + groupMap, + }); expect(azureOptions3).toEqual({ azureOpenAIApiKey: 'westus-key', azureOpenAIApiInstanceName: 'librechat-westus', @@ -504,7 +528,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { azureOpenAIApiVersion: '2023-12-01-preview', }); - const azureOptions4 = mapModelToAzureConfig({ + const { azureOptions: azureOptions4 } = mapModelToAzureConfig({ modelName: 'gpt-3.5-turbo', modelGroupMap, groupMap, @@ -516,7 +540,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { azureOpenAIApiVersion: '2023-12-01-preview', }); - const azureOptions5 = mapModelToAzureConfig({ + const { azureOptions: azureOptions5 } = mapModelToAzureConfig({ modelName: 'gpt-3.5-turbo-1106', modelGroupMap, groupMap, @@ -528,7 +552,7 @@ describe('validateAzureGroups with modelGroupMap and groupMap', () => { azureOpenAIApiVersion: '2023-12-01-preview', }); - const azureOptions6 = mapModelToAzureConfig({ + const { azureOptions: azureOptions6 } = mapModelToAzureConfig({ modelName: 'gpt-4-1106-preview', modelGroupMap, groupMap, diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index b296336a1b8..2d974604c62 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -134,13 +134,19 @@ type AzureOptions = { azureOpenAIApiVersion: string; }; +type MappedAzureConfig = { + azureOptions: AzureOptions; + baseURL?: string; + headers?: Record; +}; + export function mapModelToAzureConfig({ modelName, modelGroupMap, groupMap, }: Omit & { modelName: string; -}): AzureOptions { +}): MappedAzureConfig { const modelConfig = modelGroupMap[modelName]; if (!modelConfig) { throw new Error(`Model named "${modelName}" not found in configuration.`); @@ -182,5 +188,15 @@ export function mapModelToAzureConfig({ } } - return azureOptions; + const result: MappedAzureConfig = { azureOptions }; + + if (groupConfig.baseURL) { + result.baseURL = groupConfig.baseURL; + } + + if (groupConfig.additionalHeaders) { + result.headers = groupConfig.additionalHeaders; + } + + return result; } From e53fdf6413e6c86e7a186bd36233d6fbb0fdaa88 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 15:24:52 -0500 Subject: [PATCH 25/35] feat(createLLM): use `azureOpenAIBasePath` --- api/app/clients/llm/createLLM.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/app/clients/llm/createLLM.js b/api/app/clients/llm/createLLM.js index 62f2fe86f95..c8810b667b5 100644 --- a/api/app/clients/llm/createLLM.js +++ b/api/app/clients/llm/createLLM.js @@ -55,10 +55,13 @@ function createLLM({ } if (azure && configOptions.basePath) { - configOptions.basePath = constructAzureURL({ + const azureURL = constructAzureURL({ baseURL: configOptions.basePath, azure: azureOptions, }); + azureOptions.azureOpenAIBasePath = azureURL.split( + `/${azureOptions.azureOpenAIApiDeploymentName}`, + )[0]; } return new ChatOpenAI( From 3b472ca4b7defe3091292344fc1a1555f6a26e01 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 15:25:12 -0500 Subject: [PATCH 26/35] feat(parsers): resolveHeaders --- packages/data-provider/src/parsers.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/data-provider/src/parsers.ts b/packages/data-provider/src/parsers.ts index 216afe8a99c..2f1ba024688 100644 --- a/packages/data-provider/src/parsers.ts +++ b/packages/data-provider/src/parsers.ts @@ -125,6 +125,19 @@ export function extractEnvVariable(value: string) { return value; } +/** Resolves header values to env variables if detected */ +export function resolveHeaders(headers: Record | undefined) { + const resolvedHeaders = { ...(headers ?? {}) }; + + if (headers && typeof headers === 'object' && !Array.isArray(headers)) { + Object.keys(headers).forEach((key) => { + resolvedHeaders[key] = extractEnvVariable(headers[key]); + }); + } + + return resolvedHeaders; +} + export function getFirstDefinedValue(possibleValues: string[]) { let returnValue; for (const value of possibleValues) { From 1e9772ae3e5a29c572fe9c4ca9f46cb22987e1bc Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 15:25:41 -0500 Subject: [PATCH 27/35] refactor(extractBaseURL): handle invalid input --- api/utils/extractBaseURL.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/utils/extractBaseURL.js b/api/utils/extractBaseURL.js index 730473c4102..78e62fd8af2 100644 --- a/api/utils/extractBaseURL.js +++ b/api/utils/extractBaseURL.js @@ -12,9 +12,13 @@ * - `https://api.example.com/v1/replicate` -> `https://api.example.com/v1/replicate` * * @param {string} url - The URL to be processed. - * @returns {string} The matched pattern or input if no match is found. + * @returns {string | undefined} The matched pattern or input if no match is found. */ function extractBaseURL(url) { + if (!url || typeof url !== 'string') { + return undefined; + } + if (!url.includes('/v1')) { return url; } From 98b74de3e1677640c37fb031cb7e3cedbd566e17 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 15:26:39 -0500 Subject: [PATCH 28/35] feat(OpenAIClient): handle headers and baseURL for azureConfig --- api/app/clients/OpenAIClient.js | 36 +++++++++++++++++-- .../Endpoints/gptPlugins/initializeClient.js | 14 ++++++-- .../Endpoints/openAI/initializeClient.js | 14 ++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 503b04c07e5..72c3ff8a1af 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -3,6 +3,7 @@ const { HttpsProxyAgent } = require('https-proxy-agent'); const { ImageDetail, EModelEndpoint, + resolveHeaders, ImageDetailCost, getResponseSender, validateVisionModel, @@ -667,6 +668,16 @@ class OpenAIClient extends BaseClient { }; } + const { headers } = this.options; + if (headers && typeof headers === 'object' && !Array.isArray(headers)) { + configOptions.baseOptions = { + headers: resolveHeaders({ + ...headers, + ...configOptions?.baseOptions?.headers, + }), + }; + } + if (this.options.proxy) { configOptions.httpAgent = new HttpsProxyAgent(this.options.proxy); configOptions.httpsAgent = new HttpsProxyAgent(this.options.proxy); @@ -731,12 +742,19 @@ class OpenAIClient extends BaseClient { const azureConfig = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; if (this.azure && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - const { azureOptions } = mapModelToAzureConfig({ + const { + azureOptions, + baseURL, + headers = {}, + } = mapModelToAzureConfig({ modelName: modelOptions.model, modelGroupMap, groupMap, }); this.azure = azureOptions; + this.options.headers = resolveHeaders(headers); + this.options.reverseProxyUrl = baseURL ?? null; + this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl); } const titleChatCompletion = async () => { @@ -994,13 +1012,19 @@ ${convo} if (this.azure && this.isVisionModel && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - const { azureOptions } = mapModelToAzureConfig({ + const { + azureOptions, + baseURL, + headers = {}, + } = mapModelToAzureConfig({ modelName: modelOptions.model, modelGroupMap, groupMap, }); this.azure = azureOptions; this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); + opts.defaultHeaders = resolveHeaders(headers); + this.langchainProxy = extractBaseURL(baseURL); } if (this.azure || this.options.azure) { @@ -1054,12 +1078,20 @@ ${convo} ...modelOptions, ...this.options.addParams, }; + logger.debug('[OpenAIClient] chatCompletion: added params', { + addParams: this.options.addParams, + modelOptions, + }); } if (this.options.dropParams && Array.isArray(this.options.dropParams)) { this.options.dropParams.forEach((param) => { delete modelOptions[param]; }); + logger.debug('[OpenAIClient] chatCompletion: dropped params', { + dropParams: this.options.dropParams, + modelOptions, + }); } let UnexpectedRoleError = false; diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initializeClient.js index 31bb9832f0d..3094a091e8e 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.js +++ b/api/server/services/Endpoints/gptPlugins/initializeClient.js @@ -1,4 +1,8 @@ -const { EModelEndpoint, mapModelToAzureConfig } = require('librechat-data-provider'); +const { + EModelEndpoint, + mapModelToAzureConfig, + resolveHeaders, +} = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); const { getAzureCredentials } = require('~/utils'); const { isEnabled } = require('~/server/utils'); @@ -69,7 +73,11 @@ const initializeClient = async ({ req, res, endpointOption }) => { let apiKey = isUserProvided ? userKey : credentials[endpoint]; if (useAzure && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - const { azureOptions } = mapModelToAzureConfig({ + const { + azureOptions, + baseURL, + headers = {}, + } = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap, @@ -78,6 +86,8 @@ const initializeClient = async ({ req, res, endpointOption }) => { clientOptions.titleConvo = azureConfig.titleConvo; clientOptions.titleModel = azureConfig.titleModel; clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion'; + clientOptions.reverseProxyUrl = baseURL ?? clientOptions.reverseProxyUrl; + clientOptions.headers = resolveHeaders({ ...headers, ...(clientOptions.headers ?? {}) }); apiKey = clientOptions.azure.azureOpenAIApiKey; } else if (useAzure || (apiKey && apiKey.includes('{"azure') && !clientOptions.azure)) { diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initializeClient.js index 52bc65855ad..be3447fb9b8 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initializeClient.js @@ -1,4 +1,8 @@ -const { EModelEndpoint, mapModelToAzureConfig } = require('librechat-data-provider'); +const { + EModelEndpoint, + mapModelToAzureConfig, + resolveHeaders, +} = require('librechat-data-provider'); const { getUserKey, checkUserKeyExpiry } = require('~/server/services/UserService'); const { getAzureCredentials } = require('~/utils'); const { isEnabled } = require('~/server/utils'); @@ -57,7 +61,11 @@ const initializeClient = async ({ req, res, endpointOption }) => { if (isAzureOpenAI && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; - const { azureOptions } = mapModelToAzureConfig({ + const { + azureOptions, + baseURL, + headers = {}, + } = mapModelToAzureConfig({ modelName, modelGroupMap, groupMap, @@ -66,6 +74,8 @@ const initializeClient = async ({ req, res, endpointOption }) => { clientOptions.titleConvo = azureConfig.titleConvo; clientOptions.titleModel = azureConfig.titleModel; clientOptions.titleMethod = azureConfig.titleMethod ?? 'completion'; + clientOptions.reverseProxyUrl = baseURL ?? clientOptions.reverseProxyUrl; + clientOptions.headers = resolveHeaders({ ...headers, ...(clientOptions.headers ?? {}) }); apiKey = clientOptions.azure.azureOpenAIApiKey; } else if (isAzureOpenAI) { From a84df63e3fc56ecd470905c3bee452feb3d98e5a Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 23 Feb 2024 15:33:31 -0500 Subject: [PATCH 29/35] fix(ci): pass `OpenAIClient` tests --- api/app/clients/OpenAIClient.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 72c3ff8a1af..3e02d346d2b 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -738,8 +738,8 @@ class OpenAIClient extends BaseClient { max_tokens: 16, }; - /** @type {TAzureConfig} */ - const azureConfig = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + /** @type {TAzureConfig | undefined} */ + const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI]; if (this.azure && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; const { @@ -1007,8 +1007,8 @@ ${convo} modelOptions.max_tokens = 4000; } - /** @type {TAzureConfig} */ - const azureConfig = this.options.req.app.locals[EModelEndpoint.azureOpenAI]; + /** @type {TAzureConfig | undefined} */ + const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI]; if (this.azure && this.isVisionModel && azureConfig) { const { modelGroupMap, groupMap } = azureConfig; From 403a16f20f78fdaa457524fa2697cfceadda64ed Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 26 Feb 2024 10:59:11 -0500 Subject: [PATCH 30/35] chore: extract env var for azureOpenAI group config, baseURL --- packages/data-provider/src/azure.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index 2d974604c62..2ecfb18a1a8 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -191,7 +191,7 @@ export function mapModelToAzureConfig({ const result: MappedAzureConfig = { azureOptions }; if (groupConfig.baseURL) { - result.baseURL = groupConfig.baseURL; + result.baseURL = extractEnvVariable(groupConfig.baseURL); } if (groupConfig.additionalHeaders) { From b0b5aa80cc7d92791251d2c094e9a4274a76f292 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 26 Feb 2024 11:27:17 -0500 Subject: [PATCH 31/35] docs: azureOpenAI config setup docs --- docs/install/configuration/ai_setup.md | 220 +---------- docs/install/configuration/azure_openai.md | 406 ++++++++++++++++++++ docs/install/configuration/custom_config.md | 213 +++++++++- docs/install/configuration/index.md | 1 + docs/install/index.md | 1 + 5 files changed, 607 insertions(+), 234 deletions(-) create mode 100644 docs/install/configuration/azure_openai.md diff --git a/docs/install/configuration/ai_setup.md b/docs/install/configuration/ai_setup.md index 4f462e0926f..f4c35a8082a 100644 --- a/docs/install/configuration/ai_setup.md +++ b/docs/install/configuration/ai_setup.md @@ -236,221 +236,9 @@ Note: Using Gemini models through Vertex AI is possible but not yet supported. ## Azure OpenAI -In order to use Azure OpenAI with this project, specific environment variables must be set in your `.env` file. These variables will be used for constructing the API URLs. +### Please see the dedicated [Azure OpenAI Setup Guide.](./azure_openai.md) -The variables needed are outlined below: - -### Required Variables - -These variables construct the API URL for Azure OpenAI. - -* `AZURE_API_KEY`: Your Azure OpenAI API key. -* `AZURE_OPENAI_API_INSTANCE_NAME`: The instance name of your Azure OpenAI API. -* `AZURE_OPENAI_API_DEPLOYMENT_NAME`: The deployment name of your Azure OpenAI API. -* `AZURE_OPENAI_API_VERSION`: The version of your Azure OpenAI API. - -For example, with these variables, the URL for chat completion would look something like: -```plaintext -https://{AZURE_OPENAI_API_INSTANCE_NAME}.openai.azure.com/openai/deployments/{AZURE_OPENAI_API_DEPLOYMENT_NAME}/chat/completions?api-version={AZURE_OPENAI_API_VERSION} -``` -You should also consider changing the `AZURE_OPENAI_MODELS` variable to the models available in your deployment. - -```bash -# .env file -AZURE_OPENAI_MODELS=gpt-4-1106-preview,gpt-4,gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-4-vision-preview -``` - -Overriding the construction of the API URL will be possible but is not yet implemented. Follow progress on this feature here: **[Issue #1266](https://github.com/danny-avila/LibreChat/issues/1266)** - -### Model Deployments - -> Note: a change will be developed to improve current configuration settings, to allow multiple deployments/model configurations setup with ease: **[#1390](https://github.com/danny-avila/LibreChat/issues/1390)** - -As of 2023-12-18, the Azure API allows only one model per deployment. - -**It's highly recommended** to name your deployments *after* the model name (e.g., "gpt-3.5-turbo") for easy deployment switching. - -When you do so, LibreChat will correctly switch the deployment, while associating the correct max context per model, if you have the following environment variable set: - -```bash -AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE -``` - -For example, when you have set `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE`, the following deployment configuration provides the most seamless, error-free experience for LibreChat, including Vision support and tracking the correct max context tokens: - -![Screenshot 2023-12-18 111742](https://github.com/danny-avila/LibreChat/assets/110412045/4aa8a61c-0317-4681-8262-a6382dcaa7b0) - - -Alternatively, you can use custom deployment names and set `AZURE_OPENAI_DEFAULT_MODEL` for expected functionality. - -- **`AZURE_OPENAI_MODELS`**: List the available models, separated by commas without spaces. The first listed model will be the default. If left blank, internal settings will be used. Note that deployment names can't have periods, which are removed when generating the endpoint. - -Example use: - -```bash -# .env file -AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4,gpt-5 - -``` - -- **`AZURE_USE_MODEL_AS_DEPLOYMENT_NAME`**: Enable using the model name as the deployment name for the API URL. - -Example use: - -```bash -# .env file -AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE - -``` - -### Setting a Default Model for Azure - -This section is relevant when you are **not** naming deployments after model names as shown above. - -**Important:** The Azure OpenAI API does not use the `model` field in the payload but is a necessary identifier for LibreChat. If your deployment names do not correspond to the model names, and you're having issues with the model not being recognized, you should set this field to explicitly tell LibreChat to treat your Azure OpenAI API requests as if the specified model was selected. - -If AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is enabled, the model you set with `AZURE_OPENAI_DEFAULT_MODEL` will **not** be recognized and will **not** be used as the deployment name; instead, it will use the model selected by the user as the "deployment" name. - -- **`AZURE_OPENAI_DEFAULT_MODEL`**: Override the model setting for Azure, useful if using custom deployment names. - -Example use: - -```bash -# .env file -# MUST be a real OpenAI model, named exactly how it is recognized by OpenAI API (not Azure) -AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # do include periods in the model name here - -``` - -### Using a Specified Base URL with Azure - -The base URL for Azure OpenAI API requests can be dynamically configured. This is useful for proxying services such as [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/providers/azureopenai/), or if you wish to explicitly override the baseURL handling of the app. - -LibreChat will use the `AZURE_OPENAI_BASEURL` environment variable, which can include placeholders for the Azure OpenAI API instance and deployment names. - -In the application's environment configuration, the base URL is set like this: - -```bash -# .env file -AZURE_OPENAI_BASEURL=https://example.azure-api.net/${INSTANCE_NAME}/${DEPLOYMENT_NAME} - -# OR -AZURE_OPENAI_BASEURL=https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME} - -# Cloudflare example -AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME} -``` - -The application replaces `${INSTANCE_NAME}` and `${DEPLOYMENT_NAME}` in the `AZURE_OPENAI_BASEURL`, processed according to the other settings discussed in the guide. - -**You can also omit the placeholders completely and simply construct the baseURL with your credentials:** - -```bash -# .env file -AZURE_OPENAI_BASEURL=https://instance-1.openai.azure.com/openai/deployments/deployment-1 - -# Cloudflare example -AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/instance-1/deployment-1 -``` - -Setting these values will override all of the application's internal handling of the instance and deployment names and use your specified base URL. - -**Notes:** -- You should still provide the `AZURE_OPENAI_API_VERSION` and `AZURE_API_KEY` via the .env file as they are programmatically added to the requests. -- When specifying instance and deployment names in the `AZURE_OPENAI_BASEURL`, their respective environment variables can be omitted (`AZURE_OPENAI_API_INSTANCE_NAME` and `AZURE_OPENAI_API_DEPLOYMENT_NAME`) except for use with Plugins. -- Specifying instance and deployment names in the `AZURE_OPENAI_BASEURL` instead of placeholders creates conflicts with "plugins," "vision," "default-model," and "model-as-deployment-name" support. -- Due to the conflicts that arise with other features, it is recommended to use placeholder for instance and deployment names in the `AZURE_OPENAI_BASEURL` - -### Enabling Auto-Generated Titles with Azure - -The default titling model is set to `gpt-3.5-turbo`. - -If you're using `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME` and have "gpt-35-turbo" setup as a deployment name, this should work out-of-the-box. - -In any case, you can adjust the title model as such: `OPENAI_TITLE_MODEL=your-title-model` - -### Using GPT-4 Vision with Azure - -Currently, the best way to setup Vision is to use your deployment names as the model names, as [shown here](#model-deployments) - -This will work seamlessly as it does with the [OpenAI endpoint](#openai) (no need to select the vision model, it will be switched behind the scenes) - -Alternatively, you can set the [required variables](#required-variables) to explicitly use your vision deployment, but this may limit you to exclusively using your vision deployment for all Azure chat settings. - - -**Notes:** - -- If using `AZURE_OPENAI_BASEURL`, you should not specify instance and deployment names instead of placeholders as the vision request will fail. -- As of December 18th, 2023, Vision models seem to have degraded performance with Azure OpenAI when compared to [OpenAI](#openai) - -![image](https://github.com/danny-avila/LibreChat/assets/110412045/7306185f-c32c-4483-9167-af514cc1c2dd) - - -> Note: a change will be developed to improve current configuration settings, to allow multiple deployments/model configurations setup with ease: **[#1390](https://github.com/danny-avila/LibreChat/issues/1390)** - -### Generate images with Azure OpenAI Service (DALL-E) - -| Model ID | Feature Availability | Max Request (characters) | -|----------|----------------------|-------------------------| -| dalle2 | East US | 1000 | -| dalle3 | Sweden Central | 4000 | - -- First you need to create an Azure resource that hosts DALL-E - - At the time of writing, dall-e-3 is available in the `SwedenCentral` region, dall-e-2 in the `EastUS` region. -- Then, you need to deploy the image generation model in one of the above regions. - - Read the [Azure OpenAI Image Generation Quickstart Guide](https://learn.microsoft.com/en-us/azure/ai-services/openai/dall-e-quickstart) for further assistance -- Configure your environment variables based on Azure credentials: - -**- For DALL-E-3:** - -```bash -DALLE3_AZURE_API_VERSION=the-api-version # e.g.: 2023-12-01-preview -DALLE3_BASEURL=https://.openai.azure.com/openai/deployments// -DALLE3_API_KEY=your-azure-api-key-for-dall-e-3 -``` - -**- For DALL-E-2:** - -```bash -DALLE2_AZURE_API_VERSION=the-api-version # e.g.: 2023-12-01-preview -DALLE2_BASEURL=https://.openai.azure.com/openai/deployments// -DALLE2_API_KEY=your-azure-api-key-for-dall-e-2 -``` - -**DALL-E Notes:** - -- For DALL-E-3, the default system prompt has the LLM prefer the ["vivid" style](https://platform.openai.com/docs/api-reference/images/create#images-create-style) parameter, which seems to be the preferred setting for ChatGPT as "natural" can sometimes produce lackluster results. -- See official prompt for reference: **[DALL-E System Prompt](https://github.com/spdustin/ChatGPT-AutoExpert/blob/main/_system-prompts/dall-e.md)** -- You can adjust the system prompts to your liking: - -```bash -DALLE3_SYSTEM_PROMPT="Your DALL-E-3 System Prompt here" -DALLE2_SYSTEM_PROMPT="Your DALL-E-2 System Prompt here" -``` - -- The `DALLE_REVERSE_PROXY` environment variable is ignored when Azure credentials (DALLEx_AZURE_API_VERSION and DALLEx_BASEURL) for DALL-E are configured. - -### Optional Variables - -*These variables are currently not used by LibreChat* - -* `AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME`: The deployment name for completion. This is currently not in use but may be used in future. -* `AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME`: The deployment name for embedding. This is currently not in use but may be used in future. - -These two variables are optional but may be used in future updates of this project. - -### Using Plugins with Azure - -Note: To use the Plugins endpoint with Azure OpenAI, you need a deployment supporting **[function calling](https://techcommunity.microsoft.com/t5/azure-ai-services-blog/function-calling-is-now-available-in-azure-openai-service/ba-p/3879241)**. Otherwise, you need to set "Functions" off in the Agent settings. When you are not using "functions" mode, it's recommend to have "skip completion" off as well, which is a review step of what the agent generated. - -To use Azure with the Plugins endpoint, make sure the following environment variables are set: - -* `PLUGINS_USE_AZURE`: If set to "true" or any truthy value, this will enable the program to use Azure with the Plugins endpoint. -* `AZURE_API_KEY`: Your Azure API key must be set with an environment variable. - -**Important:** - -- If using `AZURE_OPENAI_BASEURL`, you should not specify instance and deployment names instead of placeholders as the plugin request will fail. +This was done to improve upon legacy configuration settings, to allow multiple deployments/model configurations setup with ease: **[#1390](https://github.com/danny-avila/LibreChat/issues/1390)** --- @@ -476,6 +264,10 @@ OpenRouter is integrated to the LibreChat by overriding the OpenAI endpoint. #### Setup (legacy): +**Note:** It is NOT recommended to setup OpenRouter this way with versions 0.6.6 or higher of LibreChat as it may be removed in future versions. + +As noted earlier, [review the Custom Config Guide (click here)](./custom_config.md) to add an `OpenRouter` Endpoint instead. + - Signup to **[OpenRouter](https://openrouter.ai/)** and create a key. You should name it and set a limit as well. - Set the environment variable `OPENROUTER_API_KEY` in your .env file to the key you just created. - Set something in the `OPENAI_API_KEY`, it can be anyting, but **do not** leave it blank or set to `user_provided` diff --git a/docs/install/configuration/azure_openai.md b/docs/install/configuration/azure_openai.md new file mode 100644 index 00000000000..4c36f135ee1 --- /dev/null +++ b/docs/install/configuration/azure_openai.md @@ -0,0 +1,406 @@ +# Azure OpenAI + +**Azure OpenAI Integration for LibreChat** + +To properly utilize Azure OpenAI within LibreChat, it's crucial to configure the [`librechat.yaml` file](./custom_config.md#azure-openai-object-structure) according to your specific needs. This document guides you through the essential setup process which allows seamless use of multiple deployments and models with as much flexibility as needed. + +## Setup + +1. **Open `librechat.yaml` for Editing**: Use your preferred text editor or IDE to open and edit the `librechat.yaml` file. + +2. **Configure Azure OpenAI Settings**: Follow the detailed structure outlined below to populate your Azure OpenAI settings appropriately. This includes specifying API keys, instance names, model groups, and other essential configurations. + +3. **Save Your Changes**: After accurately inputting your settings, save the `librechat.yaml` file. + +4. **Restart LibreChat**: For the changes to take effect, restart your LibreChat application. This ensures that the updated configurations are loaded and utilized. + +Here's a working example configured according to the specifications of the [Azure OpenAI Endpoint Configuration Docs:](./custom_config.md#azure-openai-object-structure) + +## Required Fields + +To properly integrate Azure OpenAI with LibreChat, specific fields must be accurately configured in your `librechat.yaml` file. These fields are validated through a combination of custom and environmental variables to ensure the correct setup. Here are the detailed requirements based on the validation process: + +### Group-Level Configuration + +1. **group** (String, Required): Unique identifier name for a group of models. Duplicate group names are not allowed and will result in validation errors. + +2. **apiKey** (String, Required): Must be a valid API key for Azure OpenAI services. It could be a direct key string or an environment variable reference (e.g., `${WESTUS_API_KEY}`). + +3. **instanceName** (String, Required): Name of the Azure OpenAI instance. This field can also support environment variable references. + +4. **deploymentName** (String, Optional): The deployment name at the group level is optional but required if any model within the group is set to `true`. + +5. **version** (String, Optional): The version of the Azure OpenAI service at the group level is optional but required if any model within the group is set to `true`. + +6. **baseURL** (String, Optional): Custom base URL for the Azure OpenAI API requests. Environment variable references are supported. This is optional and can be used for advanced routing scenarios. + +7. **additionalHeaders** (Object, Optional): Specifies any extra headers for Azure OpenAI API requests as key-value pairs. Environment variable references can be included as values. + +### Model-Level Configuration + +Within each group, the `models` field must contain a mapping of records, or model identifiers to either boolean values or object configurations. + +- The key or model identifier must match its corresponding OpenAI model name in order for it to properly reflect its known context limits and/or function in the case of vision. For example, if you intend to use gpt-4-vision, it must be configured like so: + +```yaml +models: + gpt-4-vision-preview: # matching OpenAI Model name + deploymentName: "arbitrary-deployment-name" + version: "2024-02-15-preview" # version can be any that supports vision +``` + +- See [Model Deployments](#model-deployments) for more examples. + +- If a model is set to `true`, it implies using the group-level `deploymentName` and `version` for this model. Both must be defined at the group level in this case. + +- If a model is configured as an object, it can specify its own `deploymentName` and `version`. If these are not provided, the model inherits the group's `deploymentName` and `version`. + +### Special Considerations + +1. **Unique Names**: Both model and group names must be unique across the entire configuration. Duplicate names lead to validation failures. + +2. **Missing Required Fields**: Lack of required `deploymentName` or `version` either at the group level (for boolean-flagged models) or within the models' configurations (if not inheriting or explicitly specified) will result in validation errors. + +3. **Environment Variable References**: The configuration supports environment variable references (e.g., `${VARIABLE_NAME}`). Ensure that all referenced variables are present in your environment to avoid runtime errors. The absence of defined environment variables referenced in the config will cause errors.`${INSTANCE_NAME}` and `${DEPLOYMENT_NAME}` are unique placeholders, and do not correspond to environment variables, but instead correspond to the instance and deployment name of the currently selected model. It is not recommended you use `INSTANCE_NAME` and `DEPLOYMENT_NAME` as environment variable names to avoid any potential conflicts. + +4. **Error Handling**: Any issues in the config, like duplicate names, undefined environment variables, or missing required fields, will invalidate the setup and generate descriptive error messages aiming for prompt resolution. You will not be allowed to run the server with an invalid configuration. + +Applying these setup requirements thoughtfully will ensure a correct and efficient integration of Azure OpenAI services with LibreChat through the `librechat.yaml` configuration. Always validate your configuration against the latest schema definitions and guidelines to maintain compatibility and functionality. + + +### Model Deployments + +The list of models available to your users are determined by the model groupings specified in your [`azureOpenAI` endpoint config.](./custom_config.md#models-1) + +For example: + +```yaml +# Example Azure OpenAI Object Structure +endpoints: + azureOpenAI: + groups: + - group: "my-westus" # arbitrary name + apiKey: "${WESTUS_API_KEY}" + instanceName: "actual-instance-name" # name of the resource group or instance + version: "2023-12-01-preview" + models: + gpt-4-vision-preview: + deploymentName: gpt-4-vision-preview + version: "2024-02-15-preview" + gpt-3.5-turbo: true + - group: "my-eastus" + apiKey: "${EASTUS_API_KEY}" + instanceName: "actual-eastus-instance-name" + deploymentName: gpt-4-turbo + version: "2024-02-15-preview" + models: + gpt-4-turbo: true +``` + +The above configuration would enable `gpt-4-vision-preview`, `gpt-3.5-turbo` and `gpt-4-turbo` for your users in the order they were defined. + +### Using Plugins with Azure + +To use the Plugins endpoint with Azure OpenAI, you need a deployment supporting **[function calling](https://techcommunity.microsoft.com/t5/azure-ai-services-blog/function-calling-is-now-available-in-azure-openai-service/ba-p/3879241)**. Otherwise, you need to set "Functions" off in the Agent settings. When you are not using "functions" mode, it's recommend to have "skip completion" off as well, which is a review step of what the agent generated. + +To use Azure with the Plugins endpoint, make sure the field `plugins` is set to `true` in your Azure OpenAI endpoing config: + +```yaml +# Example Azure OpenAI Object Structure +endpoints: + azureOpenAI: + plugins: true # <------- Set this + groups: + # omitted for brevity +``` + +Configuring the `plugins` field will configure Plugins to use Azure models. + +**NOTE**: The current configuration through `librechat.yaml` uses the primary model you select from the frontend for Plugin use, which is not usually how it works without Azure, where instead the "Agent" model is used. The Agent model setting can be ignored when using Plugins through Azure. + +### Using a Specified Base URL with Azure + +The base URL for Azure OpenAI API requests can be dynamically configured. This is useful for proxying services such as [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/providers/azureopenai/), or if you wish to explicitly override the baseURL handling of the app. + +LibreChat will use the baseURL field for your Azure model grouping, which can include placeholders for the Azure OpenAI API instance and deployment names. + +In the configuration, the base URL can be customized like so: + +```yaml +# librechat.yaml file, under an Azure group: +endpoints: + azureOpenAI: + groups: + - group: "group-with-custom-base-url" + baseURL: "https://example.azure-api.net/${INSTANCE_NAME}/${DEPLOYMENT_NAME}" + +# OR + baseURL: "https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME}" + +# Cloudflare example + baseURL: "https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME}" +``` + +**NOTE**: `${INSTANCE_NAME}` and `${DEPLOYMENT_NAME}` are unique placeholders, and do not correspond to environment variables, but instead correspond to the instance and deployment name of the currently selected model. It is not recommended you use INSTANCE_NAME and DEPLOYMENT_NAME as environment variable names to avoid any potential conflicts. + +**You can also omit the placeholders completely and simply construct the baseURL with your credentials:** + +```yaml + baseURL: "https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/my-secret-instance/my-deployment" +``` +**Lastly, you can specify the entire baseURL through a custom environment variable** + +```yaml + baseURL: "${MY_CUSTOM_BASEURL}" +``` + + +### Enabling Auto-Generated Titles with Azure + +To enable titling for Azure, set `titleConvo` to `true`. + +```yaml +# Example Azure OpenAI Object Structure +endpoints: + azureOpenAI: + titleConvo: true # <------- Set this + groups: + # omitted for brevity +``` + +**You can also specify the model to use for titling, with `titleModel`** provided you have configured it in your group(s). + +```yaml + titleModel: "gpt-3.5-turbo" +``` + +**Note**: "gpt-3.5-turbo" is the default value, so you can omit it if you want to use this exact model and have it configured. If not configured and `titleConvo` is set to `true`, the titling process will result in an error and no title will be generated. + + +### Using GPT-4 Vision with Azure + +To use Vision (image analysis) with Azure OpenAI, you need to make sure `gpt-4-vision-preview` is a specified model [in one of your groupings](#model-deployments) + +This will work seamlessly as it does with the [OpenAI endpoint](#openai) (no need to select the vision model, it will be switched behind the scenes) + +### Generate images with Azure OpenAI Service (DALL-E) + +| Model ID | Feature Availability | Max Request (characters) | +|----------|----------------------|-------------------------| +| dalle2 | East US | 1000 | +| dalle3 | Sweden Central | 4000 | + +- First you need to create an Azure resource that hosts DALL-E + - At the time of writing, dall-e-3 is available in the `SwedenCentral` region, dall-e-2 in the `EastUS` region. +- Then, you need to deploy the image generation model in one of the above regions. + - Read the [Azure OpenAI Image Generation Quickstart Guide](https://learn.microsoft.com/en-us/azure/ai-services/openai/dall-e-quickstart) for further assistance +- Configure your environment variables based on Azure credentials: + +**- For DALL-E-3:** + +```bash +DALLE3_AZURE_API_VERSION=the-api-version # e.g.: 2023-12-01-preview +DALLE3_BASEURL=https://.openai.azure.com/openai/deployments// +DALLE3_API_KEY=your-azure-api-key-for-dall-e-3 +``` + +**- For DALL-E-2:** + +```bash +DALLE2_AZURE_API_VERSION=the-api-version # e.g.: 2023-12-01-preview +DALLE2_BASEURL=https://.openai.azure.com/openai/deployments// +DALLE2_API_KEY=your-azure-api-key-for-dall-e-2 +``` + +**DALL-E Notes:** + +- For DALL-E-3, the default system prompt has the LLM prefer the ["vivid" style](https://platform.openai.com/docs/api-reference/images/create#images-create-style) parameter, which seems to be the preferred setting for ChatGPT as "natural" can sometimes produce lackluster results. +- See official prompt for reference: **[DALL-E System Prompt](https://github.com/spdustin/ChatGPT-AutoExpert/blob/main/_system-prompts/dall-e.md)** +- You can adjust the system prompts to your liking: + +```bash +DALLE3_SYSTEM_PROMPT="Your DALL-E-3 System Prompt here" +DALLE2_SYSTEM_PROMPT="Your DALL-E-2 System Prompt here" +``` + +- The `DALLE_REVERSE_PROXY` environment variable is ignored when Azure credentials (DALLEx_AZURE_API_VERSION and DALLEx_BASEURL) for DALL-E are configured. + +--- + +## ⚠️ Legacy Setup ⚠️ + +--- + +**Note:** The legacy instructions may be used for a simple setup but they are no longer recommended as of v0.7.0 and may break in future versions. This was done to improve upon legacy configuration settings, to allow multiple deployments/model configurations setup with ease: **[#1390](https://github.com/danny-avila/LibreChat/issues/1390)** + +**Use the recommended [Setup](#setup) in the section above.** + +**Required Variables (legacy)** + +These variables construct the API URL for Azure OpenAI. + +* `AZURE_API_KEY`: Your Azure OpenAI API key. +* `AZURE_OPENAI_API_INSTANCE_NAME`: The instance name of your Azure OpenAI API. +* `AZURE_OPENAI_API_DEPLOYMENT_NAME`: The deployment name of your Azure OpenAI API. +* `AZURE_OPENAI_API_VERSION`: The version of your Azure OpenAI API. + +For example, with these variables, the URL for chat completion would look something like: +```plaintext +https://{AZURE_OPENAI_API_INSTANCE_NAME}.openai.azure.com/openai/deployments/{AZURE_OPENAI_API_DEPLOYMENT_NAME}/chat/completions?api-version={AZURE_OPENAI_API_VERSION} +``` +You should also consider changing the `AZURE_OPENAI_MODELS` variable to the models available in your deployment. + +```bash +# .env file +AZURE_OPENAI_MODELS=gpt-4-1106-preview,gpt-4,gpt-3.5-turbo,gpt-3.5-turbo-1106,gpt-4-vision-preview +``` + +Overriding the construction of the API URL is possible as of implementing **[Issue #1266](https://github.com/danny-avila/LibreChat/issues/1266)** + +**Model Deployments (legacy)** + +> Note: a change will be developed to improve current configuration settings, to allow multiple deployments/model configurations setup with ease: **[#1390](https://github.com/danny-avila/LibreChat/issues/1390)** + +As of 2023-12-18, the Azure API allows only one model per deployment. + +**It's highly recommended** to name your deployments *after* the model name (e.g., "gpt-3.5-turbo") for easy deployment switching. + +When you do so, LibreChat will correctly switch the deployment, while associating the correct max context per model, if you have the following environment variable set: + +```bash +AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE +``` + +For example, when you have set `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE`, the following deployment configuration provides the most seamless, error-free experience for LibreChat, including Vision support and tracking the correct max context tokens: + +![Screenshot 2023-12-18 111742](https://github.com/danny-avila/LibreChat/assets/110412045/4aa8a61c-0317-4681-8262-a6382dcaa7b0) + + +Alternatively, you can use custom deployment names and set `AZURE_OPENAI_DEFAULT_MODEL` for expected functionality. + +- **`AZURE_OPENAI_MODELS`**: List the available models, separated by commas without spaces. The first listed model will be the default. If left blank, internal settings will be used. Note that deployment names can't have periods, which are removed when generating the endpoint. + +Example use: + +```bash +# .env file +AZURE_OPENAI_MODELS=gpt-3.5-turbo,gpt-4,gpt-5 + +``` + +- **`AZURE_USE_MODEL_AS_DEPLOYMENT_NAME`**: Enable using the model name as the deployment name for the API URL. + +Example use: + +```bash +# .env file +AZURE_USE_MODEL_AS_DEPLOYMENT_NAME=TRUE + +``` + +**Setting a Default Model for Azure (legacy)** + +This section is relevant when you are **not** naming deployments after model names as shown above. + +**Important:** The Azure OpenAI API does not use the `model` field in the payload but is a necessary identifier for LibreChat. If your deployment names do not correspond to the model names, and you're having issues with the model not being recognized, you should set this field to explicitly tell LibreChat to treat your Azure OpenAI API requests as if the specified model was selected. + +If AZURE_USE_MODEL_AS_DEPLOYMENT_NAME is enabled, the model you set with `AZURE_OPENAI_DEFAULT_MODEL` will **not** be recognized and will **not** be used as the deployment name; instead, it will use the model selected by the user as the "deployment" name. + +- **`AZURE_OPENAI_DEFAULT_MODEL`**: Override the model setting for Azure, useful if using custom deployment names. + +Example use: + +```bash +# .env file +# MUST be a real OpenAI model, named exactly how it is recognized by OpenAI API (not Azure) +AZURE_OPENAI_DEFAULT_MODEL=gpt-3.5-turbo # do include periods in the model name here + +``` + +**Using a Specified Base URL with Azure (legacy)** + +The base URL for Azure OpenAI API requests can be dynamically configured. This is useful for proxying services such as [Cloudflare AI Gateway](https://developers.cloudflare.com/ai-gateway/providers/azureopenai/), or if you wish to explicitly override the baseURL handling of the app. + +LibreChat will use the `AZURE_OPENAI_BASEURL` environment variable, which can include placeholders for the Azure OpenAI API instance and deployment names. + +In the application's environment configuration, the base URL is set like this: + +```bash +# .env file +AZURE_OPENAI_BASEURL=https://example.azure-api.net/${INSTANCE_NAME}/${DEPLOYMENT_NAME} + +# OR +AZURE_OPENAI_BASEURL=https://${INSTANCE_NAME}.openai.azure.com/openai/deployments/${DEPLOYMENT_NAME} + +# Cloudflare example +AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME} +``` + +The application replaces `${INSTANCE_NAME}` and `${DEPLOYMENT_NAME}` in the `AZURE_OPENAI_BASEURL`, processed according to the other settings discussed in the guide. + +**You can also omit the placeholders completely and simply construct the baseURL with your credentials:** + +```bash +# .env file +AZURE_OPENAI_BASEURL=https://instance-1.openai.azure.com/openai/deployments/deployment-1 + +# Cloudflare example +AZURE_OPENAI_BASEURL=https://gateway.ai.cloudflare.com/v1/ACCOUNT_TAG/GATEWAY/azure-openai/instance-1/deployment-1 +``` + +Setting these values will override all of the application's internal handling of the instance and deployment names and use your specified base URL. + +**Notes:** +- You should still provide the `AZURE_OPENAI_API_VERSION` and `AZURE_API_KEY` via the .env file as they are programmatically added to the requests. +- When specifying instance and deployment names in the `AZURE_OPENAI_BASEURL`, their respective environment variables can be omitted (`AZURE_OPENAI_API_INSTANCE_NAME` and `AZURE_OPENAI_API_DEPLOYMENT_NAME`) except for use with Plugins. +- Specifying instance and deployment names in the `AZURE_OPENAI_BASEURL` instead of placeholders creates conflicts with "plugins," "vision," "default-model," and "model-as-deployment-name" support. +- Due to the conflicts that arise with other features, it is recommended to use placeholder for instance and deployment names in the `AZURE_OPENAI_BASEURL` + +**Enabling Auto-Generated Titles with Azure (legacy)** + +The default titling model is set to `gpt-3.5-turbo`. + +If you're using `AZURE_USE_MODEL_AS_DEPLOYMENT_NAME` and have "gpt-35-turbo" setup as a deployment name, this should work out-of-the-box. + +In any case, you can adjust the title model as such: `OPENAI_TITLE_MODEL=your-title-model` + +**Using GPT-4 Vision with Azure (legacy)** + +Currently, the best way to setup Vision is to use your deployment names as the model names, as [shown here](#model-deployments) + +This will work seamlessly as it does with the [OpenAI endpoint](#openai) (no need to select the vision model, it will be switched behind the scenes) + +Alternatively, you can set the [required variables](#required-variables) to explicitly use your vision deployment, but this may limit you to exclusively using your vision deployment for all Azure chat settings. + + +**Notes:** + +- If using `AZURE_OPENAI_BASEURL`, you should not specify instance and deployment names instead of placeholders as the vision request will fail. +- As of December 18th, 2023, Vision models seem to have degraded performance with Azure OpenAI when compared to [OpenAI](#openai) + +![image](https://github.com/danny-avila/LibreChat/assets/110412045/7306185f-c32c-4483-9167-af514cc1c2dd) + + +> Note: a change will be developed to improve current configuration settings, to allow multiple deployments/model configurations setup with ease: **[#1390](https://github.com/danny-avila/LibreChat/issues/1390)** + +**Optional Variables (legacy)** + +*These variables are currently not used by LibreChat* + +* `AZURE_OPENAI_API_COMPLETIONS_DEPLOYMENT_NAME`: The deployment name for completion. This is currently not in use but may be used in future. +* `AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME`: The deployment name for embedding. This is currently not in use but may be used in future. + +These two variables are optional but may be used in future updates of this project. + +**Using Plugins with Azure** + +Note: To use the Plugins endpoint with Azure OpenAI, you need a deployment supporting **[function calling](https://techcommunity.microsoft.com/t5/azure-ai-services-blog/function-calling-is-now-available-in-azure-openai-service/ba-p/3879241)**. Otherwise, you need to set "Functions" off in the Agent settings. When you are not using "functions" mode, it's recommend to have "skip completion" off as well, which is a review step of what the agent generated. + +To use Azure with the Plugins endpoint, make sure the following environment variables are set: + +* `PLUGINS_USE_AZURE`: If set to "true" or any truthy value, this will enable the program to use Azure with the Plugins endpoint. +* `AZURE_API_KEY`: Your Azure API key must be set with an environment variable. + +**Important:** + +- If using `AZURE_OPENAI_BASEURL`, you should not specify instance and deployment names instead of placeholders as the plugin request will fail. \ No newline at end of file diff --git a/docs/install/configuration/custom_config.md b/docs/install/configuration/custom_config.md index 27d99e97567..6f7a5082858 100644 --- a/docs/install/configuration/custom_config.md +++ b/docs/install/configuration/custom_config.md @@ -239,30 +239,18 @@ rateLimits: - **Key**: `endpoints` - **Type**: Object - **Description**: Defines custom API endpoints for the application. - - **Sub-Key**: `assistants` - - **Type**: Object - - **Description**: Assistants endpoint-specific configuration. - - **Sub-Key**: `disableBuilder` - - **Description**: Controls the visibility and use of the builder interface for assistants. - - [More info](#disablebuilder) - - **Sub-Key**: `pollIntervalMs` - - **Description**: Specifies the polling interval in milliseconds for checking run updates or changes in assistant run states. - - [More info](#pollintervalms) - - **Sub-Key**: `timeoutMs` - - **Description**: Sets a timeout in milliseconds for assistant runs. Helps manage system load by limiting total run operation time. - - [More info](#timeoutMs) - - **Sub-Key**: `supportedIds` - - **Description**: List of supported assistant Ids. Use this or `excludedIds` but not both. - - [More info](#supportedIds) - - **Sub-Key**: `excludedIds` - - **Description**: List of excluded assistant Ids. Use this or `supportedIds` but not both (the `excludedIds` field will be ignored if so). - - [More info](#excludedIds) - - [Full Assistants Endpoint Object Structure](#assistants-endpoint-object-structure) - **Sub-Key**: `custom` - **Type**: Array of Objects - **Description**: Each object in the array represents a unique endpoint configuration. - [Full Custom Endpoint Object Structure](#custom-endpoint-object-structure) -- **Required** + - **Sub-Key**: `azureOpenAI` + - **Type**: Object + - **Description**: Azure OpenAI endpoint-specific configuration + - [Full Azure OpenAI Endpoint Object Structure](#azure-openai-object-structure) + - **Sub-Key**: `assistants` + - **Type**: Object + - **Description**: Assistants endpoint-specific configuration. + - [Full Assistants Endpoint Object Structure](#assistants-endpoint-object-structure) ## Endpoint File Config Object Structure @@ -723,3 +711,188 @@ Custom endpoints share logic with the OpenAI endpoint, and thus have default par **Note:** The `max_tokens` field is not sent to use the maximum amount of tokens available, which is default OpenAI API behavior. Some alternate APIs require this field, or it may default to a very low value and your responses may appear cut off; in this case, you should add it to `addParams` field as shown in the [Endpoint Object Structure](#endpoint-object-structure). +## Azure OpenAI Object Structure + +Integrating Azure OpenAI Service with your application allows you to seamlessly utilize multiple deployments and region models hosted by Azure OpenAI. This section details how to configure the Azure OpenAI endpoint for your needs. + +**[For a detailed guide on setting up Azure OpenAI configurations, click here](./azure_openai.md)** + +### Example Configuration + +```yaml +# Example Azure OpenAI Object Structure +endpoints: + azureOpenAI: + titleModel: "gpt-4-turbo" + plugins: true + groups: + - group: "my-westus" # arbitrary name + apiKey: "${WESTUS_API_KEY}" + instanceName: "actual-instance-name" # name of the resource group or instance + version: "2023-12-01-preview" + # baseURL: https://prod.example.com + # additionalHeaders: + # X-Custom-Header: value + models: + gpt-4-vision-preview: + deploymentName: gpt-4-vision-preview + version: "2024-02-15-preview" + gpt-3.5-turbo: + deploymentName: gpt-35-turbo + gpt-3.5-turbo-1106: + deploymentName: gpt-35-turbo-1106 + gpt-4: + deploymentName: gpt-4 + gpt-4-1106-preview: + deploymentName: gpt-4-1106-preview + - group: "my-eastus" + apiKey: "${EASTUS_API_KEY}" + instanceName: "actual-eastus-instance-name" + deploymentName: gpt-4-turbo + version: "2024-02-15-preview" + baseURL: "https://gateway.ai.cloudflare.com/v1/cloudflareId/azure/azure-openai/${INSTANCE_NAME}/${DEPLOYMENT_NAME}" # uses env variables + additionalHeaders: + X-Custom-Header: value + models: + gpt-4-turbo: true +``` + +### **groups**: + +> Configuration for groups of models by geographic location or purpose. + +- Type: Array +- **Description**: Each item in the `groups` array configures a set of models under a certain grouping, often by geographic region or distinct configuration. +- **Example**: See above. + +### **plugins**: + +> Enables or disables plugins for the Azure OpenAI endpoint. + +- Type: Boolean +- **Example**: `plugins: true` +- **Description**: When set to `true`, activates plugins associated with this endpoint. + +### Group Configuration Parameters + +#### **group**: + + > Identifier for a group of models. + + - Type: String + - **Required** + - **Example**: `"my-westus"` + +#### **apiKey**: + + > The API key for accessing the Azure OpenAI Service. + + - Type: String + - **Required** + - **Example**: `"${WESTUS_API_KEY}"` + - **Note**: It's highly recommended to use a custom env. variable reference for this field, i.e. `${YOUR_VARIABLE}` + + +#### **instanceName**: + + > Name of the Azure instance. + + - Type: String + - **Required** + - **Example**: `"my-westus"` + - **Note**: It's recommended to use a custom env. variable reference for this field, i.e. `${YOUR_VARIABLE}` + + +#### **version**: + + > API version. + + - Type: String + - **Optional** + - **Example**: `"2023-12-01-preview"` + - **Note**: It's recommended to use a custom env. variable reference for this field, i.e. `${YOUR_VARIABLE}` + +#### **baseURL**: + + > The base URL for the Azure OpenAI Service. + + - Type: String + - **Optional** + - **Example**: `"https://prod.example.com"` + - **Note**: It's recommended to use a custom env. variable reference for this field, i.e. `${YOUR_VARIABLE}` + +#### **additionalHeaders**: + + > Additional headers for API requests. + + - Type: Dictionary + - **Optional** + - **Example**: + ```yaml + additionalHeaders: + X-Custom-Header: ${YOUR_SECRET_CUSTOM_VARIABLE} + ``` + - **Note**: It's recommended to use a custom env. variable reference for the values of field, as shown in the example. + - **Note**: `api-key` header value is sent on every request + +#### **models**: + +> Configuration for individual models within a group. + +- **Description**: Configures settings for each model, including deployment name and version. Model configurations can adopt the group's deployment name and/or version when configured as a boolean (set to `true`) or an object for detailed settings of either of those fields. +- **Example**: See above example configuration. + +Within each group, models are records, either set to true, or set with a specific `deploymentName` and/or `version` where the key MUST be the matching OpenAI model name; for example, if you intend to use gpt-4-vision, it must be configured like so: + +```yaml +models: + gpt-4-vision-preview: # matching OpenAI Model name + deploymentName: "arbitrary-deployment-name" + version: "2024-02-15-preview" # version can be any that supports vision +``` + +### Model Configuration Parameters + +#### **deploymentName**: + +> The name of the deployment for the model. + +- Type: String +- **Required** +- **Example**: `"gpt-4-vision-preview"` +- **Description**: Identifies the deployment of the model within Azure. +- **Note**: This does not have to be the matching OpenAI model name as is convention, but must match the actual name of your deployment on Azure. + +#### **version**: + +> Specifies the version of the model. + +- Type: String +- **Required** +- **Example**: `"2024-02-15-preview"` +- **Description**: Defines the version of the model to be used. + +**When specifying a model as a boolean (`true`):** + +When a model is enabled (`true`) without using an object, it uses the group's configuration values for deployment name and version. + +**Example**: +```yaml +models: + gpt-4-turbo: true +``` + +**When specifying a model as an object:** + +An object allows for detailed configuration of the model, including its `deploymentName` and/or `version`. This mode is used for more granular control over the models, especially when working with multiple versions or deployments under one instance or resource group. + +**Example**: +```yaml +models: + gpt-4-vision-preview: + deploymentName: "gpt-4-vision-preview" + version: "2024-02-15-preview" +``` + +### Notes: +- **Deployment Names** and **Versions** are critical for ensuring that the correct model is used. Double-check these values for accuracy to prevent unexpected behavior. diff --git a/docs/install/configuration/index.md b/docs/install/configuration/index.md index dacc6605cd6..a369308871e 100644 --- a/docs/install/configuration/index.md +++ b/docs/install/configuration/index.md @@ -11,6 +11,7 @@ weight: 2 * 🐋 [Docker Compose Override](./docker_override.md) --- * 🤖 [AI Setup](./ai_setup.md) + * 🅰️ [Azure OpenAI](./azure_openai.md) * 🚅 [LiteLLM](./litellm.md) * 💸 [Free AI APIs](./free_ai_apis.md) --- diff --git a/docs/install/index.md b/docs/install/index.md index 7ff5503678a..ba43912e011 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -20,6 +20,7 @@ weight: 1 * 🖥️ [Custom Endpoints & Config](./configuration/custom_config.md) * 🐋 [Docker Compose Override](./configuration/docker_override.md) * 🤖 [AI Setup](./configuration/ai_setup.md) + * 🅰️ [Azure OpenAI](./configuration/azure_openai.md) * 🚅 [LiteLLM](./configuration/litellm.md) * 💸 [Free AI APIs](./configuration/free_ai_apis.md) * 🛂 [Authentication System](./configuration/user_auth_system.md) From 6c4d24b43b6ba916e91d7e5f540b29e14c45afa8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 26 Feb 2024 11:39:02 -0500 Subject: [PATCH 32/35] feat: safe check of potential conflicting env vars that map to unique placeholders --- api/server/services/AppService.js | 9 ++++++++ api/server/services/AppService.spec.js | 29 +++++++++++++++++++++++++- packages/data-provider/src/azure.ts | 9 ++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index d03cf959055..15e19428da8 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -6,6 +6,7 @@ const { validateAzureGroups, mapModelToAzureConfig, deprecatedAzureVariables, + conflictingAzureVariables, } = require('librechat-data-provider'); const { initializeFirebase } = require('./Files/Firebase/initialize'); const loadCustomConfig = require('./Config/loadCustomConfig'); @@ -99,6 +100,14 @@ const AppService = async (app) => { ); } }); + + conflictingAzureVariables.forEach(({ key }) => { + if (process.env[key]) { + logger.warn( + `The \`${key}\` environment variable should not be used in combination with the \`azureOpenAI\` endpoint configuration, as you may experience with the defined placeholders for mapping to the current model grouping using the same name.`, + ); + } + }); } if (config?.endpoints?.[EModelEndpoint.assistants]) { diff --git a/api/server/services/AppService.spec.js b/api/server/services/AppService.spec.js index 39d1c12673b..3a40a49b3e7 100644 --- a/api/server/services/AppService.spec.js +++ b/api/server/services/AppService.spec.js @@ -4,6 +4,7 @@ const { defaultSocialLogins, validateAzureGroups, deprecatedAzureVariables, + conflictingAzureVariables, } = require('librechat-data-provider'); const AppService = require('./AppService'); @@ -381,7 +382,7 @@ describe('AppService updating app.locals and issuing warnings', () => { ); }); - it('should issue expected warnings when loading Azure Groups', async () => { + it('should issue expected warnings when loading Azure Groups with deprecated Environment Variables', async () => { require('./Config/loadCustomConfig').mockImplementationOnce(() => Promise.resolve({ endpoints: { @@ -406,4 +407,30 @@ describe('AppService updating app.locals and issuing warnings', () => { ); }); }); + + it('should issue expected warnings when loading conflicting Azure Envrionment Variables', async () => { + require('./Config/loadCustomConfig').mockImplementationOnce(() => + Promise.resolve({ + endpoints: { + [EModelEndpoint.azureOpenAI]: { + groups: azureGroups, + }, + }, + }), + ); + + conflictingAzureVariables.forEach((varInfo) => { + process.env[varInfo.key] = 'test'; + }); + + const app = { locals: {} }; + await require('./AppService')(app); + + const { logger } = require('~/config'); + conflictingAzureVariables.forEach(({ key }) => { + expect(logger.warn).toHaveBeenCalledWith( + `The \`${key}\` environment variable should not be used in combination with the \`azureOpenAI\` endpoint configuration, as you may experience with the defined placeholders for mapping to the current model grouping using the same name.`, + ); + }); + }); }); diff --git a/packages/data-provider/src/azure.ts b/packages/data-provider/src/azure.ts index 2ecfb18a1a8..87ef9f8522f 100644 --- a/packages/data-provider/src/azure.ts +++ b/packages/data-provider/src/azure.ts @@ -37,6 +37,15 @@ export const deprecatedAzureVariables = [ }, ]; +export const conflictingAzureVariables = [ + { + key: 'INSTANCE_NAME', + }, + { + key: 'DEPLOYMENT_NAME', + }, +]; + export function validateAzureGroups(configs: TAzureGroups): TValidatedAzureConfig & { isValid: boolean; errors: (ZodError | string)[]; From e6c2bd42ad14df3a3abafa391adab3a10b2dcfe4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 26 Feb 2024 12:49:57 -0500 Subject: [PATCH 33/35] fix: reset apiKey when model switches from originally requested model (vision or title) --- api/app/clients/OpenAIClient.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 3e02d346d2b..0ab75e881e7 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -755,6 +755,7 @@ class OpenAIClient extends BaseClient { this.options.headers = resolveHeaders(headers); this.options.reverseProxyUrl = baseURL ?? null; this.langchainProxy = extractBaseURL(this.options.reverseProxyUrl); + this.apiKey = azureOptions.azureOpenAIApiKey; } const titleChatCompletion = async () => { @@ -1025,6 +1026,7 @@ ${convo} this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); opts.defaultHeaders = resolveHeaders(headers); this.langchainProxy = extractBaseURL(baseURL); + this.apiKey = azureOptions.azureOpenAIApiKey; } if (this.azure || this.options.azure) { From 7141c5e896ca2b3608948fdddcaa6c65845f6f65 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 26 Feb 2024 14:00:30 -0500 Subject: [PATCH 34/35] chore: linting --- api/server/services/Config/loadCustomConfig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/server/services/Config/loadCustomConfig.js b/api/server/services/Config/loadCustomConfig.js index dc29e3329ce..7440a79d684 100644 --- a/api/server/services/Config/loadCustomConfig.js +++ b/api/server/services/Config/loadCustomConfig.js @@ -18,7 +18,7 @@ let i = 0; async function loadCustomConfig() { // Use CONFIG_PATH if set, otherwise fallback to defaultConfigPath const configPath = process.env.CONFIG_PATH || defaultConfigPath; - + const customConfig = loadYaml(configPath); if (!customConfig) { i === 0 && From ff9050812ee0e9a1e7d3eecfaa418710a98bad0c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 26 Feb 2024 14:06:21 -0500 Subject: [PATCH 35/35] docs: CONFIG_PATH notes in custom_config.md --- docs/install/configuration/custom_config.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/install/configuration/custom_config.md b/docs/install/configuration/custom_config.md index 6f7a5082858..a9e857edd2c 100644 --- a/docs/install/configuration/custom_config.md +++ b/docs/install/configuration/custom_config.md @@ -30,6 +30,12 @@ You can copy the [example config file](#example-config) as a good starting point The example config file has some options ready to go for Mistral AI and Openrouter. +**Note:** You can set an alternate filepath for the `librechat.yaml` file through an environment variable: + +```bash +CONFIG_PATH="/alternative/path/to/librechat.yaml" +``` + ## Docker Setup For Docker, you need to make use of an [override file](./docker_override.md), named `docker-compose.override.yml`, to ensure the config file works for you. @@ -46,9 +52,11 @@ version: '3.4' services: api: volumes: - - ./librechat.yaml:/app/librechat.yaml + - ./librechat.yaml:/app/librechat.yaml # local/filepath:container/filepath ``` +- **Note:** If you are using `CONFIG_PATH` for an alternative filepath for this file, make sure to specify it accordingly. + - Start docker again, and you should see your config file settings apply ```bash docker compose up # no need to rebuild