Llm provider settings

docs/agent/intro.md

@@ -16,14 +16,19 @@ Suppose we need to write a Pizza restaurant order AI Bot. First, specify a name
 BotSharp uses the latest large language model in natural language understanding, can interact with OpenAI's ChatGPT, and also supports the most widely used open source large language model [LLaMA](https://ai.meta.com/blog/large-language-model-llama-meta-ai/) and its fine-tuning model. In this example, we use [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) as the LLM engine. 
 
 ```json
-"AzureOpenAi": {
-    "ApiKey": "",
-    "Endpoint": "",
-    "DeploymentModel": {
-        "ChatCompletionModel": "",
-        "TextCompletionModel": ""
+"LlmProviders": [
+    {
+        "Provider": "azure-openai",
+        "Models": [{
+            "Name": "gpt-35-turbo",
+            "ApiKey": "",
+            "Endpoint": "https://gpt-35-turbo.openai.azure.com/",
+            "Type": "chat",
+            "PromptCost": 0.0015,
+            "CompletionCost": 0.002
+        }]
     }
-}
+]
 ```
 
 If you use the installation package to run, please ensure that the [BotSharp.Plugin.AzureOpenAI](https://www.nuget.org/packages/BotSharp.Plugin.AzureOpenAI) plugin package is installed.

docs/conf.py

@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '0.20'
+version = '0.21'
 # The full version, including alpha/beta/rc tags.
-release = '0.20.0'
+release = '0.21.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/quick-start/installation.md

@@ -19,14 +19,29 @@ PS D:\> dotnet build
 `BotSharp` can work with serveral LLM providers. Update `appsettings.json` in your project. Below config is tasking Azure OpenAI as the LLM backend
 
 ```json
-"AzureOpenAi": {
-    "ApiKey": "",
-    "Endpoint": "https://xxx.openai.azure.com/",
-    "DeploymentModel": {
-      "ChatCompletionModel": "",
-      "TextCompletionModel": ""
-    }
-}
+"LlmProviders": [
+  {
+    "Provider": "azure-openai",
+    "Models": [
+      {
+        "Name": "gpt-35-turbo",
+        "ApiKey": "",
+        "Endpoint": "https://gpt-35-turbo.openai.azure.com/",
+        "Type": "chat",
+        "PromptCost": 0.0015,
+        "CompletionCost": 0.002
+      },
+      {
+        "Name": "gpt-35-turbo-instruct",
+        "ApiKey": "",
+        "Endpoint": "https://gpt-35-turbo-instruct.openai.azure.com/",
+        "Type": "text",
+        "PromptCost": 0.0015,
+        "CompletionCost": 0.002
+      }
+    ]
+  }
+]
 ```
 
 ### Run backend web project

docs/quick-start/overview.md

@@ -21,15 +21,4 @@ Even with this simple question, you can see conversational experience are hard t
 
 Your code would have to handle all these different types of requests ro carry out the same logic: looking up some forecast information for a feature. For this reason, a traditional computer interface would tend to force users to input a well-known, standard request at the detriment of the user experience, because it's just easier.
 
-However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand.
-
-Features
--------------
-
-* Built-in multi-Agents management, easy to build Bot as a Service platform.
-* Integrate with multiple LLMs like ChatGPT and LLaMA.
-* Using plug-in design, it is easy to expand functions. 
-* Working with multiple Vector Stores for senmatic search.
-* Supporting different UI providers like [Chatbot UI](https://github.com/SciSharp/chatbot-ui) and [HuggingChat UI](https://github.com/huggingface/chat-ui).
-* Integrated with popular social platforms like Facebook Messenger, Slack and Telegram.
-* Providing REST APIs to work with your own UI.
+However, BotSharp lets you easily achieve a conversational user experience by handling the natural language understanding (NLU) for you. When you use BotSharp, you can create agents that can understand the meaning of natural language and the nuances and trainslate that to structured meaning your software can understand.

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs

@@ -14,5 +14,6 @@ public enum AgentField
     Function,
     Template,
     Response,
-    Sample
+    Sample,
+    LlmConfig
 }

src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs

@@ -11,6 +11,12 @@ public class Agent
     public DateTime CreatedDateTime { get; set; }
     public DateTime UpdatedDateTime { get; set; }
 
+    /// <summary>
+    /// Default LLM settings
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public AgentLlmConfig? LlmConfig { get; set; }
+
     /// <summary>
     /// Instruction
     /// </summary>
@@ -94,6 +100,7 @@ public static Agent Clone(Agent agent)
             AllowRouting = agent.AllowRouting,
             Profiles = agent.Profiles,
             RoutingRules = agent.RoutingRules,
+            LlmConfig = agent.LlmConfig,
             CreatedDateTime = agent.CreatedDateTime,
             UpdatedDateTime = agent.UpdatedDateTime,
         };
@@ -176,4 +183,10 @@ public Agent SetRoutingRules(List<RoutingRule> rules)
         RoutingRules = rules ?? new List<RoutingRule>();
         return this;
     }
+
+    public Agent SetLlmConfig(AgentLlmConfig? llmConfig)
+    {
+        LlmConfig = llmConfig;
+        return this;
+    }
 }

src/Infrastructure/BotSharp.Abstraction/Agents/Models/AgentLlmConfig.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Abstraction.Agents.Models;
+
+public class AgentLlmConfig
+{
+    /// <summary>
+    /// Completion Provider
+    /// </summary>
+    [JsonPropertyName("provider")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Provider { get; set; }
+
+    /// <summary>
+    /// Model name
+    /// </summary>
+    [JsonPropertyName("model")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Model { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/TokenStatsModel.cs

@@ -2,18 +2,10 @@ namespace BotSharp.Abstraction.Conversations.Models;
 
 public class TokenStatsModel
 {
+    public string Provider { get; set; }
     public string Model { get; set; }
     public string Prompt { get; set; }
     public int PromptCount { get; set; }
     public int CompletionCount { get; set; }
-
-    /// <summary>
-    /// Prompt cost per 1K token
-    /// </summary>
-    public float PromptCost { get; set; }
-
-    /// <summary>
-    /// Completion cost per 1K token
-    /// </summary>
-    public float CompletionCost { get; set; }
+    public AgentLlmConfig LlmConfig { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Settings/EvaluatorSetting.cs

@@ -2,7 +2,5 @@ namespace BotSharp.Abstraction.Evaluations.Settings;
 
 public class EvaluatorSetting
 {
-    public string EvaluatorId { get; set; }
-    public string Provider { get; set; }
-    public string Model { get; set; }
+    public string AgentId { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderSettingService.cs

@@ -0,0 +1,8 @@
+using BotSharp.Abstraction.MLTasks.Settings;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+public interface ILlmProviderSettingService
+{
+    LlmModelSetting GetSetting(string provider, string model);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -0,0 +1,30 @@
+namespace BotSharp.Abstraction.MLTasks.Settings;
+
+public class LlmModelSetting
+{
+    public string Name { get; set; }
+    public string ApiKey { get; set; }
+    public string Endpoint { get; set; }
+    public LlmModelType Type { get; set; } = LlmModelType.Chat;
+
+    /// <summary>
+    /// Prompt cost per 1K token
+    /// </summary>
+    public float PromptCost { get; set; }
+
+    /// <summary>
+    /// Completion cost per 1K token
+    /// </summary>
+    public float CompletionCost { get; set; }
+
+    public override string ToString()
+    {
+        return $"[{Type}] {Name} {Endpoint}";
+    }
+}
+
+public enum LlmModelType
+{
+    Text = 1,
+    Chat = 2
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmProviderSetting.cs

@@ -0,0 +1,15 @@
+namespace BotSharp.Abstraction.MLTasks.Settings;
+
+public class LlmProviderSetting
+{
+    public string Provider { get; set; } 
+        = "azure-openai";
+
+    public List<LlmModelSetting> Models { get; set; } 
+        = new List<LlmModelSetting>();
+
+    public override string ToString()
+    {
+        return $"{Provider} with {Models.Count} models";
+    }
+}

src/Infrastructure/BotSharp.Abstraction/Routing/Models/RoutingContext.cs

@@ -22,14 +22,14 @@ public RoutingContext(RoutingSettings setting)
     /// Agent that can handl user original goal.
     /// </summary>
     public string OriginAgentId
-        => _stack.Where(x => x != _setting.RouterId).Last();
+        => _stack.Where(x => x != _setting.AgentId).Last();
 
     public bool IsEmpty => !_stack.Any();
     public string GetCurrentAgentId()
     {
         if (_stack.Count == 0)
         {
-            _stack.Push(_setting.RouterId);
+            _stack.Push(_setting.AgentId);
         }
         return _stack.Peek();
     }

src/Infrastructure/BotSharp.Abstraction/Routing/Settings/RoutingSettings.cs

@@ -5,10 +5,7 @@ public class RoutingSettings
     /// <summary>
     /// Router Agent Id
     /// </summary>
-    public string RouterId { get; set; } = string.Empty;
+    public string AgentId { get; set; } = string.Empty;
 
     public string Planner { get; set; } = string.Empty;
-    public string Provider {  get; set; } = string.Empty;
-
-    public string Model { get; set; } = string.Empty;
 }

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs

@@ -39,7 +39,8 @@ public async Task<Agent> CreateAgent(Agent agent)
                        .SetInstruction(foundAgent.Instruction)
                        .SetTemplates(foundAgent.Templates)
                        .SetFunctions(foundAgent.Functions)
-                       .SetResponses(foundAgent.Responses);
+                       .SetResponses(foundAgent.Responses)
+                       .SetLlmConfig(foundAgent.LlmConfig);
         }
 
         var user = _db.GetUserById(_user.Id);

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs

@@ -27,6 +27,7 @@ public async Task UpdateAgent(Agent agent, AgentField updateField)
         record.Templates = agent.Templates ?? new List<AgentTemplate>();
         record.Responses = agent.Responses ?? new List<AgentResponse>();
         record.Samples = agent.Samples ?? new List<string>();
+        record.LlmConfig = agent.LlmConfig;
 
         _db.UpdateAgent(record, updateField);
         await Task.CompletedTask;
@@ -58,7 +59,8 @@ public async Task UpdateAgentFromFile(string id)
                        .SetTemplates(foundAgent.Templates)
                        .SetFunctions(foundAgent.Functions)
                        .SetResponses(foundAgent.Responses)
-                       .SetSamples(foundAgent.Samples);
+                       .SetSamples(foundAgent.Samples)
+                       .SetLlmConfig(foundAgent.LlmConfig);
 
             _db.UpdateAgent(clonedAgent, AgentField.All);
         }

src/Infrastructure/BotSharp.Core/BotSharpCoreExtensions.cs

@@ -19,6 +19,8 @@
 using BotSharp.Abstraction.MLTasks.Settings;
 using BotSharp.Abstraction.Planning;
 using BotSharp.Core.Planning;
+using BotSharp.Abstraction.MLTasks;
+using static Dapper.SqlMapper;
 
 namespace BotSharp.Core;
 
@@ -27,7 +29,7 @@ public static class BotSharpCoreExtensions
     public static IServiceCollection AddBotSharpCore(this IServiceCollection services, IConfiguration config)
     {
         services.AddScoped<IUserService, UserService>();
-
+        services.AddScoped<ILlmProviderSettingService, LlmProviderSettingService>();
         services.AddScoped<IAgentService, AgentService>();
 
         var agentSettings = new AgentSettings();
@@ -51,13 +53,16 @@ public static IServiceCollection AddBotSharpCore(this IServiceCollection service
         config.Bind("Database", myDatabaseSettings);
         services.AddSingleton((IServiceProvider x) => myDatabaseSettings);
 
-        var textCompletionSettings = new TextCompletionSetting();
-        config.Bind("TextCompletion", textCompletionSettings);
-        services.AddSingleton((IServiceProvider x) => textCompletionSettings);
-
-        var chatCompletionSettings = new ChatCompletionSetting();
-        config.Bind("ChatCompletion", chatCompletionSettings);
-        services.AddSingleton((IServiceProvider x) => chatCompletionSettings);
+        var llmProviders = new List<LlmProviderSetting>();
+        config.Bind("LlmProviders", llmProviders);
+        services.AddSingleton((IServiceProvider x) =>
+        {
+            foreach (var llmProvider in llmProviders)
+            {
+                Console.WriteLine($"Loaded LlmProvider {llmProvider.Provider} settings with {llmProvider.Models.Count} models.");
+            }
+            return llmProviders;
+        });
 
         RegisterPlugins(services, config);
 

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs

@@ -59,7 +59,7 @@ public async Task<bool> SendMessage(string agentId,
             var routing = _services.GetRequiredService<IRoutingService>();
             var settings = _services.GetRequiredService<RoutingSettings>();
 
-            response = agentId == settings.RouterId ?
+            response = agentId == settings.AgentId ?
                 await routing.InstructLoop(message) :
                 await routing.ExecuteDirectly(agent, message);