From 32a084289e46b21b0cbc4c9942c2b0435be122d6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 11 Apr 2024 02:00:17 -0500 Subject: [PATCH 1/2] refine agent refresh --- .../Agents/IAgentService.cs | 4 +- .../Repositories/IBotSharpRepository.cs | 3 +- .../Services/AgentService.RefreshAgents.cs | 92 ++++++++++++------- .../Services/AgentService.UpdateAgent.cs | 27 ++++-- .../Repository/BotSharpDbContext.cs | 69 ++++---------- .../FileRepository/FileRepository.Agent.cs | 10 +- .../FileRepository.AgentTask.cs | 16 ++-- .../Tasks/Services/AgentTaskService.cs | 2 +- .../Controllers/AgentController.cs | 8 +- .../Repository/MongoRepository.Agent.cs | 20 ++++ .../Repository/MongoRepository.AgentTask.cs | 12 ++- 11 files changed, 148 insertions(+), 115 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index e67cd6b87..5ab249b87 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -10,7 +10,7 @@ namespace BotSharp.Abstraction.Agents; public interface IAgentService { Task CreateAgent(Agent agent); - Task RefreshAgents(); + Task RefreshAgents(); Task> GetAgents(AgentFilter filter); /// @@ -37,7 +37,7 @@ public interface IAgentService Task DeleteAgent(string id); Task UpdateAgent(Agent agent, AgentField updateField); - Task UpdateAgentFromFile(string id); + Task UpdateAgentFromFile(string id); string GetDataDir(); string GetAgentDataDir(string agentId); diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index e177b3546..a11249756 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -32,6 +32,7 @@ public interface IBotSharpRepository void BulkInsertAgents(List agents); void BulkInsertUserAgents(List userAgents); bool DeleteAgents(); + bool DeleteAgent(string agentId); List GetAgentResponses(string agentId, string prefix, string intent); string GetAgentTemplate(string agentId, string templateName); #endregion @@ -42,7 +43,7 @@ public interface IBotSharpRepository void InsertAgentTask(AgentTask task); void BulkInsertAgentTasks(List tasks); void UpdateAgentTask(AgentTask task, AgentTaskField field); - bool DeleteAgentTask(string agentId, string taskId); + bool DeleteAgentTask(string agentId, List taskIds); bool DeleteAgentTasks(); #endregion diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index 9c3b34f9b..3077e9294 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -1,55 +1,79 @@ -using BotSharp.Abstraction.Tasks.Models; using System.IO; namespace BotSharp.Core.Agents.Services; public partial class AgentService { - public async Task RefreshAgents() + public async Task RefreshAgents() { - var isAgentDeleted = _db.DeleteAgents(); - var isTaskDeleted = _db.DeleteAgentTasks(); - if (!isAgentDeleted) return; - var dbSettings = _services.GetRequiredService(); var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, _agentSettings.DataDir); + string refreshResult; + if (!Directory.Exists(agentDir)) + { + refreshResult = $"Cannot find the directory: {agentDir}"; + return refreshResult; + } + var user = _db.GetUserById(_user.Id); - var agents = new List(); - var userAgents = new List(); - var agentTasks = new List(); + var refreshedAgents = new List(); foreach (var dir in Directory.GetDirectories(agentDir)) { - var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json")); - var agent = JsonSerializer.Deserialize(agentJson, _options); - if (agent == null) continue; - - var functions = FetchFunctionsFromFile(dir); - var instruction = FetchInstructionFromFile(dir); - var responses = FetchResponsesFromFile(dir); - var templates = FetchTemplatesFromFile(dir); - var samples = FetchSamplesFromFile(dir); - agent.SetInstruction(instruction) - .SetTemplates(templates) - .SetFunctions(functions) - .SetResponses(responses) - .SetSamples(samples); - agents.Add(agent); - - var userAgent = BuildUserAgent(agent.Id, user.Id); - userAgents.Add(userAgent); - - var tasks = FetchTasksFromFile(dir); - agentTasks.AddRange(tasks); + try + { + var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json")); + var agent = JsonSerializer.Deserialize(agentJson, _options); + + if (agent == null) + { + _logger.LogError($"Cannot find agent in file directory: {dir}"); + continue; + } + + var functions = FetchFunctionsFromFile(dir); + var instruction = FetchInstructionFromFile(dir); + var responses = FetchResponsesFromFile(dir); + var templates = FetchTemplatesFromFile(dir); + var samples = FetchSamplesFromFile(dir); + agent.SetInstruction(instruction) + .SetTemplates(templates) + .SetFunctions(functions) + .SetResponses(responses) + .SetSamples(samples); + + var userAgent = BuildUserAgent(agent.Id, user.Id); + var tasks = FetchTasksFromFile(dir); + + var isAgentDeleted = _db.DeleteAgent(agent.Id); + if (isAgentDeleted) + { + _db.BulkInsertAgents(new List { agent }); + _db.BulkInsertUserAgents(new List { userAgent }); + _db.BulkInsertAgentTasks(tasks); + refreshedAgents.Add(agent.Name); + } + } + catch (Exception ex) + { + _logger.LogError($"Failed to migrate agent in file directory: {dir}\r\nError: {ex.Message}"); + } } - _db.BulkInsertAgents(agents); - _db.BulkInsertUserAgents(userAgents); - _db.BulkInsertAgentTasks(agentTasks); + if (!refreshedAgents.IsNullOrEmpty()) + { + Utilities.ClearCache(); + refreshResult = $"Agents are migrated! {string.Join("\r\n", refreshedAgents)}"; + } + else + { + refreshResult = "No agent gets refreshed!"; + } - Utilities.ClearCache(); + _logger.LogInformation(refreshResult); + return refreshResult; } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index 4577af6d0..5eca3f8c6 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -39,11 +39,13 @@ public async Task UpdateAgent(Agent agent, AgentField updateField) await Task.CompletedTask; } - public async Task UpdateAgentFromFile(string id) + public async Task UpdateAgentFromFile(string id) { var agent = _db.GetAgent(id); - - if (agent == null) return; + if (agent == null) + { + return $"Cannot find agent ${id}"; + } var dbSettings = _services.GetRequiredService(); var agentSettings = _services.GetRequiredService(); @@ -53,7 +55,12 @@ public async Task UpdateAgentFromFile(string id) var clonedAgent = Agent.Clone(agent); var foundAgent = FetchAgentFileById(agent.Id, filePath); - if (foundAgent != null) + if (foundAgent == null) + { + return $"Cannot find agent {agent.Name} in file directory: {filePath}"; + } + + try { clonedAgent.SetId(foundAgent.Id) .SetName(foundAgent.Name) @@ -71,15 +78,19 @@ public async Task UpdateAgentFromFile(string id) .SetLlmConfig(foundAgent.LlmConfig); _db.UpdateAgent(clonedAgent, AgentField.All); - Utilities.ClearCache(); + return $"Agent {agent.Name} has been migrated!"; + } + catch (Exception ex) + { + return $"Failed to migrate agent {agent.Name} in file directory {filePath}.\r\nError: {ex.Message}"; } - - await Task.CompletedTask; } - private Agent FetchAgentFileById(string agentId, string filePath) + private Agent? FetchAgentFileById(string agentId, string filePath) { + if (!Directory.Exists(filePath)) return null; + foreach (var dir in Directory.GetDirectories(filePath)) { var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json")); diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index d1361d028..0605c5252 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -73,86 +73,57 @@ public int Transaction(Action action) #region Agent public Agent GetAgent(string agentId) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public List GetAgents(AgentFilter filter) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public List GetAgentsByUser(string userId) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public void UpdateAgent(Agent agent, AgentField field) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public string GetAgentTemplate(string agentId, string templateName) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public List GetAgentResponses(string agentId, string prefix, string intent) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public void BulkInsertAgents(List agents) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public void BulkInsertUserAgents(List userAgents) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public bool DeleteAgents() - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); + + public bool DeleteAgent(string agentId) + => throw new NotImplementedException(); #endregion #region Agent Task public PagedItems GetAgentTasks(AgentTaskFilter filter) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public AgentTask? GetAgentTask(string agentId, string taskId) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public void InsertAgentTask(AgentTask task) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public void BulkInsertAgentTasks(List tasks) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); public void UpdateAgentTask(AgentTask task, AgentTaskField field) - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); - public bool DeleteAgentTask(string agentId, string taskId) - { - throw new NotImplementedException(); - } + public bool DeleteAgentTask(string agentId, List taskIds) + => throw new NotImplementedException(); public bool DeleteAgentTasks() - { - throw new NotImplementedException(); - } + => throw new NotImplementedException(); #endregion #region Conversation diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index 1fbca99ca..20f8bcddb 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -1,9 +1,4 @@ -using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Routing.Models; -using BotSharp.Abstraction.Tasks.Models; -using Microsoft.Extensions.Logging; using System.IO; namespace BotSharp.Core.Repository @@ -419,5 +414,10 @@ public bool DeleteAgents() { return false; } + + public bool DeleteAgent(string agentId) + { + return false; + } } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs index 73a8e3ed4..df80bc8b6 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -1,7 +1,5 @@ -using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Tasks.Models; using System.IO; -using System.Threading.Tasks; namespace BotSharp.Core.Repository; @@ -192,18 +190,22 @@ public void UpdateAgentTask(AgentTask task, AgentTaskField field) File.WriteAllText(taskFile, fileContent); } - public bool DeleteAgentTask(string agentId, string taskId) + public bool DeleteAgentTask(string agentId, List taskIds) { var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); - if (!Directory.Exists(agentDir)) return false; + if (!Directory.Exists(agentDir) || taskIds.IsNullOrEmpty()) return false; var taskDir = Path.Combine(agentDir, "tasks"); if (!Directory.Exists(taskDir)) return false; - var taskFile = FindTaskFileById(taskDir, taskId); - if (string.IsNullOrWhiteSpace(taskFile)) return false; + foreach (var taskId in taskIds) + { + var taskFile = FindTaskFileById(taskDir, taskId); + if (string.IsNullOrWhiteSpace(taskFile)) return false; - File.Delete(taskFile); + File.Delete(taskFile); + } + return true; } diff --git a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs index 1e7cdfdf9..13ffc6464 100644 --- a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs +++ b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs @@ -72,7 +72,7 @@ public async Task UpdateTask(AgentTask task, AgentTaskField field) public async Task DeleteTask(string agentId, string taskId) { var db = _services.GetRequiredService(); - var isDeleted = db.DeleteAgentTask(agentId, taskId); + var isDeleted = db.DeleteAgentTask(agentId, new List { taskId }); return await Task.FromResult(isDeleted); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index 5d7f7b56e..77f730613 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -84,15 +84,15 @@ public async Task CreateAgent(AgentCreationModel agent) } [HttpPost("/refresh-agents")] - public async Task RefreshAgents() + public async Task RefreshAgents() { - await _agentService.RefreshAgents(); + return await _agentService.RefreshAgents(); } [HttpPut("/agent/file/{agentId}")] - public async Task UpdateAgentFromFile([FromRoute] string agentId) + public async Task UpdateAgentFromFile([FromRoute] string agentId) { - await _agentService.UpdateAgentFromFile(agentId); + return await _agentService.UpdateAgentFromFile(agentId); } [HttpPut("/agent/{agentId}")] diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 3272930b3..07d309844 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -398,7 +398,27 @@ public bool DeleteAgents() { return false; } + } + + public bool DeleteAgent(string agentId) + { + try + { + if (string.IsNullOrEmpty(agentId)) return false; + + var agentFilter = Builders.Filter.Eq(x => x.Id, agentId); + var agentUserFilter = Builders.Filter.Eq(x => x.AgentId, agentId); + var agentTaskFilter = Builders.Filter.Eq(x => x.AgentId, agentId); + _dc.Agents.DeleteOne(agentFilter); + _dc.UserAgents.DeleteMany(agentUserFilter); + _dc.AgentTasks.DeleteMany(agentTaskFilter); + return true; + } + catch + { + return false; + } } private Agent TransformAgentDocument(AgentDocument? agentDoc) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs index 125f3cd89..a3df61da8 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs @@ -155,12 +155,16 @@ public void UpdateAgentTask(AgentTask task, AgentTaskField field) _dc.AgentTasks.ReplaceOne(filter, taskDoc); } - public bool DeleteAgentTask(string agentId, string taskId) + public bool DeleteAgentTask(string agentId, List taskIds) { - if (string.IsNullOrEmpty(taskId)) return false; + if (taskIds.IsNullOrEmpty()) return false; - var filter = Builders.Filter.Eq(x => x.Id, taskId); - var taskDeleted = _dc.AgentTasks.DeleteOne(filter); + var builder = Builders.Filter; + var filters = new List> + { + builder.In(x => x.Id, taskIds) + }; + var taskDeleted = _dc.AgentTasks.DeleteMany(builder.And(filters)); return taskDeleted.DeletedCount > 0; } From e4e3ee56682af77ace0f0bd11c2479bee966d401 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 11 Apr 2024 02:04:54 -0500 Subject: [PATCH 2/2] minor change --- .../Repository/FileRepository/FileRepository.AgentTask.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs index df80bc8b6..d542b28ba 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -198,15 +198,17 @@ public bool DeleteAgentTask(string agentId, List taskIds) var taskDir = Path.Combine(agentDir, "tasks"); if (!Directory.Exists(taskDir)) return false; + var deletedTasks = new List(); foreach (var taskId in taskIds) { var taskFile = FindTaskFileById(taskDir, taskId); - if (string.IsNullOrWhiteSpace(taskFile)) return false; + if (string.IsNullOrWhiteSpace(taskFile)) continue; File.Delete(taskFile); + deletedTasks.Add(taskId); } - return true; + return deletedTasks.Any(); } public bool DeleteAgentTasks()