Merge pull request #324 from iceljc/features/add-function-name

add delete log

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -59,7 +59,7 @@ public interface IBotSharpRepository
     PagedItems<Conversation> GetConversations(ConversationFilter filter);
     void UpdateConversationTitle(string conversationId, string title);
     List<Conversation> GetLastConversations();
-    bool TruncateConversation(string conversationId, string messageId);
+    bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false);
     #endregion
 
     #region Execution Log

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs

@@ -7,7 +7,7 @@ public partial class ConversationService : IConversationService
     public async Task<bool> TruncateConversation(string conversationId, string messageId)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var isSaved = db.TruncateConversation(conversationId, messageId);
+        var isSaved = db.TruncateConversation(conversationId, messageId, true);
         return await Task.FromResult(isSaved);
     }
 }

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -217,7 +217,7 @@ public void UpdateConversationStatus(string conversationId, string status)
         throw new NotImplementedException();
     }
 
-    public bool TruncateConversation(string conversationId, string messageId)
+    public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
     {
         throw new NotImplementedException();
     }

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
 using System.Globalization;
@@ -263,7 +264,7 @@ public List<Conversation> GetLastConversations()
         }
 
 
-        public bool TruncateConversation(string conversationId, string messageId)
+        public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
         {
             if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false;
 
@@ -288,6 +289,12 @@ public bool TruncateConversation(string conversationId, string messageId)
             var states = CollectConversationStates(stateDir);
             isSaved = HandleTruncatedStates(stateDir, states, refTime);
 
+            // Remove logs
+            if (cleanLog)
+            {
+                HandleTruncatedLogs(convDir, refTime);
+            }
+
             return isSaved;
         }
 
@@ -392,6 +399,44 @@ private bool HandleTruncatedStates(string stateDir, List<StateKeyValue> states,
             return isSaved;
         }
 
+        private bool HandleTruncatedLogs(string convDir, DateTime refTime)
+        {
+            var contentLogDir = Path.Combine(convDir, "content_log");
+            var stateLogDir = Path.Combine(convDir, "state_log");
+
+            if (Directory.Exists(contentLogDir))
+            {
+                foreach (var file in Directory.GetFiles(contentLogDir))
+                {
+                    var text = File.ReadAllText(file);
+                    var log = JsonSerializer.Deserialize<ConversationContentLogModel>(text);
+                    if (log == null) continue;
+
+                    if (log.CreateTime >= refTime)
+                    {
+                        File.Delete(file);
+                    }
+                }
+            }
+
+            if (Directory.Exists(stateLogDir))
+            {
+                foreach (var file in Directory.GetFiles(stateLogDir))
+                {
+                    var text = File.ReadAllText(file);
+                    var log = JsonSerializer.Deserialize<ConversationStateLogModel>(text);
+                    if (log == null) continue;
+
+                    if (log.CreateTime >= refTime)
+                    {
+                        File.Delete(file);
+                    }
+                }
+            }
+
+            return true;
+        }
+
         private bool SaveTruncatedDialogs(string dialogDir, List<DialogElement> dialogs)
         {
             if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false;

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs

@@ -85,12 +85,10 @@ public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenS
 
         var conversationId = _state.GetConversationId();
         var agent = await _agentService.LoadAgent(message.CurrentAgentId);
-        var logSource = string.Empty;
 
         var log = tokenStats.Prompt;
-        logSource = ContentLogSource.Prompt;
         await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated",
-            BuildContentLog(conversationId, agent?.Name, log, logSource, message));
+            BuildContentLog(conversationId, agent?.Name, log, ContentLogSource.Prompt, message));
     }
 
     /// <summary>

src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs

@@ -3,6 +3,7 @@
 using BotSharp.Abstraction.Repositories.Models;
 using BotSharp.Plugin.MongoStorage.Collections;
 using BotSharp.Plugin.MongoStorage.Models;
+using MongoDB.Driver;
 
 namespace BotSharp.Plugin.MongoStorage.Repository;
 
@@ -273,7 +274,7 @@ public List<Conversation> GetLastConversations()
         }).ToList();
     }
 
-    public bool TruncateConversation(string conversationId, string messageId)
+    public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false)
     {
         if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false;
 
@@ -308,6 +309,28 @@ public bool TruncateConversation(string conversationId, string messageId)
         foundStates.States = truncatedStates;
         _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog);
         _dc.ConversationStates.ReplaceOne(stateFilter, foundStates);
+
+        // Remove logs
+        if (cleanLog)
+        {
+            var contentLogBuilder = Builders<ConversationContentLogDocument>.Filter;
+            var stateLogBuilder = Builders<ConversationStateLogDocument>.Filter;
+
+            var contentLogFilters = new List<FilterDefinition<ConversationContentLogDocument>>()
+            {
+                contentLogBuilder.Eq(x => x.ConversationId, conversationId),
+                contentLogBuilder.Gte(x => x.CreateTime, refTime)
+            };
+            var stateLogFilters = new List<FilterDefinition<ConversationStateLogDocument>>()
+            {
+                stateLogBuilder.Eq(x => x.ConversationId, conversationId),
+                stateLogBuilder.Gte(x => x.CreateTime, refTime)
+            };
+
+            _dc.ContentLogs.DeleteMany(contentLogBuilder.And(contentLogFilters));
+            _dc.StateLogs.DeleteMany(stateLogBuilder.And(stateLogFilters));
+        }
+        
         return true;
     }
 }