Merge pull request #750 from iceljc/master

iceljc · web-flow · commit 2619472a5a34 · 2024-11-15T10:36:57.000-06:00
add similar search
diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs
@@ -30,7 +30,7 @@ public interface IKnowledgeService
     /// <param name="collectionName"></param>
     /// <param name="files"></param>
     /// <returns></returns>
-    Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files);
+    Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files, ChunkOption? option = null);
     /// <summary>
     /// Save document content to knowledgebase without saving the document
     /// </summary>
diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Models/ChunkOption.cs
@@ -13,4 +13,15 @@ public class ChunkOption
     public int Conjunction { get; set; }
 
     public bool SplitByWord { get; set; }
+
+
+    public static ChunkOption Default()
+    {
+        return new ChunkOption
+        {
+            Size = 1024,
+            Conjunction = 12,
+            SplitByWord = true,
+        };
+    }
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginFilter.cs
@@ -4,5 +4,6 @@ public class PluginFilter
     {
         public Pagination Pager { get; set; } = new Pagination();
         public IEnumerable<string>? Names { get; set; }
+        public string? SimilarName { get; set; }
     }
 }
diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/AgentFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/AgentFilter.cs
@@ -4,6 +4,7 @@ public class AgentFilter
 {
     public Pagination Pager { get; set; } = new Pagination();
     public string? AgentName { get; set; }
+    public string? SimilarName { get; set; }
     public bool? Disabled { get; set; }
     public bool? Installed { get; set; }
     public string? Type { get; set; }
diff --git a/src/Infrastructure/BotSharp.Core.SideCar/BotSharpSideCarPlugin.cs b/src/Infrastructure/BotSharp.Core.SideCar/BotSharpSideCarPlugin.cs
@@ -8,7 +8,7 @@ namespace BotSharp.Core.SideCar;
 public class BotSharpSideCarPlugin : IBotSharpPlugin
 {
     public string Id => "06e5a276-bba0-45af-9625-889267c341c9";
-    public string Name => "Side car";
+    public string Name => "Side Car";
     public string Description => "Provides side car for calling agent cluster in conversation";
 
     public SettingsMeta Settings => new SettingsMeta("SideCar");
diff --git a/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs b/src/Infrastructure/BotSharp.Core/Plugins/PluginLoader.cs
@@ -4,6 +4,7 @@
 using System.Drawing;
 using System.IO;
 using System.Reflection;
+using System.Text.RegularExpressions;
 using System.Xml;
 
 namespace BotSharp.Core.Plugins;
@@ -132,6 +133,12 @@ public PagedItems<PluginDef> GetPagedPlugins(IServiceProvider services, PluginFi
             plugins = plugins.Where(x => filter.Names.Any(n => x.Name.IsEqualTo(n))).ToList();
         }
 
+        if (!string.IsNullOrEmpty(filter.SimilarName))
+        {
+            var regex = new Regex(filter.SimilarName, RegexOptions.Compiled | RegexOptions.IgnoreCase);
+            plugins = plugins.Where(x => regex.IsMatch(x.Name)).ToList();
+        }
+
         return new PagedItems<PluginDef>
         {
             Items = plugins.Skip(pager.Offset).Take(pager.Size),
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs
@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Routing.Models;
 using System.IO;
+using System.Text.RegularExpressions;
 
 namespace BotSharp.Core.Repository
 {
@@ -62,6 +63,8 @@ public void UpdateAgent(Agent agent, AgentField field)
                 default:
                     break;
             }
+
+            _agents = [];
         }
 
         #region Update Agent Fields
@@ -366,6 +369,12 @@ public List<Agent> GetAgents(AgentFilter filter)
                 query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower());
             }
 
+            if (!string.IsNullOrEmpty(filter.SimilarName))
+            {
+                var regex = new Regex(filter.SimilarName, RegexOptions.Compiled | RegexOptions.IgnoreCase);
+                query = query.Where(x => regex.IsMatch(x.Name));
+            }
+
             if (filter.Disabled.HasValue)
             {
                 query = query.Where(x => x.Disabled == filter.Disabled);
@@ -478,7 +487,8 @@ public void BulkInsertAgents(List<Agent> agents)
                     File.WriteAllText(instFile, agent.Instruction);
                 }
             }
-            Reset();
+
+            ResetLocalAgents();
         }
 
         public void BulkInsertUserAgents(List<UserAgent> userAgents)
@@ -511,7 +521,7 @@ public void BulkInsertUserAgents(List<UserAgent> userAgents)
                 Thread.Sleep(50);
             }
 
-            Reset();
+            ResetLocalAgents();
         }
 
         public bool DeleteAgents()
@@ -566,7 +576,7 @@ public bool DeleteAgent(string agentId)
 
                 // Delete agent folder
                 Directory.Delete(agentDir, true);
-                Reset();
+                ResetLocalAgents();
                 return true;
             }
             catch
@@ -575,7 +585,7 @@ public bool DeleteAgent(string agentId)
             }
         }
 
-        private void Reset()
+        private void ResetLocalAgents()
         {
             _agents = [];
             _userAgents = [];
diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs
@@ -54,6 +54,7 @@ public bool DeleteConversations(IEnumerable<string> conversationIds)
 
                 Directory.Delete(convDir, true);
             }
+
             return true;
         }
 
diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Document.cs
@@ -10,7 +10,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;
 
 public partial class KnowledgeService
 {
-    public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)
+    public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName,
+        IEnumerable<ExternalFileModel> files, ChunkOption? option = null)
     {
         var res = new UploadKnowledgeResponse
         {
@@ -48,7 +49,7 @@ public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string col
             {
                 // Get document info
                 var (contentType, bytes) = await GetFileInfo(file);
-                var contents = await GetFileContent(contentType, bytes);
+                var contents = await GetFileContent(contentType, bytes, option ?? ChunkOption.Default());
                 
                 // Save document
                 var fileId = Guid.NewGuid();
@@ -369,13 +370,13 @@ public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string col
     }
 
     #region Read doc content
-    private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes)
+    private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes, ChunkOption option)
     {
         IEnumerable<string> results = new List<string>();
 
         if (contentType.IsEqualTo(MediaTypeNames.Text.Plain))
         {
-            results = await ReadTxt(bytes);
+            results = await ReadTxt(bytes, option);
         }
         else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))
         {
@@ -385,20 +386,15 @@ private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[
         return results;
     }
 
-    private async Task<IEnumerable<string>> ReadTxt(byte[] bytes)
+    private async Task<IEnumerable<string>> ReadTxt(byte[] bytes, ChunkOption option)
     {
         using var stream = new MemoryStream(bytes);
         using var reader = new StreamReader(stream);
         var content = await reader.ReadToEndAsync();
         reader.Close();
         stream.Close();
 
-        var lines = TextChopper.Chop(content, new ChunkOption
-        {
-            Size = 1024,
-            Conjunction = 12,
-            SplitByWord = true,
-        });
+        var lines = TextChopper.Chop(content, option);
         return lines;
     }
 
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs
@@ -298,6 +298,11 @@ public List<Agent> GetAgents(AgentFilter filter)
             filters.Add(builder.Eq(x => x.Name, filter.AgentName));
         }
 
+        if (!string.IsNullOrEmpty(filter.SimilarName))
+        {
+            filters.Add(builder.Regex(x => x.Name, new BsonRegularExpression(filter.SimilarName, "i")));
+        }
+
         if (filter.Disabled.HasValue)
         {
             filters.Add(builder.Eq(x => x.Disabled, filter.Disabled.Value));
diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.User.cs
@@ -18,7 +18,7 @@ public partial class MongoRepository
     public User? GetUserByPhone(string phone)
     {
         string phoneSecond = string.Empty;
-        // 如果电话号码长度小于 4，直接返回 null
+        // if phone number length is less than 4, return null
         if (phone?.Length < 4)
         {
             return null;

Original file line number	Diff line number	Diff line change
`@@ -4,5 +4,6 @@ public class PluginFilter`
`4`	`4`	`{`
`5`	`5`	`public Pagination Pager { get; set; } = new Pagination();`
`6`	`6`	`public IEnumerable<string>? Names { get; set; }`
	`7`	`+ public string? SimilarName { get; set; }`
`7`	`8`	`}`
`8`	`9`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@ public class AgentFilter`
`4`	`4`	`{`
`5`	`5`	`public Pagination Pager { get; set; } = new Pagination();`
`6`	`6`	`public string? AgentName { get; set; }`
	`7`	`+ public string? SimilarName { get; set; }`
`7`	`8`	`public bool? Disabled { get; set; }`
`8`	`9`	`public bool? Installed { get; set; }`
`9`	`10`	`public string? Type { get; set; }`
Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@ namespace BotSharp.Core.SideCar;`
`8`	`8`	`public class BotSharpSideCarPlugin : IBotSharpPlugin`
`9`	`9`	`{`
`10`	`10`	`public string Id => "06e5a276-bba0-45af-9625-889267c341c9";`
`11`		`- public string Name => "Side car";`
	`11`	`+ public string Name => "Side Car";`
`12`	`12`	`public string Description => "Provides side car for calling agent cluster in conversation";`
`13`	`13`
`14`	`14`	`public SettingsMeta Settings => new SettingsMeta("SideCar");`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,6 @@`
`1`	`1`	`using BotSharp.Abstraction.Routing.Models;`
`2`	`2`	`using System.IO;`
	`3`	`+using System.Text.RegularExpressions;`
`3`	`4`
`4`	`5`	`namespace BotSharp.Core.Repository`
`5`	`6`	`{`
`@@ -62,6 +63,8 @@ public void UpdateAgent(Agent agent, AgentField field)`
`62`	`63`	`default:`
`63`	`64`	`break;`
`64`	`65`	`}`
	`66`	`+`
	`67`	`+ _agents = [];`
`65`	`68`	`}`
`66`	`69`
`67`	`70`	`#region Update Agent Fields`
`@@ -366,6 +369,12 @@ public List<Agent> GetAgents(AgentFilter filter)`
`366`	`369`	`query = query.Where(x => x.Name.ToLower() == filter.AgentName.ToLower());`
`367`	`370`	`}`
`368`	`371`
	`372`	`+ if (!string.IsNullOrEmpty(filter.SimilarName))`
	`373`	`+ {`
	`374`	`+ var regex = new Regex(filter.SimilarName, RegexOptions.Compiled \| RegexOptions.IgnoreCase);`
	`375`	`+ query = query.Where(x => regex.IsMatch(x.Name));`
	`376`	`+ }`
	`377`	`+`
`369`	`378`	`if (filter.Disabled.HasValue)`
`370`	`379`	`{`
`371`	`380`	`query = query.Where(x => x.Disabled == filter.Disabled);`
`@@ -478,7 +487,8 @@ public void BulkInsertAgents(List<Agent> agents)`
`478`	`487`	`File.WriteAllText(instFile, agent.Instruction);`
`479`	`488`	`}`
`480`	`489`	`}`
`481`		`- Reset();`
	`490`	`+`
	`491`	`+ ResetLocalAgents();`
`482`	`492`	`}`
`483`	`493`
`484`	`494`	`public void BulkInsertUserAgents(List<UserAgent> userAgents)`
`@@ -511,7 +521,7 @@ public void BulkInsertUserAgents(List<UserAgent> userAgents)`
`511`	`521`	`Thread.Sleep(50);`
`512`	`522`	`}`
`513`	`523`
`514`		`- Reset();`
	`524`	`+ ResetLocalAgents();`
`515`	`525`	`}`
`516`	`526`
`517`	`527`	`public bool DeleteAgents()`
`@@ -566,7 +576,7 @@ public bool DeleteAgent(string agentId)`
`566`	`576`
`567`	`577`	`// Delete agent folder`
`568`	`578`	`Directory.Delete(agentDir, true);`
`569`		`- Reset();`
	`579`	`+ ResetLocalAgents();`
`570`	`580`	`return true;`
`571`	`581`	`}`
`572`	`582`	`catch`
`@@ -575,7 +585,7 @@ public bool DeleteAgent(string agentId)`
`575`	`585`	`}`
`576`	`586`	`}`
`577`	`587`
`578`		`- private void Reset()`
	`588`	`+ private void ResetLocalAgents()`
`579`	`589`	`{`
`580`	`590`	`_agents = [];`
`581`	`591`	`_userAgents = [];`
Original file line number	Diff line number	Diff line change
`@@ -54,6 +54,7 @@ public bool DeleteConversations(IEnumerable<string> conversationIds)`
`54`	`54`
`55`	`55`	`Directory.Delete(convDir, true);`
`56`	`56`	`}`
	`57`	`+`
`57`	`58`	`return true;`
`58`	`59`	`}`
`59`	`60`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,8 @@ namespace BotSharp.Plugin.KnowledgeBase.Services;`
`10`	`10`
`11`	`11`	`public partial class KnowledgeService`
`12`	`12`	`{`
`13`		`- public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName, IEnumerable<ExternalFileModel> files)`
	`13`	`+ public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string collectionName,`
	`14`	`+ IEnumerable<ExternalFileModel> files, ChunkOption? option = null)`
`14`	`15`	`{`
`15`	`16`	`var res = new UploadKnowledgeResponse`
`16`	`17`	`{`
`@@ -48,7 +49,7 @@ public async Task<UploadKnowledgeResponse> UploadDocumentsToKnowledge(string col`
`48`	`49`	`{`
`49`	`50`	`// Get document info`
`50`	`51`	`var (contentType, bytes) = await GetFileInfo(file);`
`51`		`- var contents = await GetFileContent(contentType, bytes);`
	`52`	`+ var contents = await GetFileContent(contentType, bytes, option ?? ChunkOption.Default());`
`52`	`53`
`53`	`54`	`// Save document`
`54`	`55`	`var fileId = Guid.NewGuid();`
`@@ -369,13 +370,13 @@ public async Task<FileBinaryDataModel> GetKnowledgeDocumentBinaryData(string col`
`369`	`370`	`}`
`370`	`371`
`371`	`372`	`#region Read doc content`
`372`		`- private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes)`
	`373`	`+ private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[] bytes, ChunkOption option)`
`373`	`374`	`{`
`374`	`375`	`IEnumerable<string> results = new List<string>();`
`375`	`376`
`376`	`377`	`if (contentType.IsEqualTo(MediaTypeNames.Text.Plain))`
`377`	`378`	`{`
`378`		`- results = await ReadTxt(bytes);`
	`379`	`+ results = await ReadTxt(bytes, option);`
`379`	`380`	`}`
`380`	`381`	`else if (contentType.IsEqualTo(MediaTypeNames.Application.Pdf))`
`381`	`382`	`{`
`@@ -385,20 +386,15 @@ private async Task<IEnumerable<string>> GetFileContent(string contentType, byte[`
`385`	`386`	`return results;`
`386`	`387`	`}`
`387`	`388`
`388`		`- private async Task<IEnumerable<string>> ReadTxt(byte[] bytes)`
	`389`	`+ private async Task<IEnumerable<string>> ReadTxt(byte[] bytes, ChunkOption option)`
`389`	`390`	`{`
`390`	`391`	`using var stream = new MemoryStream(bytes);`
`391`	`392`	`using var reader = new StreamReader(stream);`
`392`	`393`	`var content = await reader.ReadToEndAsync();`
`393`	`394`	`reader.Close();`
`394`	`395`	`stream.Close();`
`395`	`396`
`396`		`- var lines = TextChopper.Chop(content, new ChunkOption`
`397`		`- {`
`398`		`- Size = 1024,`
`399`		`- Conjunction = 12,`
`400`		`- SplitByWord = true,`
`401`		`- });`
	`397`	`+ var lines = TextChopper.Chop(content, option);`
`402`	`398`	`return lines;`
`403`	`399`	`}`
`404`	`400`
Original file line number	Diff line number	Diff line change
`@@ -298,6 +298,11 @@ public List<Agent> GetAgents(AgentFilter filter)`
`298`	`298`	`filters.Add(builder.Eq(x => x.Name, filter.AgentName));`
`299`	`299`	`}`
`300`	`300`
	`301`	`+ if (!string.IsNullOrEmpty(filter.SimilarName))`
	`302`	`+ {`
	`303`	`+ filters.Add(builder.Regex(x => x.Name, new BsonRegularExpression(filter.SimilarName, "i")));`
	`304`	`+ }`
	`305`	`+`
`301`	`306`	`if (filter.Disabled.HasValue)`
`302`	`307`	`{`
`303`	`308`	`filters.Add(builder.Eq(x => x.Disabled, filter.Disabled.Value));`