use select file option

src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>
@@ -28,9 +28,11 @@
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
+    <PackageReference Include="System.Memory.Data" Version="8.0.0" />
     <PackageReference Include="System.Text.Json" Version="8.0.4" />
     <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
     <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />

src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs

@@ -21,9 +21,6 @@ public interface IFileInstructService
     #endregion
 
     #region Select file
-    Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId,
-        string? agentId = null, string? template = null, string? description = null,
-        bool includeBotFile = false, bool fromBreakpoint = false,
-        int? offset = null, IEnumerable<string>? contentTypes = null);
+    Task<IEnumerable<MessageFileModel>> SelectMessageFiles(string conversationId, SelectFileOptions options);
     #endregion
 }

src/Infrastructure/BotSharp.Abstraction/Files/IFileBasicService.cs → src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs

@@ -2,7 +2,7 @@
 
 namespace BotSharp.Abstraction.Files;
 
-public interface IFileBasicService
+public interface IFileStorageService
 {
     #region Conversation
     /// <summary>

src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs

@@ -1,23 +1,7 @@
 
 namespace BotSharp.Abstraction.Files.Models;
 
-public class BotSharpFile
+public class BotSharpFile : FileBase
 {
-    [JsonPropertyName("file_name")]
-    public string FileName { get; set; } = string.Empty;
-
-    /// <summary>
-    /// File data, e.g., "data:image/png;base64,aaaaaaaa"
-    /// </summary>
-    [JsonPropertyName("file_data")]
-    public string FileData { get; set; } = string.Empty;
-
-    [JsonPropertyName("file_url")]
-    public string FileUrl { get; set; } = string.Empty;
-
-    [JsonPropertyName("content_type")]
-    public string ContentType { get; set; } = string.Empty;
-
-    [JsonPropertyName("file_storage_url")]
-    public string FileStorageUrl { get; set; } = string.Empty;
+
 }

src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs

@@ -0,0 +1,46 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class FileBase
+{
+    /// <summary>
+    /// External file url
+    /// </summary>
+    [JsonPropertyName("file_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileUrl { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Internal file storage url
+    /// </summary>
+    [JsonPropertyName("file_storage_url")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileStorageUrl { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File name without extension
+    /// </summary>
+    [JsonPropertyName("file_name")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileName { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File data, e.g., "data:image/png;base64,aaaaaaaa"
+    /// </summary>
+    [JsonPropertyName("file_data")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileData { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File content type
+    /// </summary>
+    [JsonPropertyName("content_type")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ContentType { get; set; } = string.Empty;
+
+    /// <summary>
+    /// File extension without dot
+    /// </summary>
+    [JsonPropertyName("file_type")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? FileType { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs

@@ -1,34 +1,10 @@
 namespace BotSharp.Abstraction.Files.Models;
 
-public class MessageFileModel
+public class MessageFileModel : FileBase
 {
     [JsonPropertyName("message_id")]
     public string MessageId { get; set; }
 
-    /// <summary>
-    /// External file url
-    /// </summary>
-    [JsonPropertyName("file_url")]
-    public string FileUrl { get; set; }
-
-    /// <summary>
-    /// Internal file storage url
-    /// </summary>
-    [JsonPropertyName("file_storage_url")]
-    public string FileStorageUrl { get; set; }
-
-    /// <summary>
-    /// File name without extension
-    /// </summary>
-    [JsonPropertyName("file_name")]
-    public string FileName { get; set; }
-
-    [JsonPropertyName("file_type")]
-    public string FileType { get; set; }
-
-    [JsonPropertyName("content_type")]
-    public string ContentType { get; set; }
-
     [JsonPropertyName("file_source")]
     public string FileSource { get; set; } = FileSourceType.User;
 

src/Infrastructure/BotSharp.Abstraction/Files/Models/SelectFileOptions.cs

@@ -0,0 +1,14 @@
+namespace BotSharp.Abstraction.Files.Models;
+
+public class SelectFileOptions
+{
+    public string? Provider { get; set; }
+    public string? ModelId { get; set; }
+    public string? AgentId { get; set; }
+    public string? Template { get; set; }
+    public string? Description { get; set; }
+    public bool IncludeBotFile { get; set; }
+    public bool FromBreakpoint { get; set; }
+    public int? Offset { get; set; }
+    public IEnumerable<string>? ContentTypes { get; set; }
+}

src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs

@@ -1,4 +1,10 @@
+using BotSharp.Abstraction.Repositories.Enums;
 using Microsoft.AspNetCore.StaticFiles;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Net.Mime;
 
 namespace BotSharp.Abstraction.Files.Utilities;
 
@@ -37,4 +43,25 @@ public static string GetFileContentType(string filePath)
 
         return contentType;
     }
+
+    public static async Task<byte[]> GetFileBytes(IServiceProvider services, FileBase file)
+    {
+        var bytes = new byte[0];
+        var settings = services.GetRequiredService<FileStorageSettings>();
+
+        if (settings.Default == FileStorageEnum.LocalFileStorage)
+        {
+            using var fs = File.OpenRead(file.FileStorageUrl);
+            var binary = BinaryData.FromStream(fs);
+            bytes = binary.ToArray();
+            fs.Close();
+        }
+        else
+        {
+            var http = services.GetRequiredService<IHttpClientFactory>();
+            using var client = http.CreateClient();
+            bytes = await client.GetByteArrayAsync(file.FileUrl);
+        }
+        return bytes;
+    }
 }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs

@@ -5,10 +5,9 @@ public partial class ConversationService : IConversationService
     public async Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var fileService = _services.GetRequiredService<IFileBasicService>();
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var deleteMessageIds = db.TruncateConversation(conversationId, messageId, cleanLog: true);
-
-        fileService.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId);
+        fileStorage.DeleteMessageFiles(conversationId, deleteMessageIds, messageId, newMessageId);
 
         var hooks = _services.GetServices<IConversationHook>().ToList();
         foreach (var hook in hooks)

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs

@@ -37,9 +37,9 @@ public ConversationService(
     public async Task<bool> DeleteConversations(IEnumerable<string> ids)
     {
         var db = _services.GetRequiredService<IBotSharpRepository>();
-        var fileService = _services.GetRequiredService<IFileBasicService>();
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
         var isDeleted = db.DeleteConversations(ids);
-        fileService.DeleteConversationFiles(ids);
+        fileStorage.DeleteConversationFiles(ids);
         return await Task.FromResult(isDeleted);
     }
 

src/Infrastructure/BotSharp.Core/Files/FilePlugin.cs

@@ -10,17 +10,18 @@ public class FilePlugin : IBotSharpPlugin
 
     public string Name => "File";
 
-    public string Description => "Provides file analysis.";
+    public string Description => "Provides file storage and analysis.";
 
 
     public void RegisterDI(IServiceCollection services, IConfiguration config)
     {
         var myFileStorageSettings = new FileStorageSettings();
         config.Bind("FileStorage", myFileStorageSettings);
+        services.AddSingleton(myFileStorageSettings);
 
         if (myFileStorageSettings.Default == FileStorageEnum.LocalFileStorage)
         {
-            services.AddScoped<IFileBasicService, FileBasicService>();
+            services.AddScoped<IFileStorageService, LocalFileStorageService>();
         }
         services.AddScoped<IFileInstructService, FileInstructService>();
     }

src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Common.cs

@@ -2,7 +2,7 @@
 
 namespace BotSharp.Core.Files.Services;
 
-public partial class FileBasicService
+public partial class LocalFileStorageService
 {
     public string GetDirectory(string conversationId)
     {

src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.Conversation.cs

@@ -4,7 +4,7 @@
 
 namespace BotSharp.Core.Files.Services;
 
-public partial class FileBasicService
+public partial class LocalFileStorageService
 {
     public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
         IEnumerable<RoleDialogModel> dialogs, IEnumerable<string>? contentTypes = null,

src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.User.cs

@@ -2,7 +2,7 @@
 
 namespace BotSharp.Core.Files.Services;
 
-public partial class FileBasicService
+public partial class LocalFileStorageService
 {
     public string GetUserAvatar()
     {

src/Infrastructure/BotSharp.Core/Files/Services/Basic/FileBasicService.cs → src/Infrastructure/BotSharp.Core/Files/Services/Basic/LocalFileStorageService.cs

@@ -2,12 +2,12 @@
 
 namespace BotSharp.Core.Files.Services;
 
-public partial class FileBasicService : IFileBasicService
+public partial class LocalFileStorageService : IFileStorageService
 {
     private readonly BotSharpDatabaseSettings _dbSettings;
     private readonly IServiceProvider _services;
     private readonly IUserIdentity _user;
-    private readonly ILogger<FileBasicService> _logger;
+    private readonly ILogger<LocalFileStorageService> _logger;
     private readonly string _baseDir;
     private readonly IEnumerable<string> _imageTypes = new List<string>
     {
@@ -24,10 +24,10 @@ public partial class FileBasicService : IFileBasicService
     private const string USER_AVATAR_FOLDER = "avatar";
     private const string SESSION_FOLDER = "sessions";
 
-    public FileBasicService(
+    public LocalFileStorageService(
         BotSharpDatabaseSettings dbSettings,
         IUserIdentity user,
-        ILogger<FileBasicService> logger,
+        ILogger<LocalFileStorageService> logger,
         IServiceProvider services)
     {
         _dbSettings = dbSettings;

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs

@@ -16,8 +16,8 @@ public async Task<string> ReadPdf(string? provider, string? model, string? model
 
         var guid = Guid.NewGuid().ToString();
 
-        var sessionDir = _fileBasic.BuildDirectory(SESSION_FOLDER, guid);
-        DeleteIfExistDirectory(sessionDir);
+        var sessionDir = _fileStorage.BuildDirectory(SESSION_FOLDER, guid);
+        DeleteIfExistDirectory(sessionDir, true);
 
         try
         {
@@ -46,7 +46,7 @@ public async Task<string> ReadPdf(string? provider, string? model, string? model
         }
         finally
         {
-            _fileBasic.DeleteDirectory(sessionDir);
+            _fileStorage.DeleteDirectory(sessionDir);
         }
     }
 
@@ -78,11 +78,11 @@ private async Task<IEnumerable<string>> DownloadFiles(string dir, List<BotSharpF
                 if (!bytes.IsNullOrEmpty())
                 {
                     var guid = Guid.NewGuid().ToString();
-                    var fileDir = _fileBasic.BuildDirectory(dir, guid);
-                    DeleteIfExistDirectory(fileDir);
+                    var fileDir = _fileStorage.BuildDirectory(dir, guid);
+                    DeleteIfExistDirectory(fileDir, true);
 
-                    var pdfDir = _fileBasic.BuildDirectory(fileDir, $"{guid}.{extension}");
-                    _fileBasic.SaveFileBytesToPath(pdfDir, bytes);
+                    var pdfDir = _fileStorage.BuildDirectory(fileDir, $"{guid}.{extension}");
+                    _fileStorage.SaveFileBytesToPath(pdfDir, bytes);
                     locs.Add(pdfDir);
                 }
             }
@@ -108,8 +108,8 @@ private async Task<IEnumerable<string>> ConvertPdfToImages(IEnumerable<string> f
         {
             try
             {
-                var dir = _fileBasic.GetParentDir(file);
-                var folder = _fileBasic.BuildDirectory(dir, "screenshots");
+                var dir = _fileStorage.GetParentDir(file);
+                var folder = _fileStorage.BuildDirectory(dir, "screenshots");
                 var urls = await converter.ConvertPdfToImages(file, folder);
                 images.AddRange(urls);
             }