extend email with attachments

src/Infrastructure/BotSharp.Abstraction/Files/IBotSharpFileService.cs

@@ -3,9 +3,22 @@ namespace BotSharp.Abstraction.Files;
 public interface IBotSharpFileService
 {
     string GetDirectory(string conversationId);
-    Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source,
+
+    /// <summary>
+    /// Get the files that have been uploaded in the chat.
+    /// If includeScreenShot is true, it will take the screenshots of non-image files, such as pdf, and return the screenshots instead of the original file.
+    /// </summary>
+    /// <param name="conversationId"></param>
+    /// <param name="source"></param>
+    /// <param name="conversations"></param>
+    /// <param name="contentTypes"></param>
+    /// <param name="includeScreenShot"></param>
+    /// <param name="offset"></param>
+    /// <returns></returns>
+    Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
         IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
         bool includeScreenShot = false, int? offset = null);
+
     IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnumerable<string> messageIds, string source, bool imageOnly = false);
     string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName);
     IEnumerable<MessageFileModel> GetMessagesWithFile(string conversationId, IEnumerable<string> messageIds);

src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs

@@ -5,12 +5,21 @@ public class MessageFileModel
     [JsonPropertyName("message_id")]
     public string MessageId { get; set; }
 
+    /// <summary>
+    /// External file url
+    /// </summary>
     [JsonPropertyName("file_url")]
     public string FileUrl { get; set; }
 
+    /// <summary>
+    /// Internal file storage url
+    /// </summary>
     [JsonPropertyName("file_storage_url")]
     public string FileStorageUrl { get; set; }
 
+    /// <summary>
+    /// File name without extension
+    /// </summary>
     [JsonPropertyName("file_name")]
     public string FileName { get; set; }
 

src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Conversation.cs

@@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services;
 
 public partial class BotSharpFileService
 {
-    public async Task<IEnumerable<MessageFileModel>> GetChatImages(string conversationId, string source,
+    public async Task<IEnumerable<MessageFileModel>> GetChatFiles(string conversationId, string source,
         IEnumerable<RoleDialogModel> conversations, IEnumerable<string> contentTypes,
         bool includeScreenShot = false, int? offset = null)
     {
@@ -69,9 +69,7 @@ public IEnumerable<MessageFileModel> GetMessageFiles(string conversationId, IEnu
                     }
 
                     var fileName = Path.GetFileNameWithoutExtension(file);
-                    var extension = Path.GetExtension(file);
-                    var fileType = extension.Substring(1);
-
+                    var fileType = Path.GetExtension(file).Substring(1);
                     var model = new MessageFileModel()
                     {
                         MessageId = messageId,
@@ -290,10 +288,13 @@ private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, s
                         contentType = GetFileContentType(screenShot);
                         if (!_imageTypes.Contains(contentType)) continue;
 
+                        var fileName = Path.GetFileNameWithoutExtension(screenShot);
+                        var fileType = Path.GetExtension(file).Substring(1);
                         var model = new MessageFileModel()
                         {
                             MessageId = messageId,
-                            FileName = Path.GetFileName(screenShot),
+                            FileName = fileName,
+                            FileType = fileType,
                             FileStorageUrl = screenShot,
                             ContentType = contentType,
                             FileSource = source
@@ -307,10 +308,13 @@ private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, s
                     foreach (var image in images)
                     {
                         contentType = GetFileContentType(image);
+                        var fileName = Path.GetFileNameWithoutExtension(image);
+                        var fileType = Path.GetExtension(image).Substring(1);
                         var model = new MessageFileModel()
                         {
                             MessageId = messageId,
-                            FileName = Path.GetFileName(image),
+                            FileName = fileName,
+                            FileType = fileType,
                             FileStorageUrl = image,
                             ContentType = contentType,
                             FileSource = source
@@ -321,10 +325,13 @@ private async Task<IEnumerable<MessageFileModel>> GetMessageFiles(string file, s
             }
             else
             {
+                var fileName = Path.GetFileNameWithoutExtension(file);
+                var fileType = Path.GetExtension(file).Substring(1);
                 var model = new MessageFileModel()
                 {
                     MessageId = messageId,
-                    FileName = Path.GetFileName(file),
+                    FileName = fileName,
+                    FileType = fileType,
                     FileStorageUrl = file,
                     ContentType = contentType,
                     FileSource = source

src/Infrastructure/BotSharp.Core/Users/Services/UserIdentity.cs

@@ -20,7 +20,7 @@ public string Id
 
     [JsonPropertyName("user_name")]
     public string UserName
-        => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Name)?.Value!;
+        => _claims?.FirstOrDefault(x => x.Type == "name")?.Value!;
 
     public string Email
         => _claims?.FirstOrDefault(x => x.Type == ClaimTypes.Email)?.Value!;

src/Plugins/BotSharp.Plugin.EmailHandler/BotSharp.Plugin.EmailHandler.csproj

@@ -12,6 +12,7 @@
 
   <ItemGroup>
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\handle_email_request.json" />
+    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\email_attachment_prompt.liquid" />
     <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_request.fn.liquid" />
   </ItemGroup>
 
@@ -22,6 +23,9 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\handle_email_request.fn.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\email_attachment_prompt.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 
   <ItemGroup>

src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailRequestFn.cs

@@ -1,12 +1,7 @@
-using BotSharp.Abstraction.Email.Settings;
-using BotSharp.Plugin.EmailHandler.LlmContexts;
-using MailKit;
 using MailKit.Net.Smtp;
 using MailKit.Security;
-using Microsoft.AspNetCore.Http;
-using Microsoft.Extensions.Logging;
 using MimeKit;
-using System.Net.Http;
+using System.IO;
 
 namespace BotSharp.Plugin.EmailHandler.Functions;
 
@@ -22,12 +17,13 @@ public class HandleEmailRequestFn : IFunctionCallback
     private readonly BotSharpOptions _options;
     private readonly EmailHandlerSettings _emailSettings;
 
-    public HandleEmailRequestFn(IServiceProvider services,
-                ILogger<HandleEmailRequestFn> logger,
-                IHttpClientFactory httpClientFactory,
-                IHttpContextAccessor context,
-                BotSharpOptions options,
-                EmailHandlerSettings emailPluginSettings)
+    public HandleEmailRequestFn(
+        IServiceProvider services,
+        ILogger<HandleEmailRequestFn> logger,
+        IHttpClientFactory httpClientFactory,
+        IHttpContextAccessor context,
+        BotSharpOptions options,
+        EmailHandlerSettings emailPluginSettings)
     {
         _services = services;
         _logger = logger;
@@ -42,20 +38,28 @@ public async Task<bool> Execute(RoleDialogModel message)
         var recipient = args?.ToAddress;
         var body = args?.Content;
         var subject = args?.Subject;
+        var isNeedAttachments = args?.IsNeedAttachemnts ?? false;
+        var bodyBuilder = new BodyBuilder();
 
         try
         {
             var mailMessage = new MimeMessage();
             mailMessage.From.Add(new MailboxAddress(_emailSettings.Name, _emailSettings.EmailAddress));
             mailMessage.To.Add(new MailboxAddress("", recipient));
             mailMessage.Subject = subject;
-            mailMessage.Body = new TextPart("plain")
+            bodyBuilder.TextBody = body;
+
+            if (isNeedAttachments)
             {
-                Text = body
-            };
+                var files = await GetConversationFiles();
+                BuildEmailAttachments(bodyBuilder, files);
+            }
+
+            mailMessage.Body = bodyBuilder.ToMessageBody();
             var response = await HandleSendEmailBySMTP(mailMessage);
-            _logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]");
             message.Content = response;
+
+            _logger.LogWarning($"Email successfully send over to {recipient}. Email Subject: {subject} [{response}]");
             return true;
         }
         catch (Exception ex)
@@ -67,7 +71,76 @@ public async Task<bool> Execute(RoleDialogModel message)
         }
     }
 
-    public async Task<string> HandleSendEmailBySMTP(MimeMessage mailMessage)
+    private async Task<IEnumerable<MessageFileModel>> GetConversationFiles()
+    {
+        var convService = _services.GetService<IConversationService>();
+        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        var conversationId = convService.ConversationId;
+        var dialogs = convService.GetDialogHistory(fromBreakpoint: false);
+        var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList();
+        var files = fileService.GetMessageFiles(conversationId, messageIds, FileSourceType.User);
+        return await SelectFiles(files, dialogs);
+    }
+
+    private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<MessageFileModel> files, List<RoleDialogModel> dialogs)
+    {
+        if (files.IsNullOrEmpty()) return new List<MessageFileModel>();
+
+        var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
+        var render = _services.GetRequiredService<ITemplateRender>();
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+
+        try
+        {
+            var promptFiles = files.Select((x, idx) =>
+            {
+                return $"id: {idx + 1}, file_name: {x.FileName}.{x.FileType}, content_type: {x.ContentType}";
+            }).ToList();
+            var prompt = db.GetAgentTemplate(BuiltInAgentId.UtilityAssistant, "email_attachment_prompt");
+            prompt = render.Render(prompt, new Dictionary<string, object>
+            {
+                { "file_list", promptFiles }
+            });
+
+            var agent = new Agent
+            {
+                Id = BuiltInAgentId.UtilityAssistant,
+                Name = "Utility Assistant",
+                Instruction = prompt
+            };
+
+            var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == "openai");
+            var model = llmProviderService.GetProviderModel(provider: provider, id: "gpt-4");
+            var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);
+            var response = await completion.GetChatCompletions(agent, dialogs);
+            var content = response?.Content ?? string.Empty;
+            var fids = JsonSerializer.Deserialize<List<int>>(content) ?? new List<int>();
+            return files.Where((x, idx) => fids.Contains(idx + 1)).ToList();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning($"Error when getting the email file response. {ex.Message}\r\n{ex.InnerException}");
+            return new List<MessageFileModel>();
+        }
+    }
+
+    private void BuildEmailAttachments(BodyBuilder builder, IEnumerable<MessageFileModel> files)
+    {
+        if (files.IsNullOrEmpty()) return;
+
+        foreach (var file in files)
+        {
+            if (string.IsNullOrEmpty(file.FileStorageUrl)) continue;
+
+            using var fs = File.OpenRead(file.FileStorageUrl);
+            var binary = BinaryData.FromStream(fs);
+            builder.Attachments.Add($"{file.FileName}.{file.FileType}", binary.ToArray(), ContentType.Parse(file.ContentType));
+            fs.Close();
+            Thread.Sleep(100);
+        }
+    }
+
+    private async Task<string> HandleSendEmailBySMTP(MimeMessage mailMessage)
     {
         using var smtpClient = new SmtpClient();
         await smtpClient.ConnectAsync(_emailSettings.SMTPServer, _emailSettings.SMTPPort, SecureSocketOptions.StartTls);

src/Plugins/BotSharp.Plugin.EmailHandler/LlmContexts/LlmContextIn.cs

@@ -1,20 +1,18 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 using System.Text.Json.Serialization;
-using System.Threading.Tasks;
 
-namespace BotSharp.Plugin.EmailHandler.LlmContexts
+namespace BotSharp.Plugin.EmailHandler.LlmContexts;
+
+public class LlmContextIn
 {
-    public class LlmContextIn
-    {
-        [JsonPropertyName("to_address")]
-        public string? ToAddress { get; set; }
+    [JsonPropertyName("to_address")]
+    public string? ToAddress { get; set; }
+
+    [JsonPropertyName("email_content")]
+    public string? Content { get; set; }
+
+    [JsonPropertyName("subject")]
+    public string? Subject { get; set; }
 
-        [JsonPropertyName("email_content")]
-        public string? Content { get; set; }
-        [JsonPropertyName("subject")]
-        public string? Subject { get; set; }
-    }
+    [JsonPropertyName("is_need_attachments")]
+    public bool IsNeedAttachemnts { get; set; }
 }

src/Plugins/BotSharp.Plugin.EmailHandler/Using.cs

@@ -1,18 +1,31 @@
 global using System;
 global using System.Collections.Generic;
 global using System.Text;
+global using System.Linq;
+global using System.Text.Json;
+global using System.Net.Http;
+global using System.Threading;
+global using System.Threading.Tasks;
+global using Microsoft.AspNetCore.Http;
+global using Microsoft.Extensions.Logging;
+global using Microsoft.Extensions.DependencyInjection;
 global using BotSharp.Abstraction.Conversations;
 global using BotSharp.Abstraction.Plugins;
-global using System.Text.Json;
 global using BotSharp.Abstraction.Conversations.Models;
-global using System.Threading.Tasks;
 global using BotSharp.Abstraction.Functions;
 global using BotSharp.Abstraction.Agents.Models;
 global using BotSharp.Abstraction.Templating;
-global using Microsoft.Extensions.DependencyInjection;
-global using System.Linq;
 global using BotSharp.Abstraction.Utilities;
 global using BotSharp.Abstraction.Messaging;
 global using BotSharp.Abstraction.Messaging.Models.RichContent;
 global using BotSharp.Abstraction.Options;
-global using BotSharp.Abstraction.Messaging.Enums;
+global using BotSharp.Abstraction.Messaging.Enums;
+global using BotSharp.Abstraction.Agents.Enums;
+global using BotSharp.Abstraction.Email.Settings;
+global using BotSharp.Abstraction.Files;
+global using BotSharp.Abstraction.Files.Enums;
+global using BotSharp.Abstraction.Files.Models;
+global using BotSharp.Abstraction.MLTasks;
+global using BotSharp.Abstraction.Repositories;
+global using BotSharp.Core.Infrastructures;
+global using BotSharp.Plugin.EmailHandler.LlmContexts;

src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_email_request.json

@@ -15,8 +15,12 @@
       "subject": {
         "type": "string",
         "description": "The subject of the email which needs to be send over."
+      },
+      "is_need_attachments": {
+        "type": "boolean",
+        "description": "If the user request to send email with attachemnt(s), then this value should be true. Otherwise, this value should be false."
       }
     },
-    "required": [ "to_address", "email_content", "subject" ]
+    "required": [ "to_address", "email_content", "subject", "is_need_attachments" ]
   }
 }

src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/email_attachment_prompt.liquid

@@ -0,0 +1,20 @@
+Please take a look at the files in the [FILES] section from the conversation and select the files based on the conversation with user.
+Your response must be a list of file ids.
+** Please only output the list. Do not prepend or append anything.
+
+For example:
+Suppose there are three files:
+
+id: 1, file_name: example_file.png, content_type: image/png
+id: 2, file_name: example_file.jpeg, content_type: image/jpeg
+id: 3, file_name: example_file.pdf, content_type: application/pdf
+
+If user wants the first file and the third file, the ouput should be [1, 3].
+If user wants the all the images, the output should be [1, 2].
+If user wants the pdf file, the output should be [3].
+If user does not want any files, the ouput should be [];
+
+[FILES]
+{% for file in file_list -%}
+{{ file }}{{ "\r\n" }}
+{%- endfor %}

src/Plugins/BotSharp.Plugin.EmailHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/templates/handle_email_request.fn.liquid

@@ -1 +1,2 @@
+Suppose user has uploaded some attachments.
 Please call handle_email_request if user wants to send out an email.