Copilot Chat: Feature/tesseract ocr Issue #1440

.gitignore

@@ -477,4 +477,5 @@ playwright-report/
 # Static Web App deployment config
 swa-cli.config.json
 **/copilot-chat-app/webapp/build
-**/copilot-chat-app/webapp/node_modules
+**/copilot-chat-app/webapp/node_modules
+**/copilot-chat-app/webapi/data/eng.traineddata

samples/apps/copilot-chat-app/webapi/CopilotChat/Controllers/DocumentImportController.cs

@@ -18,6 +18,8 @@
 using SemanticKernel.Service.CopilotChat.Models;
 using SemanticKernel.Service.CopilotChat.Options;
 using SemanticKernel.Service.CopilotChat.Storage;
+using SemanticKernel.Service.Services;
+using Tesseract;
 using UglyToad.PdfPig;
 using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
 using static SemanticKernel.Service.CopilotChat.Models.MemorySource;
@@ -44,6 +46,21 @@ private enum SupportedFileType
         /// .pdf
         /// </summary>
         Pdf,
+
+        /// <summary>
+        /// .jpg
+        /// </summary>
+        Jpg,
+
+        /// <summary>
+        /// .png
+        /// </summary>
+        Png,
+
+        /// <summary>
+        /// .tif or .tiff
+        /// </summary>
+        Tiff
     };
 
     private readonly ILogger<DocumentImportController> _logger;
@@ -54,6 +71,7 @@ private enum SupportedFileType
     private readonly ChatParticipantRepository _participantRepository;
     private const string GlobalDocumentUploadedClientCall = "GlobalDocumentUploaded";
     private const string ChatDocumentUploadedClientCall = "ChatDocumentUploaded";
+    private readonly ITesseractEngine _tesseractEngine;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="DocumentImportController"/> class.
@@ -64,14 +82,16 @@ public DocumentImportController(
         ChatSessionRepository sessionRepository,
         ChatMemorySourceRepository sourceRepository,
         ChatMessageRepository messageRepository,
-        ChatParticipantRepository participantRepository)
+        ChatParticipantRepository participantRepository,
+        ITesseractEngine tesseractEngine)
     {
         this._logger = logger;
         this._options = documentMemoryOptions.Value;
         this._sessionRepository = sessionRepository;
         this._sourceRepository = sourceRepository;
         this._messageRepository = messageRepository;
         this._participantRepository = participantRepository;
+        this._tesseractEngine = tesseractEngine;
     }
 
     /// <summary>
@@ -259,6 +279,14 @@ private async Task<ImportResult> ImportDocumentHelperAsync(IKernel kernel, IForm
             case SupportedFileType.Pdf:
                 documentContent = this.ReadPdfFile(formFile);
                 break;
+            case SupportedFileType.Jpg:
+            case SupportedFileType.Png:
+            case SupportedFileType.Tiff:
+            {
+                documentContent = await this.ReadTextFromImageFileAsync(formFile);
+                break;
+            }
+
             default:
                 // This should never happen. Validation should have already caught this.
                 return ImportResult.Fail();
@@ -391,10 +419,35 @@ private SupportedFileType GetFileType(string fileName)
         {
             ".txt" => SupportedFileType.Txt,
             ".pdf" => SupportedFileType.Pdf,
+            ".jpg" => SupportedFileType.Jpg,
+            ".jpeg" => SupportedFileType.Jpg,
+            ".png" => SupportedFileType.Png,
+            ".tif" => SupportedFileType.Tiff,
+            ".tiff" => SupportedFileType.Tiff,
             _ => throw new ArgumentOutOfRangeException($"Unsupported file type: {extension}"),
         };
     }
 
+    /// <summary>
+    /// Reads the text content from an image file.
+    /// </summary>
+    /// <param name="file">An IFormFile object.</param>
+    /// <returns>A string of the content of the file.</returns>
+    private async Task<string> ReadTextFromImageFileAsync(IFormFile file)
+    {
+        await using (var ms = new MemoryStream())
+        {
+            await file.CopyToAsync(ms);
+            var fileBytes = ms.ToArray();
+            await using var imgStream = new MemoryStream(fileBytes);
+
+            using var img = Pix.LoadFromMemory(imgStream.ToArray());
+
+            using var page = this._tesseractEngine.Process(img);
+            return page.GetText();
+        }
+    }
+
     /// <summary>
     /// Read the content of a text file.
     /// </summary>

samples/apps/copilot-chat-app/webapi/CopilotChat/Extensions/ServiceExtensions.cs

@@ -11,6 +11,8 @@
 using SemanticKernel.Service.CopilotChat.Options;
 using SemanticKernel.Service.CopilotChat.Storage;
 using SemanticKernel.Service.Options;
+using SemanticKernel.Service.Services;
+using Tesseract;
 
 namespace SemanticKernel.Service.CopilotChat.Extensions;
 
@@ -68,13 +70,50 @@ public static IServiceCollection AddCopilotChatOptions(this IServiceCollection s
             .ValidateOnStart()
             .PostConfigure(TrimStringProperties);
 
+        // OCR support options
+        services.AddOptions<OcrSupportOptions>()
+            .Bind(configuration.GetSection(OcrSupportOptions.PropertyName))
+            .ValidateOnStart()
+            .PostConfigure(TrimStringProperties);
+
+        return services;
+    }
+
+    /// <summary>
+    /// Adds persistent OCR support service.
+    /// </summary>
+    /// <exception cref="InvalidOperationException"></exception>
+    public static IServiceCollection AddPersistentOcrSupport(this IServiceCollection services)
+    {
+        OcrSupportOptions ocrSupportConfig = services.BuildServiceProvider().GetRequiredService<IOptions<OcrSupportOptions>>().Value;
+
+        switch (ocrSupportConfig.Type)
+        {
+            case OcrSupportOptions.OcrSupportType.Tesseract:
+            {
+                services.AddSingleton<ITesseractEngine>(sp => new TesseractEngineWrapper(new TesseractEngine(ocrSupportConfig.Tesseract!.FilePath, ocrSupportConfig.Tesseract!.Language, EngineMode.Default)));
+                break;
+            }
+
+            case OcrSupportOptions.OcrSupportType.None:
+            {
+                services.AddSingleton<ITesseractEngine>(sp => new NullTesseractEngine());
+                break;
+            }
+
+            default:
+            {
+                throw new InvalidOperationException($"Unsupported OcrSupport:Type '{ocrSupportConfig.Type}'");
+            }
+        }
+
         return services;
     }
 
     /// <summary>
     /// Add persistent chat store services.
     /// </summary>
-    public static void AddPersistentChatStore(this IServiceCollection services)
+    public static IServiceCollection AddPersistentChatStore(this IServiceCollection services)
     {
         IStorageContext<ChatSession> chatSessionStorageContext;
         IStorageContext<ChatMessage> chatMessageStorageContext;
@@ -144,6 +183,8 @@ public static void AddPersistentChatStore(this IServiceCollection services)
         services.AddSingleton<ChatMessageRepository>(new ChatMessageRepository(chatMessageStorageContext));
         services.AddSingleton<ChatMemorySourceRepository>(new ChatMemorySourceRepository(chatMemorySourceStorageContext));
         services.AddSingleton<ChatParticipantRepository>(new ChatParticipantRepository(chatParticipantStorageContext));
+
+        return services;
     }
 
     /// <summary>

samples/apps/copilot-chat-app/webapi/CopilotChat/Options/OcrSupportOptions.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using SemanticKernel.Service.Options;
+
+namespace SemanticKernel.Service.CopilotChat.Options;
+
+/// <summary>
+/// Ocr Support Configuration Options
+/// </summary>
+public class OcrSupportOptions
+{
+    public const string PropertyName = "OcrSupport";
+
+    public enum OcrSupportType
+    {
+        /// <summary>
+        /// No OCR Support
+        /// </summary>
+        None,
+
+        /// <summary>
+        /// Tesseract OCR Support
+        /// </summary>
+        Tesseract
+    }
+
+    /// <summary>
+    /// Gets or sets the type of OCR support to use.
+    /// </summary>
+    public OcrSupportType Type { get; set; } = OcrSupportType.None;
+
+    /// <summary>
+    /// Gets or sets the configuration for the Tesseract OCR support.
+    /// </summary>
+    [RequiredOnPropertyValue(nameof(Type), OcrSupportType.Tesseract)]
+    public TesseractOptions? Tesseract { get; set; }
+}

samples/apps/copilot-chat-app/webapi/CopilotChat/Options/TesseractOptions.cs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+using SemanticKernel.Service.Options;
+
+namespace SemanticKernel.Service.CopilotChat.Options;
+
+/// <summary>
+/// Configuration options for Tesseract OCR support.
+/// </summary>
+public sealed class TesseractOptions
+{
+    public const string PropertyName = "Tesseract";
+
+    /// <summary>
+    /// The file path where the Tesseract language file is stored (e.g. "./data")
+    /// </summary>
+    [Required, NotEmptyOrWhitespace]
+    public string? FilePath { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The language file prefix name (e.g. "eng")
+    /// </summary>
+    [Required, NotEmptyOrWhitespace]
+    public string? Language { get; set; } = string.Empty;
+}

samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.csproj

@@ -62,11 +62,19 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+
+    <PackageReference Include="Tesseract" Version="5.2.0" />
   </ItemGroup>
 
   <ItemGroup>
     <AssemblyAttribute Include="System.CLSCompliantAttribute">
       <_Parameter1>false</_Parameter1>
     </AssemblyAttribute>
   </ItemGroup>
+
+  <ItemGroup>
+    <None Update="data\eng.traineddata">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
 </Project>

samples/apps/copilot-chat-app/webapi/CopilotChatWebApi.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.33530.505
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CopilotChatWebApi", "CopilotChatWebApi.csproj", "{35CC3A68-E577-4B21-B94C-BF674F8FA505}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{35CC3A68-E577-4B21-B94C-BF674F8FA505}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{35CC3A68-E577-4B21-B94C-BF674F8FA505}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{35CC3A68-E577-4B21-B94C-BF674F8FA505}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{35CC3A68-E577-4B21-B94C-BF674F8FA505}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {1F60AC39-60D2-4CD2-B2FC-71E174DDFC1A}
+	EndGlobalSection
+EndGlobal

samples/apps/copilot-chat-app/webapi/Program.cs

@@ -47,7 +47,8 @@ public static async Task Main(string[] args)
         builder.Services
             .AddCopilotChatOptions(builder.Configuration)
             .AddCopilotChatPlannerServices()
-            .AddPersistentChatStore();
+            .AddPersistentChatStore()
+            .AddPersistentOcrSupport();
 
         // Add SignalR as the real time relay service
         builder.Services.AddSignalR();

samples/apps/copilot-chat-app/webapi/README.md

@@ -9,8 +9,9 @@ while allowing user interfaces to be developed using frontend frameworks such as
 Before you get started, make sure you have the following requirements in place:
 
 1. [.NET 6.0](https://dotnet.microsoft.com/en-us/download/dotnet/6.0) for building and deploying .NET 6 projects.
-2. Update the properties in `./appsettings.json` to configure your Azure OpenAI resource or OpenAI account.
-3. Generate and trust a localhost developer certificate.
+2. **(Optional)** [Visual Studio Code](http://aka.ms/vscode) or [Visual Studio](http://aka.ms/vsdownload).
+3. Update the properties in `./appsettings.json` to configure your Azure OpenAI resource or OpenAI account.
+4. Generate and trust a localhost developer certificate.
    - For Windows and Mac run
      ```bash
      dotnet dev-certs https --trust
@@ -25,7 +26,9 @@ Before you get started, make sure you have the following requirements in place:
 
    > To clean your system of the developer certificate, run `dotnet run dev-certs https --clean`
 
-4. **(Optional)** [Visual Studio Code](http://aka.ms/vscode) or [Visual Studio](http://aka.ms/vsdownload).
+5. **(Optional)** To enable support for uploading image file formats such as png, jpg and tiff, we have included the [Tesseract](https://www.nuget.org/packages/Tesseract) nuget package.  
+   - You will need to obtain one or more [tessdata language data files](https://github.com/tesseract-ocr/tessdata) such as `eng.traineddata` and add them to your `./data` directory or the location specified in the `Tesseract.FilePath` location in `./appsettings.json`.  
+   - Set the `Copy to Output Directory` value to `Copy if newer`.
 
 # Start the WebApi Service
 

samples/apps/copilot-chat-app/webapi/Services/ITesseractEngine.cs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Tesseract;
+
+namespace SemanticKernel.Service.Services;
+
+/// <summary>
+/// Wrapper for the Tesseract engine.
+/// </summary>
+public interface ITesseractEngine
+{
+    //
+    // Summary:
+    //     Processes the specific image.
+    //
+    // Parameters:
+    //   image:
+    //     The image to process.
+    //
+    //   pageSegMode:
+    //     The page layout analyasis method to use.
+    //
+    // Remarks:
+    //     You can only have one result iterator open at any one time.
+    Page Process(Pix image);
+}

samples/apps/copilot-chat-app/webapi/Services/NullTesseractEngine.cs

@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using Tesseract;
+
+namespace SemanticKernel.Service.Services;
+
+/// <summary>
+/// Used to mock the TesseractEngine in the event that the Tesseract language file is not installed.
+/// </summary>
+public class NullTesseractEngine : ITesseractEngine
+{
+    /// <summary>
+    /// Throws an exception to let the user know they need to install the Tesseract language file.
+    /// </summary>
+    /// <param name="image">Not used</param>
+    /// <returns>This will always throw a NotImplementedException</returns>
+    /// <exception cref="NotImplementedException"></exception>
+    public Page Process(Pix image)
+    {
+        throw new NotImplementedException("You must have the Tesseract language file to use the image upload feature. See the README.md");
+    }
+}