Add Image Generation

Author: ericstj

imageGeneratorSample.ServiceDefaults/Extensions.cs

@@ -30,7 +30,22 @@ public static TBuilder AddServiceDefaults<TBuilder>(this TBuilder builder) where
 #pragma warning restore EXTEXP0001
 
             // Turn on resilience by default
-            http.AddStandardResilienceHandler();
+            http.AddStandardResilienceHandler(httpResilienceOptions =>
+            {
+                // Set a longer timeouts.  Our image models can take a while to generate images.
+                httpResilienceOptions.TotalRequestTimeout = new()
+                {
+                    Timeout = TimeSpan.FromMinutes(10)
+                };
+
+                httpResilienceOptions.AttemptTimeout = new()
+                {
+                    Timeout = TimeSpan.FromMinutes(2)
+                };
+
+                httpResilienceOptions.CircuitBreaker.SamplingDuration = TimeSpan.FromMinutes(5);
+
+            });
 
             // Turn on service discovery by default
             http.AddServiceDiscovery();

imageGeneratorSample.Web/Components/Pages/Chat/Chat.razor

@@ -1,8 +1,12 @@
 @page "/"
 @using System.ComponentModel
+@using imageGeneratorSample.Web.Services.Images
 @inject IChatClient ChatClient
+@inject IImageGenerator ImageGenerator
+@inject IHttpClientFactory HttpClientFactory
 @inject NavigationManager Nav
 @inject SemanticSearch Search
+@inject IImageCacheService ImageCache
 @implements IDisposable
 
 <PageTitle>Chat</PageTitle>
@@ -28,6 +32,7 @@
         You are an assistant who answers questions about information you retrieve.
         Do not answer questions about anything else.
         Use only simple markdown to format your responses.
+        When sharing images use image links in the following format: ![image description](image-url).
 
         Use the search tool to find relevant information. When you do this, end your
         reply with citations in the special XML format:
@@ -46,11 +51,17 @@
     private ChatMessage? currentResponseMessage;
     private ChatInput? chatInput;
     private ChatSuggestions? chatSuggestions;
+    private HttpClient? httpClient;
 
     protected override void OnInitialized()
     {
         messages.Add(new(ChatRole.System, SystemPrompt));
-        chatOptions.Tools = [AIFunctionFactory.Create(SearchAsync)];
+        chatOptions.Tools = [
+            AIFunctionFactory.Create(SearchAsync),
+            AIFunctionFactory.Create(GenerateImagesAsync),
+            AIFunctionFactory.Create(EditImageAsync)
+        ];
+        httpClient = HttpClientFactory.CreateClient();
     }
 
     private async Task AddUserMessageAsync(ChatMessage userMessage)
@@ -111,6 +122,88 @@
             $"<result filename=\"{result.DocumentId}\" page_number=\"{result.PageNumber}\">{result.Text}</result>");
     }
 
+
+    [Description("Generates images based on a text description")]
+    private async IAsyncEnumerable<AIContent> GenerateImagesAsync(
+        [Description("A detailed description of the image to generate")] string prompt,
+        [Description("The number of images to generate.  Some models only support generating one image at a time.")] int count = 1)
+    {
+        await InvokeAsync(StateHasChanged);
+        var options = new ImageGenerationOptions()
+        {
+            Count = count,
+        };
+
+        var response = await ImageGenerator.GenerateImagesAsync(prompt, options);
+
+        foreach (var content in response.Contents)
+        {
+            var cachedContent = await CacheContent(content);
+            yield return cachedContent;
+        }
+    }
+
+    [Description("Edits an existing image based on a text description")]
+    private async IAsyncEnumerable<AIContent> EditImageAsync(
+        [Description("The URL of the image to edit")] string imageUrl,
+        [Description("A detailed description of the image to edit")] string prompt,
+        [Description("The number of images to generate.  Some models only support generating one image at a time.")] int count = 1)
+    {
+        await InvokeAsync(StateHasChanged);
+        var options = new ImageGenerationOptions()
+        {
+            Count = count,
+        };
+
+        DataContent dataContent;
+        var cachedData = await ImageCache.GetCachedImageAsync(imageUrl);
+        if (cachedData is not null)
+        {
+            // the image is cached, so we can use it directly
+            dataContent = new DataContent(cachedData.Value.imageBytes, cachedData.Value.contentType);
+        }
+        else
+        {
+            // download the image from the URL
+            var ImageGenerationResponse = await httpClient!.GetAsync(imageUrl);
+            var imageBytes = await ImageGenerationResponse.Content.ReadAsByteArrayAsync();
+
+            dataContent = new DataContent(imageBytes, ImageGenerationResponse.Content.Headers.ContentType?.MediaType ?? "image/png");
+        }
+
+        var response = await ImageGenerator.EditImagesAsync([dataContent], prompt, options);
+
+        foreach (var content in response.Contents)
+        {
+            var cachedContent = await CacheContent(content);
+            yield return cachedContent;
+        }
+    }
+
+    private async Task<AIContent> CacheContent(AIContent content)
+    {
+        if (content is DataContent dataContent)
+        {
+            var cacheUri = await ImageCache.CacheImageAsync(dataContent.Data, dataContent.MediaType);
+
+            return new UriContent(Nav.ToAbsoluteUri(cacheUri), dataContent.MediaType);
+        }
+        else if (content is UriContent uriContent)
+        {
+            // the URI returned from the client contains a secret, download and cache the file.
+            var stream = await httpClient!.GetStreamAsync(uriContent.Uri);
+
+            var cacheUri = await ImageCache.CacheImageAsync(stream, uriContent.MediaType);
+
+            return new UriContent(Nav.ToAbsoluteUri(cacheUri), uriContent.MediaType);
+        }
+        else
+        {
+            return content;
+        }
+    }
+
+
     public void Dispose()
         => currentResponseCancellation?.Cancel();
 }

imageGeneratorSample.Web/Components/Pages/Chat/ChatInput.razor

@@ -1,7 +1,7 @@
 @inject IJSRuntime JS
 
 <EditForm Model="@this" OnValidSubmit="@SendMessageAsync">
-    <label class="input-box page-width">
+    <label class="input-box page-width" @ref="@inputContainer">
         <textarea @ref="@textArea" @bind="@messageText" placeholder="Type your message..." rows="1"></textarea>
 
         <div class="tools">
@@ -16,7 +16,9 @@
 
 @code {
     private ElementReference textArea;
+    private ElementReference inputContainer;
     private string? messageText;
+    private IJSObjectReference? jsModule;
 
     [Parameter]
     public EventCallback<ChatMessage> OnSend { get; set; }
@@ -39,9 +41,22 @@
         {
             try
             {
-                var module = await JS.InvokeAsync<IJSObjectReference>("import", "./Components/Pages/Chat/ChatInput.razor.js");
-                await module.InvokeVoidAsync("init", textArea);
-                await module.DisposeAsync();
+                jsModule = await JS.InvokeAsync<IJSObjectReference>("import", "./Components/Pages/Chat/ChatInput.razor.js");
+                await jsModule.InvokeVoidAsync("init", textArea, inputContainer);
+            }
+            catch (JSDisconnectedException)
+            {
+            }
+        }
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        if (jsModule is not null)
+        {
+            try
+            {
+                await jsModule.DisposeAsync();
             }
             catch (JSDisconnectedException)
             {

imageGeneratorSample.Web/Components/Pages/Chat/ChatInput.razor.css

@@ -1,17 +1,24 @@
 .input-box {
-    display: flex; 
-    flex-direction: column; 
+    display: flex;
+    flex-direction: column;
     background: white;
     border: 1px solid rgb(229, 231, 235);
     border-radius: 8px;
     padding: 0.5rem 0.75rem;
-    margin-top: 0.75rem; 
+    margin-top: 0.75rem;
+    transition: border-color 0.2s ease, background-color 0.2s ease;
 }
 
     .input-box:focus-within {
         outline: 2px solid #4152d5;
     }
 
+    .input-box.drag-over {
+        border-color: #4152d5;
+        border-style: dashed;
+        background-color: rgba(65, 82, 213, 0.05);
+    }
+
 textarea {
     resize: none;
     border: none;

imageGeneratorSample.Web/Components/Pages/Chat/ChatInput.razor.js

@@ -1,4 +1,4 @@
-export function init(elem) {
+export function init(elem, container) {
     elem.focus();
 
     // Auto-resize whenever the user types or if the value is set programmatically
@@ -13,6 +13,9 @@
             elem.closest('form').dispatchEvent(new CustomEvent('submit', { bubbles: true, cancelable: true }));
         }
     });
+
+    // Add drag and drop functionality
+    setupDragAndDrop(container, elem)
 }
 
 function resizeToFit(elem) {
@@ -41,3 +44,85 @@ function getPropertyDescriptor(target, propertyName) {
     return Object.getOwnPropertyDescriptor(target, propertyName)
         || getPropertyDescriptor(Object.getPrototypeOf(target), propertyName);
 }
+
+function setupDragAndDrop(container, textArea) {
+    let dragOverCount = 0;
+
+    ['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
+        container.addEventListener(eventName, e => {
+            e.preventDefault();
+            e.stopPropagation();
+        }, false);
+    });
+
+    ['dragenter', 'dragover'].forEach(eventName => {
+        container.addEventListener(eventName, () => {
+            dragOverCount++;
+            container.classList.add('drag-over');
+        }, false);
+    });
+
+    ['dragleave', 'drop'].forEach(eventName => {
+        container.addEventListener(eventName, () => {
+            dragOverCount--;
+            if (dragOverCount === 0) {
+                container.classList.remove('drag-over');
+            }
+        }, false);
+    });
+
+    container.addEventListener('drop', async (e) => {
+        dragOverCount = 0;
+        container.classList.remove('drag-over');
+
+        const files = e.dataTransfer.files;
+        for (let i = 0; i < files.length; i++) {
+            const file = files[i];
+            if (file.type.startsWith('image/')) {
+                try {
+                    const reader = new FileReader();
+                    reader.onload = async () => {
+                        const base64 = reader.result.split(',')[1];
+                        await uploadImageAndUpdateTextArea(file.name, base64, file.type, textArea);
+                    };
+                    reader.readAsDataURL(file);
+                } catch (error) {
+                    console.error('Error processing image:', error);
+                }
+            }
+        }
+    }, false);
+}
+
+async function uploadImageAndUpdateTextArea(fileName, base64Data, contentType, textArea) {
+    try {
+        const response = await fetch('/api/images', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                base64Data: base64Data,
+                contentType: contentType,
+                fileName: fileName
+            })
+        });
+
+        if (response.ok) {
+            const result = await response.json();
+            const markdownText = `![${fileName}](${result.imageUri})`;
+            const currentValue = textArea.value || '';
+            // can we insert at the cursor position?
+            const newValue = currentValue.length > 0 ? `${currentValue}${markdownText}` : markdownText;
+
+            textArea.value = newValue;
+            textArea.dispatchEvent(new CustomEvent('input', { bubbles: true }));
+            textArea.dispatchEvent(new CustomEvent('change', { bubbles: true }));
+            resizeToFit(textArea);
+        } else {
+            console.error('Failed to upload image:', response.statusText);
+        }
+    } catch (error) {
+        console.error('Error uploading image:', error);
+    }
+}

imageGeneratorSample.Web/Components/Pages/Chat/ChatMessageItem.razor

@@ -5,7 +5,7 @@
 @if (Message.Role == ChatRole.User)
 {
     <div class="user-message">
-        @Message.Text
+        <markdown-message markdown="@Message.Text" />
     </div>
 }
 else if (Message.Role == ChatRole.Assistant)
@@ -24,7 +24,7 @@ else if (Message.Role == ChatRole.Assistant)
                 </div>
                 <div class="assistant-message-header">Assistant</div>
                 <div class="assistant-message-text">
-                    <assistant-message markdown="@text"></assistant-message>
+                    <markdown-message markdown="@text"></markdown-message>
 
                     @foreach (var citation in citations ?? [])
                     {