Add session resumption, transcription, and server events support

Enhanced multimodal live client with session resumption and transcription handling. Added server-side events (`GoAway`, `SessionResumableUpdate`) and extended functionality in WPF sample for better UI and transcription display. Updated types to support session resumption, context compression, and URL metadata.

Author: Gunpal Jain

Google_GenerativeAI.sln.DotSettings

@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeEditing/SuppressUninitializedWarningFix/Enabled/@EntryValue">False</s:Boolean></wpf:ResourceDictionary>

samples/TwoWayAudioCommunicationWpf/MainWindow.xaml

@@ -53,6 +53,14 @@
                 </StackPanel>
 
             </ui:Card>
+            <StackPanel DockPanel.Dock="Bottom">
+                <TextBlock Text="Input Transcript" Margin="0,0,0,8"></TextBlock>
+                <TextBox TextWrapping="Wrap" Text="{Binding InputTranscript}" Height="100"></TextBox>
+            </StackPanel>
+            <StackPanel DockPanel.Dock="Bottom">
+                <TextBlock Text="Input Transcript" Margin="0,0,0,8"></TextBlock>
+                <TextBox TextWrapping="Wrap" Text="{Binding OutputTranscript}" Height="100"></TextBox>
+            </StackPanel>
             <ListBox x:Name="Responses" ItemsSource="{Binding ModelResponses, ElementName=Window}"
                      VerticalAlignment="Stretch"
                      BorderThickness="0"

samples/TwoWayAudioCommunicationWpf/MainWindow.xaml.cs

@@ -3,7 +3,9 @@
 using System.Diagnostics;
 using System.IO;
 using System.Runtime.CompilerServices;
+using System.Text.Json;
 using System.Windows;
+using System.Windows.Forms.VisualStyles;
 using GenerativeAI;
 using GenerativeAI.Live;
 using GenerativeAI.Types;
@@ -26,6 +28,28 @@ public partial class MainWindow : FluentWindow, INotifyPropertyChanged
     private ModelResponse? _currentResponse;
     private readonly NAudioHelper _audioHelper = new();
 
+    public string InputTranscript
+    {
+        get => _inputTranscript;
+        set
+        {
+            if (value == _inputTranscript) return;
+            _inputTranscript = value;
+            OnPropertyChanged();
+        }
+    }
+
+    public string OutputTranscript
+    {
+        get => _outputTranscript;
+        set
+        {
+            if (value == _outputTranscript) return;
+            _outputTranscript = value;
+            OnPropertyChanged();
+        }
+    }
+
     public bool IsRecording
     {
         get => _isRecording;
@@ -37,8 +61,12 @@ public bool IsRecording
         }
     }
 
+    private const string Model = "gemini-2.0-flash-live-001";
+
     private CancellationTokenSource? _cancellationTokenSource;
     private bool _isRecording = false;
+    private string _inputTranscript;
+    private string _outputTranscript;
 
     /// <summary>
     /// Gets or sets the collection of model responses displayed in the UI.
@@ -161,7 +189,14 @@ private async Task InitializeChat()
 
             RegisterClientEvents();
 
-            await _multiModalLiveClient.ConnectAsync(true, cancellationToken);
+            await _multiModalLiveClient.ConnectAsync(false, cancellationToken);
+            await _multiModalLiveClient.SendSetupAsync(new BidiGenerateContentSetup()
+            {
+                Model = Model.ToModelId(),
+                GenerationConfig = config,
+                OutputAudioTranscription = new AudioTranscriptionConfig(),
+                InputAudioTranscription = new AudioTranscriptionConfig()
+            }, cancellationToken);
             StartRecording(device); //No need to pass cancellation token, as it's handled within StartRecording
             btnStartChat.Content = AppConstants.StopChatText;
         }
@@ -212,6 +247,24 @@ private void RegisterClientEvents()
             }
         };
 
+        _multiModalLiveClient.TextChunkReceived += (sender, args) =>
+        {
+            Console.WriteLine(args.Text);
+        };
+        _multiModalLiveClient.InputTranscriptionReceived += (sender, args) =>
+        {
+            InputTranscript = $"{InputTranscript}{args.Text}";
+        };
+        
+        _multiModalLiveClient.OutputTranscriptionReceived += (sender, args) =>
+        {
+            OutputTranscript = $"{OutputTranscript}{args.Text}";
+        };
+
+        _multiModalLiveClient.SessionResumableUpdateReceived += (sender, args) =>
+        {
+            Console.WriteLine(JsonSerializer.Serialize(args));
+        };
         _multiModalLiveClient.AudioChunkReceived += (sender, args) =>
         {
             Dispatcher.Invoke(() => // Batch UI updates for better performance
@@ -237,6 +290,7 @@ private void RegisterClientEvents()
             _audioHelper.ClearPlayback(); //Consider if this needs to be inside the Dispatcher.Invoke
             var tmpFile = Path.GetTempFileName() + ".wav";
 
+           
             //Consider handling IOException
             try
             {

samples/TwoWayAudioCommunicationWpf/TwoWayAudioCommunicationWpf.csproj

@@ -11,7 +11,6 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Google_GenerativeAI.Auth" Version="2.5.6" />
       <PackageReference Include="MdXaml.Full" Version="1.27.0" />
       <PackageReference Include="NAudio" Version="2.2.0" />
       <PackageReference Include="PvRecorder" Version="1.2.10" />

src/GenerativeAI.Live/Events/AudioBufferReceivedEventArgs.cs

@@ -1,3 +1,5 @@
+using GenerativeAI.Types;
+
 namespace GenerativeAI.Live;
 
 /// <summary>
@@ -14,6 +16,9 @@ public class AudioBufferReceivedEventArgs : EventArgs
     /// Gets or sets the header information for the audio data.
     /// </summary>
     public AudioHeaderInfo HeaderInfo { get; set; }
+    
+    public Transcription? InputTranscription { get; set; }
+    public Transcription? OutputTranscription { get; set; }
 
     public AudioBufferReceivedEventArgs(byte[] buffer, AudioHeaderInfo audioHeaderInfo)
     {

src/GenerativeAI.Live/GenerativeAI.Live.csproj.DotSettings

@@ -0,0 +1,5 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=classes/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=events/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=models/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/CodeInspection/NamespaceProvider/NamespaceFoldersToSkip/=types/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

src/GenerativeAI.Live/Models/MultiModalLiveClient.cs

@@ -169,8 +169,36 @@ public MultiModalLiveClient(IPlatformAdapter platformAdapter, string modelName,
     /// </summary>
     public event EventHandler<ErrorEventArgs>? ErrorOccurred;
 
+    /// <summary>
+    /// Event triggered when a chunk of text is received from the server during the live API session.
+    /// </summary>
     public event EventHandler<TextChunkReceivedArgs>? TextChunkReceived;
 
+    /// <summary>
+    /// Event triggered upon receiving input transcription data.
+    /// </summary>
+    public event EventHandler<Transcription>? InputTranscriptionReceived;
+
+    /// <summary>
+    /// An event triggered when an output transcription is received from the system.
+    /// </summary>
+    public event EventHandler<Transcription>? OutputTranscriptionReceived;
+
+    /// <summary>
+    /// Message sent by the server to indicate that the current connection should be terminated
+    /// and the client should cease sending further requests on this stream.
+    /// This is often used for graceful shutdown or when the server is no longer able to
+    /// process requests on the current stream.
+    /// </summary>
+    public event EventHandler<LiveServerGoAway>? GoAwayReceived;
+
+    /// <summary>
+    /// Occurs when the server sends an update that allows the current session to be resumed.
+    /// This event provides information related to session resumption, enabling the client to continue
+    /// an existing session without starting over.
+    /// </summary>
+    public event EventHandler<LiveServerSessionResumptionUpdate>? SessionResumableUpdateReceived; 
+
     #endregion
 
     #region Private Methods
@@ -202,6 +230,10 @@ private void ProcessReceivedMessage(ResponseMessage msg)
             }
             ProcessTextChunk(responsePayload);
             ProcessAudioChunk(responsePayload);
+            ProcessInputTranscription(responsePayload);
+            ProcessOutputTranscription(responsePayload);
+            ProcessSessionResumableUpdate(responsePayload);
+            ProcessGoAway(responsePayload);
 
             MessageReceived?.Invoke(this, new MessageReceivedEventArgs(responsePayload));
         }
@@ -255,12 +287,15 @@ private void ProcessAudioChunk(BidiResponsePayload responsePayload)
         {
             var audioBlobs = responsePayload.ServerContent.ModelTurn.Parts
                 .Where(p => p.InlineData?.MimeType?.Contains("audio") == true)
-                .Select(p => p.InlineData!)
+                
                 .ToList();
 
             foreach (var blob in audioBlobs)
             {
-                ProcessAudioBlob(blob);
+                if (blob.InlineData != null)
+                {
+                    ProcessAudioBlob(blob.InlineData);
+                }
             }
         }
 
@@ -293,7 +328,8 @@ private void ProcessAudioBlob(Blob blob)
             this._lastHeaderInfo = headerInfo;
 
             var bufferReceived = new AudioBufferReceivedEventArgs(audioBuffer, headerInfo);
-
+        
+            
             _audioBuffer.AddRange(audioBuffer);
             _logger?.LogAudioChunkReceived(sampleRate, hasHeader, bufferReceived.Buffer.Length);
             AudioChunkReceived?.Invoke(this, bufferReceived);
@@ -347,6 +383,38 @@ private void HandleInterruption()
         _audioBuffer.Clear();
     }
 
+    private void ProcessInputTranscription(BidiResponsePayload responsePayload)
+    {
+        if (responsePayload.ServerContent?.InputTranscription != null)
+        {
+            InputTranscriptionReceived?.Invoke(this, responsePayload.ServerContent.InputTranscription);
+        }
+    }
+    
+    private void ProcessOutputTranscription(BidiResponsePayload responsePayload)
+    {
+        if (responsePayload.ServerContent?.OutputTranscription != null)
+        {
+            OutputTranscriptionReceived?.Invoke(this, responsePayload.ServerContent.OutputTranscription);
+        }
+    }
+    
+    private void ProcessSessionResumableUpdate(BidiResponsePayload responsePayload)
+    {
+        if (responsePayload.SessionResumptionUpdate != null)
+        {
+            SessionResumableUpdateReceived?.Invoke(this, responsePayload.SessionResumptionUpdate);
+        }
+    }
+    
+    private void ProcessGoAway(BidiResponsePayload responsePayload)
+    {
+        if (responsePayload.GoAway != null)
+        {
+            GoAwayReceived?.Invoke(this, responsePayload.GoAway);
+        }
+    }
+    
     private async Task CallFunctions(BidiGenerateContentToolCall responsePayloadToolCall,
         CancellationToken cancellationToken = default)
     {
@@ -531,6 +599,8 @@ public async Task DisconnectAsync(CancellationToken cancellationToken = default)
     /// </returns>
     public async Task SendSetupAsync(BidiGenerateContentSetup setup, CancellationToken cancellationToken = default)
     {
+        if(!setup.Model.Contains("/"))
+            throw new ArgumentException("Please provide a valid model name such as 'models/gemini-2.0-flash-live-001'.");
         var payload = new BidiClientPayload { Setup = setup };
         await SendAsync(payload, cancellationToken).ConfigureAwait(false);
     }

src/GenerativeAI/Types/MultimodalLive/BidiGenerateContentServerContent.cs

@@ -14,7 +14,6 @@ public class BidiGenerateContentServerContent
     /// </summary>
     [JsonPropertyName("turnComplete")]
     public bool? TurnComplete { get; set; }
-
     /// <summary>
     /// Output only. If true, indicates that a client message has interrupted current model generation. If the client is playing out the content in real time, this is a good signal to stop and empty the current playback queue.
     /// </summary>
@@ -32,4 +31,96 @@ public class BidiGenerateContentServerContent
     /// </summary>
     [JsonPropertyName("modelTurn")]
     public Content? ModelTurn { get; set; }
+    
+    /// <summary>
+    /// Output only. If true, indicates that the model has completed generating all content for the current request.
+    /// </summary>
+    [JsonPropertyName("generationComplete")]
+    public bool? GenerationComplete { get; set; }
+    
+    /// <summary>
+    /// Output only. Transcription of the input content.
+    /// </summary>
+    [JsonPropertyName("inputTranscription")]
+    public Transcription? InputTranscription { get; set; }
+    
+    /// <summary>
+    /// Output only. Transcription of the output content.
+    /// </summary>
+    [JsonPropertyName("outputTranscription")]
+    public Transcription? OutputTranscription { get; set; }
+    
+    /// <summary>
+    /// Output only. Metadata for URL context.
+    /// </summary>
+    [JsonPropertyName("urlContextMetadata")]
+    public UrlContextMetadata? UrlContextMetadata { get; set; }
+}
+
+/// <summary>
+/// Audio transcription in Server Content.
+/// </summary>
+public class Transcription
+{
+    /// <summary>
+    /// The bool indicates the end of the transcription.
+    /// </summary>
+    [JsonPropertyName("finished")]
+    public bool? Finished { get; set; }
+
+    /// <summary>
+    /// Transcription text.
+    /// </summary>
+    [JsonPropertyName("text")]
+    public string? Text { get; set; }
+}
+/// <summary>
+/// Status of the url retrieval.
+/// </summary>
+public enum UrlRetrievalStatus
+{
+    /// <summary>
+    /// Default value. This value is unused.
+    /// </summary>
+    URL_RETRIEVAL_STATUS_UNSPECIFIED,
+
+    /// <summary>
+    /// Url retrieval is successful.
+    /// </summary>
+    URL_RETRIEVAL_STATUS_SUCCESS,
+
+    /// <summary>
+    /// Url retrieval is failed due to error.
+    /// </summary>
+    URL_RETRIEVAL_STATUS_ERROR
+}
+
+/// <summary>
+/// Context for a single url retrieval.
+/// </summary>
+public class UrlMetadata
+{
+    /// <summary>
+    /// The URL retrieved by the tool.
+    /// </summary>
+    [JsonPropertyName("retrievedUrl")]
+    public string? RetrievedUrl { get; set; }
+
+    /// <summary>
+    /// Status of the url retrieval.
+    /// </summary>
+    [JsonPropertyName("urlRetrievalStatus")]
+    public UrlRetrievalStatus? UrlRetrievalStatus { get; set; }
+}
+
+/// <summary>
+/// Metadata related to url context retrieval tool.
+/// </summary>
+public class UrlContextMetadata
+{
+    /// <summary>
+    /// List of url context.
+    /// </summary>
+    [JsonPropertyName("urlMetadata")]
+    public List<UrlMetadata>? UrlMetadata { get; set; }
 }