Skip to content

Commit 957608c

Browse files
rogerbarretorogerbarreto
authored
.Net: Fix Google Gemini Tools SchemaGeneration to comply with Open API 3.0 (#11751)
### Motivation and Context Similar to the StructuredOutput solution, this fix will now also cover the Tool generation when the StructuredOutput feature is not used. This change also brings a new Integration test to ensure the compatible behavior as well as a sample inspired in the reproduction code in the issue below. - Fixes #11675
1 parent 68b11d6 commit 957608c

File tree

7 files changed

+139
-7
lines changed

7 files changed

+139
-7
lines changed

‎dotnet/samples/Concepts/FunctionCalling/Gemini_FunctionCalling.cs‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System.ComponentModel;
34
using Microsoft.SemanticKernel;
45
using Microsoft.SemanticKernel.ChatCompletion;
56
using Microsoft.SemanticKernel.Connectors.Google;
@@ -10,7 +11,7 @@ namespace FunctionCalling;
1011
/// <summary>
1112
/// These examples demonstrate two ways functions called by the Gemini LLM can be invoked using the SK streaming and non-streaming AI API:
1213
///
13-
/// 1. Automatic Invocation by SK:
14+
/// 1. Automatic Invocation by SK (with and without nullable properties):
1415
/// Functions called by the LLM are invoked automatically by SK. The results of these function invocations
1516
/// are automatically added to the chat history and returned to the LLM. The LLM reasons about the chat history
1617
/// and generates the final response.
@@ -86,6 +87,92 @@ public async Task VertexAIChatCompletionWithFunctionCalling()
8687
await this.RunSampleAsync(kernel);
8788
}
8889

90+
[RetryFact]
91+
public async Task GoogleAIFunctionCallingNullable()
92+
{
93+
Console.WriteLine("============= Google AI - Gemini Chat Completion with function calling (nullable properties) =============");
94+
95+
Assert.NotNull(TestConfiguration.GoogleAI.ApiKey);
96+
97+
var kernelBuilder = Kernel.CreateBuilder()
98+
.AddGoogleAIGeminiChatCompletion(
99+
modelId: TestConfiguration.VertexAI.Gemini.ModelId,
100+
apiKey: TestConfiguration.GoogleAI.ApiKey);
101+
102+
kernelBuilder.Plugins.AddFromType<MyWeatherPlugin>();
103+
104+
var promptExecutionSettings = new GeminiPromptExecutionSettings()
105+
{
106+
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(),
107+
};
108+
109+
var kernel = kernelBuilder.Build();
110+
111+
var response = await kernel.InvokePromptAsync("Hi, what's the weather in New York?", new(promptExecutionSettings));
112+
113+
Console.WriteLine(response.ToString());
114+
}
115+
116+
private sealed class MyWeatherPlugin
117+
{
118+
[KernelFunction]
119+
[Description("Get the weather for a given location.")]
120+
private string GetWeather(WeatherRequest request)
121+
{
122+
return $"The weather in {request?.Location} is sunny.";
123+
}
124+
}
125+
126+
[RetryFact]
127+
public async Task VertexAIFunctionCallingNullable()
128+
{
129+
Console.WriteLine("============= Vertex AI - Gemini Chat Completion with function calling (nullable properties) =============");
130+
131+
Assert.NotNull(TestConfiguration.VertexAI.BearerKey);
132+
Assert.NotNull(TestConfiguration.VertexAI.Location);
133+
Assert.NotNull(TestConfiguration.VertexAI.ProjectId);
134+
135+
var kernelBuilder = Kernel.CreateBuilder()
136+
.AddVertexAIGeminiChatCompletion(
137+
modelId: TestConfiguration.VertexAI.Gemini.ModelId,
138+
bearerKey: TestConfiguration.VertexAI.BearerKey,
139+
location: TestConfiguration.VertexAI.Location,
140+
projectId: TestConfiguration.VertexAI.ProjectId);
141+
142+
// To generate bearer key, you need installed google sdk or use Google web console with command:
143+
//
144+
// gcloud auth print-access-token
145+
//
146+
// Above code pass bearer key as string, it is not recommended way in production code,
147+
// especially if IChatCompletionService will be long-lived, tokens generated by google sdk lives for 1 hour.
148+
// You should use bearer key provider, which will be used to generate token on demand:
149+
//
150+
// Example:
151+
//
152+
// Kernel kernel = Kernel.CreateBuilder()
153+
// .AddVertexAIGeminiChatCompletion(
154+
// modelId: TestConfiguration.VertexAI.Gemini.ModelId,
155+
// bearerKeyProvider: () =>
156+
// {
157+
// // This is just example, in production we recommend using Google SDK to generate your BearerKey token.
158+
// // This delegate will be called on every request,
159+
// // when providing the token consider using caching strategy and refresh token logic when it is expired or close to expiration.
160+
// return GetBearerKey();
161+
// },
162+
// location: TestConfiguration.VertexAI.Location,
163+
// projectId: TestConfiguration.VertexAI.ProjectId);
164+
165+
kernelBuilder.Plugins.AddFromType<MyWeatherPlugin>();
166+
167+
var promptExecutionSettings = new GeminiPromptExecutionSettings()
168+
{
169+
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(),
170+
};
171+
var kernel = kernelBuilder.Build();
172+
var response = await kernel.InvokePromptAsync("Hi, what's the weather in New York?", new(promptExecutionSettings));
173+
Console.WriteLine(response.ToString());
174+
}
175+
89176
private async Task RunSampleAsync(Kernel kernel)
90177
{
91178
// Add a plugin with some helper functions we want to allow the model to utilize.
@@ -214,4 +301,9 @@ private async Task RunSampleAsync(Kernel kernel)
214301
}
215302
*/
216303
}
304+
305+
private sealed class WeatherRequest
306+
{
307+
public string? Location { get; set; }
308+
}
217309
}

‎dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiFunctionTests.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ public void ItCanConvertToFunctionDefinitionWithNullParameters()
6868
var result = sut.ToFunctionDeclaration();
6969

7070
// Assert
71-
Assert.Null(result.Parameters);
71+
Assert.NotNull(result.Parameters);
72+
Assert.Equal(JsonValueKind.Null, result.Parameters.Value.ValueKind);
7273
}
7374

7475
[Fact]

‎dotnet/src/Connectors/Connectors.Google.UnitTests/Extensions/KernelFunctionMetadataExtensionsTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ public void ItCanCreateValidGeminiFunctionManualForPrompt()
235235
// Assert
236236
Assert.NotNull(result);
237237
Assert.Equal(
238-
"""{"type":"object","required":["parameter1","parameter2"],"properties":{"parameter1":{"type":"string","description":"String parameter"},"parameter2":{"enum":["Value1","Value2"],"description":"Enum parameter"}}}""",
238+
"""{"type":"object","required":["parameter1","parameter2"],"properties":{"parameter1":{"type":"string","description":"String parameter"},"parameter2":{"enum":["Value1","Value2"],"description":"Enum parameter","type":"string"}}}""",
239239
JsonSerializer.Serialize(result.Parameters)
240240
);
241241
}

‎dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiRequest.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,7 @@ private static void AddConfiguration(GeminiPromptExecutionSettings executionSett
343343
/// - Replaces the type array with a single type value
344344
/// - Adds "nullable": true as a property
345345
/// </remarks>
346-
private static JsonElement TransformToOpenApi3Schema(JsonElement jsonElement)
346+
internal static JsonElement TransformToOpenApi3Schema(JsonElement jsonElement)
347347
{
348348
JsonNode? node = JsonNode.Parse(jsonElement.GetRawText());
349349
if (node is JsonObject rootObject)

‎dotnet/src/Connectors/Connectors.Google/Core/Gemini/Models/GeminiTool.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

33
using System.Collections.Generic;
4-
using System.Text.Json.Nodes;
4+
using System.Text.Json;
55
using System.Text.Json.Serialization;
66

77
namespace Microsoft.SemanticKernel.Connectors.Google.Core;
@@ -54,6 +54,6 @@ internal sealed class FunctionDeclaration
5454
/// </summary>
5555
[JsonPropertyName("parameters")]
5656
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
57-
public JsonNode? Parameters { get; set; }
57+
public JsonElement? Parameters { get; set; }
5858
}
5959
}

‎dotnet/src/Connectors/Connectors.Google/Models/Gemini/GeminiFunction.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ internal GeminiTool.FunctionDeclaration ToFunctionDeclaration()
162162
Name = this.FullyQualifiedName,
163163
Description = this.Description ?? throw new InvalidOperationException(
164164
$"Function description is required. Please provide a description for the function {this.FullyQualifiedName}."),
165-
Parameters = JsonSerializer.SerializeToNode(resultParameters),
165+
Parameters = GeminiRequest.TransformToOpenApi3Schema(JsonSerializer.SerializeToElement(resultParameters)),
166166
};
167167
}
168168

‎dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiFunctionCallingTests.cs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,28 @@ public async Task ChatStreamingAutoInvokeShouldCallFunctionsMultipleTimesAndRetu
238238
Assert.Contains("105", content, StringComparison.OrdinalIgnoreCase);
239239
}
240240

241+
[RetryTheory]
242+
[InlineData(ServiceType.GoogleAI, Skip = "This test is for manual verification.")]
243+
[InlineData(ServiceType.VertexAI, Skip = "This test is for manual verification.")]
244+
public async Task ChatGenerationAutoInvokeNullablePropertiesWorksAsync(ServiceType serviceType)
245+
{
246+
var kernel = new Kernel();
247+
kernel.ImportPluginFromType<NullableTestPlugin>();
248+
var sut = this.GetChatService(serviceType);
249+
250+
var executionSettings = new GeminiPromptExecutionSettings()
251+
{
252+
FunctionChoiceBehavior = FunctionChoiceBehavior.Auto(),
253+
};
254+
255+
ChatHistory chatHistory = [];
256+
chatHistory.AddUserMessage("Hi, what's the weather in New York?");
257+
258+
var response = await sut.GetChatMessageContentAsync(chatHistory, executionSettings);
259+
260+
Assert.NotNull(response);
261+
}
262+
241263
[RetryTheory]
242264
[InlineData(ServiceType.GoogleAI, Skip = "This test is for manual verification.")]
243265
[InlineData(ServiceType.VertexAI, Skip = "This test is for manual verification.")]
@@ -442,4 +464,21 @@ public int Sum([Description("Numbers to sum")] int[] numbers)
442464
return numbers.Sum();
443465
}
444466
}
467+
468+
#pragma warning disable CA1812 // Uninstantiated internal types
469+
private sealed class NullableTestPlugin
470+
{
471+
[KernelFunction]
472+
[Description("Get the weather for a given location.")]
473+
private string GetWeather(Request request)
474+
{
475+
return $"The weather in {request?.Location} is sunny.";
476+
}
477+
}
478+
479+
private sealed class Request
480+
{
481+
public string? Location { get; set; }
482+
}
483+
#pragma warning disable CA1812 // Uninstantiated internal types
445484
}

0 commit comments

Comments
 (0)