-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Description
I am currently running the Mistral-Small-24B-Instruct-2501 model on an internal server using vLLM, and it operates as expected. To integrate function calling capabilities, I've developed a test application in C#. Understanding that Mistral does not support automatic function calling, I've implemented a straightforward method to extract and execute the necessary function calls.
However, I've encountered an issue with the chat history structure generated by Semantic Kernel (SK). Specifically, the JSON data lacks the "type": "function" field that the Mistral model requires, leading to errors during execution. The expected structure, as outlined in the Mistral model documentation, includes this "type": "function" field.
In my attempts to resolve this, I've utilized the FunctionCallContent class in SK. Unfortunately, it doesn't seem to allow the addition of custom properties, such as "type": "function".
Here is a snippet of the code I've been working with:
var functionCallContent = new FunctionCallContent(
functionName: functionName,
pluginName: pluginName,
id: callId,
arguments: new KernelArguments(toolCall.Arguments.ToDictionary(kvp => kvp.Key, kvp => (object)kvp.Value))
);
// Attempt to add custom metadata
functionCallContent.Metadata = new ReadOnlyDictionary<string, object?>(new Dictionary<string, object?>
{
{ "type", "function" }
});
Despite these efforts, the "type": "function" property is not included in the serialized JSON sent to the Mistral model.
Has anyone faced a similar issue or can provide guidance on how to include the "type": "function" field in the JSON structure generated by Semantic Kernel? Any insights or suggestions would be greatly appreciated.
This is the json data sent to the llm server:
{
"model": "mistralai/Mistral-Small-24B-Instruct-2501",
"messages": [
{
"role": "system",
"content": "You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\r\n\t\tYour knowledge base was last updated on 2023-10-01. The current date is 2025-01-30.\r\n\t\tWhen you\u0027re not sure about some information, you say that you don\u0027t have the information and don\u0027t make up anything.\r\n\t\tIf the user\u0027s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request \r\n\t\t(e.g. \u0022What are some good restaurants around me?\u0022 =\u003E \u0022Where are you?\u0022 or \u0022When is the next flight to Tokyo\u0022 =\u003E \u0022Where do you travel from?\u0022). Answer in maximum 50 words. If you are asked a question, use functions as primary source"
},
{
"role": "user",
"content": "V\u00E6ret oslo norge?"
},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "chatcmpl-b814d48e7419490e980e2a4290fe8c3f",
"function": {
"name": "WeatherPlugin1-GetWeather",
"arguments": "{\u0022location\u0022:\u0022Oslo, 03\u0022}"
}
}
]
},
{
"role": "tool",
"content": "17\u00B0C\nWind: 23 KMPH\nHumidity: 59%\nMostly cloudy",
"name": "WeatherPlugin1-GetWeather",
"tool_call_id": "chatcmpl-b814d48e7419490e980e2a4290fe8c3f"
}
],
"temperature": 0.7,
"top_p": 1,
"stream": false,
"safe_prompt": false,
"tools": [
{
"type": "function",
"function": {
"name": "OrderPizza-get_pizza_menu",
"description": "",
"parameters": {
"type": "object",
"properties": {},
"required": []
}
}
},
{
"type": "function",
"function": {
"name": "WeatherPlugin1-GetWeather",
"description": "Get the current weather in a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"description": "The city and department, e.g. Marseille, 13",
"type": "string"
}
},
"required": [ "location" ]
}
}
}
],
"tool_choice": "auto"
}
This gives the following error from mistral:
Field required [type=missing, input_value={'id': 'chatcmpl-68b1c676... '{"location":"Oslo"}'}}, input_type=dict]
This is how the json should be:
{
"model": "mistralai/Mistral-Small-24B-Instruct-2501",
"messages": [
{
"role": "system",
"content": "You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\r\n\t\tYour knowledge base was last updated on 2023-10-01. The current date is 2025-01-30.\r\n\t\tWhen you\u0027re not sure about some information, you say that you don\u0027t have the information and don\u0027t make up anything.\r\n\t\tIf the user\u0027s question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request \r\n\t\t(e.g. \u0022What are some good restaurants around me?\u0022 =\u003E \u0022Where are you?\u0022 or \u0022When is the next flight to Tokyo\u0022 =\u003E \u0022Where do you travel from?\u0022). Answer in maximum 50 words. If you are asked a question, use functions as primary source"
},
{
"role": "user",
"content": "V\u00E6ret oslo norge?"
},
{
"role": "assistant",
"content": "",
"type": "function", <--- Should be added
"tool_calls": [
{
"id": "chatcmpl-b814d48e7419490e980e2a4290fe8c3f",
"function": {
"name": "WeatherPlugin1-GetWeather",
"arguments": "{\u0022location\u0022:\u0022Oslo, 03\u0022}"
}
}
]
},
{
"role": "tool",
"content": "17\u00B0C\nWind: 23 KMPH\nHumidity: 59%\nMostly cloudy",
"name": "WeatherPlugin1-GetWeather",
"tool_call_id": "chatcmpl-b814d48e7419490e980e2a4290fe8c3f"
}
],
"temperature": 0.7,
"top_p": 1,
"stream": false,
"safe_prompt": false,
"tools": [
{
"type": "function",
"function": {
"name": "OrderPizza-get_pizza_menu",
"description": "",
"parameters": {
"type": "object",
"properties": {},
"required": []
}
}
},
{
"type": "function",
"function": {
"name": "WeatherPlugin1-GetWeather",
"description": "Get the current weather in a given location.",
"parameters": {
"type": "object",
"properties": {
"location": {
"description": "The city and department, e.g. Marseille, 13",
"type": "string"
}
},
"required": [ "location" ]
}
}
}
],
"tool_choice": "auto"
}
** Nuget versions **
<PackageReference Include="Microsoft.SemanticKernel" Version="1.41.0" />
<PackageReference Include="Microsoft.SemanticKernel.Connectors.MistralAI" Version="1.41.0" />
** Complete example **
#pragma warning disable SKEXP0070
#pragma warning disable SKEXP0010
#pragma warning disable SKEXP0060
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using System.Collections.ObjectModel;
using SemanticKernelStripped;
using Microsoft.SemanticKernel.Connectors.MistralAI;
using System.ComponentModel;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
class Program
{
static async Task Main()
{
var kernelBuilder = Kernel.CreateBuilder();
string modelId = "mistralai/Mistral-Small-24B-Instruct-2501";
Uri endpointUri = new Uri("http://xx.com:4040/v1");
kernelBuilder.AddMistralChatCompletion(modelId, "fake", endpointUri);
var settings = new MistralAIPromptExecutionSettings { ToolCallBehavior = MistralAIToolCallBehavior.EnableKernelFunctions };
kernelBuilder.Plugins.AddFromType<OrderPizzaPlugin>("OrderPizza");
WeatherPlugin1 weatherPlugin = new WeatherPlugin1();
kernelBuilder.Plugins.AddFromObject(weatherPlugin);
var kernel = kernelBuilder.Build();
var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
string systemPrompt = @"You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.
Your knowledge base was last updated on 2023-10-01. The current date is 2025-01-30.
When you're not sure about some information, you say that you don't have the information and don't make up anything.
If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request
(e.g. ""What are some good restaurants around me?"" => ""Where are you?"" or ""When is the next flight to Tokyo"" => ""Where do you travel from?""). Answer in maximum 50 words. If you are asked a question, use functions as primary source";
var chatHistory = new ChatHistory();
chatHistory.AddSystemMessage(systemPrompt);
while (true)
{
Console.WriteLine("Your question:");
var question = Console.ReadLine();
chatHistory.AddUserMessage(question);
var chatMessageContentsAsync = await chatCompletionService.GetChatMessageContentsAsync(chatHistory, settings, kernel);
var result = chatMessageContentsAsync.First();
if (result.Content.Contains("[TOOL_CALLS]"))
{
var resultContent = result.Content ?? "";
var callId = result.Metadata.FirstOrDefault(x => x.Key == "Id").Value.ToString();
var match = Regex.Match(resultContent, @"\[TOOL_CALLS\](.*)");
string json = match.Success ? match.Groups[1].Value.Trim() : "[]"; // JSON-delen
string assistantToolCallMessageContent = Regex.Replace(resultContent, @"\[TOOL_CALLS\].*", "").Trim(); // Fjerner TOOL_CALLS fra teksten
try
{
var toolCalls = JsonSerializer.Deserialize<List<ToolCall>>(json);
foreach (var toolCall in toolCalls)
{
var splitName = toolCall.Name.Split('-');
if (splitName.Length != 2) continue;
string pluginName = splitName[0]; // "WeatherPlugin1"
string functionName = splitName[1]; // "GetWeather"
var functionResult = await kernel.InvokeAsync(
pluginName, functionName,
new KernelArguments { ["location"] = toolCall.Arguments["location"] });
var functionValue = functionResult.GetValue<string>() ?? "";
var functionCallContent = new FunctionCallContent(
functionName: functionName,
pluginName: pluginName,
id: callId,
arguments: new KernelArguments
{
["location"] = toolCall.Arguments["location"]
}
);
functionCallContent.Metadata = new ReadOnlyDictionary<string, object?>(new Dictionary<string, object?>
{
{ "type", "function" }
});
chatHistory.Add(new()
{
Role = AuthorRole.Assistant,
Items = [functionCallContent]
});
//Må nå bygge opp Tool-message:
var toolResponse = new FunctionResultContent(
functionName: functionName, // Navnet på funksjonen
pluginName: pluginName, // Navnet på pluginen
result: functionValue, // Selve returverdien fra funksjonen
callId: callId
);
chatHistory.Add(new ChatMessageContent
{
Role = AuthorRole.Tool,
Items = [toolResponse],
});
}
chatMessageContentsAsync = await chatCompletionService.GetChatMessageContentsAsync(chatHistory, settings, kernel);
var content = chatMessageContentsAsync.First().Content;
chatHistory.Add(new ChatMessageContent
{
Role = AuthorRole.Assistant,
Content = content
});
Console.WriteLine(content);
}
catch (Exception ex)
{
Console.WriteLine($"Error parsing TOOL_CALLS: {ex.Message}");
}
}
else
{
chatHistory.AddAssistantMessage(result.Content ?? "");
Console.WriteLine(result);
}
}
}
}
public class ToolCall
{
[JsonPropertyName("name")]
public string Name { get; set; }
[JsonPropertyName("arguments")]
public Dictionary<string, string> Arguments { get; set; }
}
public sealed class WeatherPlugin1
{
[KernelFunction]
[Description("Get the current weather in a given location.")]
public string GetWeather(
[Description("The city and department, e.g. Marseille, 13")] string location
) => "17°C\nWind: 23 KMPH\nHumidity: 59%\nMostly cloudy";
}
Metadata
Metadata
Assignees
Labels
Type
Projects
Status