Skip to content

Commit d13d389

Browse files
welcoMatticwelcoMattic
committed
Add tool calling support and examples for Docker Model Runner platform
1 parent 45675c5 commit d13d389

File tree

8 files changed

+224
-71
lines changed

8 files changed

+224
-71
lines changed

‎examples/dockermodelrunner/embeddings.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@
2121
country was very peaceful and prosperous. The people lived happily ever after.
2222
TEXT);
2323

24-
echo 'Dimensions: '.$response->asVectors()[0]->getDimensions().\PHP_EOL;
24+
print_vectors($response);

‎examples/dockermodelrunner/toolcall.php‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
use Symfony\AI\Agent\Agent;
13+
use Symfony\AI\Agent\Toolbox\AgentProcessor;
14+
use Symfony\AI\Agent\Toolbox\Tool\Wikipedia;
15+
use Symfony\AI\Agent\Toolbox\Toolbox;
16+
use Symfony\AI\Platform\Bridge\DockerModelRunner\Completions;
17+
use Symfony\AI\Platform\Bridge\DockerModelRunner\PlatformFactory;
18+
use Symfony\AI\Platform\Message\Message;
19+
use Symfony\AI\Platform\Message\MessageBag;
20+
21+
require_once dirname(__DIR__).'/bootstrap.php';
22+
23+
$platform = PlatformFactory::create(env('DOCKER_MODEL_RUNNER_HOST_URL'), http_client());
24+
$model = new Completions(Completions::GEMMA_3_N);
25+
26+
$wikipedia = new Wikipedia(http_client());
27+
$toolbox = new Toolbox([$wikipedia]);
28+
$processor = new AgentProcessor($toolbox);
29+
$agent = new Agent($platform, $model, [$processor], [$processor], logger: logger());
30+
31+
$messages = new MessageBag(Message::ofUser('Who is the actual Prime Minister of France?'));
32+
$result = $agent->call($messages);
33+
34+
echo $result->getContent().\PHP_EOL;

‎src/platform/src/Bridge/DockerModelRunner/Completions.php‎

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -36,37 +36,12 @@ class Completions extends Model
3636
public const SMOLLM_2 = 'ai/smollm2';
3737
public const SMOLLM_3 = 'ai/smollm3';
3838

39-
private const TOOL_PATTERNS = [
40-
'/./' => [
41-
Capability::INPUT_MESSAGES,
42-
Capability::OUTPUT_TEXT,
43-
Capability::OUTPUT_STRUCTURED,
44-
],
45-
'/^llama\D*3(\D*\d+)/' => [
46-
Capability::TOOL_CALLING,
47-
],
48-
'/^qwen\d(\.\d)?(-coder)?$/' => [
49-
Capability::TOOL_CALLING,
50-
],
51-
'/^(deepseek|mistral|smollm|seed)/' => [
52-
Capability::TOOL_CALLING,
53-
],
54-
];
55-
5639
public function __construct(
5740
string $name = self::SMOLLM_2,
5841
array $options = [],
5942
) {
60-
$capabilities = [];
61-
62-
foreach (self::TOOL_PATTERNS as $pattern => $possibleCapabilities) {
63-
if (1 === preg_match($pattern, $name)) {
64-
foreach ($possibleCapabilities as $capability) {
65-
$capabilities[] = $capability;
66-
}
67-
}
68-
}
69-
70-
parent::__construct($name, $capabilities, $options);
43+
// All capabilities are assumed to be supported since we cannot know in advance
44+
// whether Docker Model Runner and/or each model allows for a particular capability.
45+
parent::__construct($name, Capability::cases(), $options);
7146
}
7247
}

‎src/platform/src/Bridge/DockerModelRunner/Completions/ResultConverter.php‎

Lines changed: 161 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,29 +12,182 @@
1212
namespace Symfony\AI\Platform\Bridge\DockerModelRunner\Completions;
1313

1414
use Symfony\AI\Platform\Bridge\DockerModelRunner\Completions;
15-
use Symfony\AI\Platform\Bridge\OpenAi\Gpt\ResultConverter as OpenAiResponseConverter;
15+
use Symfony\AI\Platform\Exception\ContentFilterException;
16+
use Symfony\AI\Platform\Exception\ExceedContextSizeException;
17+
use Symfony\AI\Platform\Exception\RuntimeException;
1618
use Symfony\AI\Platform\Model;
19+
use Symfony\AI\Platform\Result\ChoiceResult;
20+
use Symfony\AI\Platform\Result\RawHttpResult;
1721
use Symfony\AI\Platform\Result\RawResultInterface;
1822
use Symfony\AI\Platform\Result\ResultInterface;
23+
use Symfony\AI\Platform\Result\StreamResult;
24+
use Symfony\AI\Platform\Result\TextResult;
25+
use Symfony\AI\Platform\Result\ToolCall;
26+
use Symfony\AI\Platform\Result\ToolCallResult;
1927
use Symfony\AI\Platform\ResultConverterInterface;
28+
use Symfony\Component\HttpClient\Chunk\ServerSentEvent;
29+
use Symfony\Component\HttpClient\EventSourceHttpClient;
30+
use Symfony\Component\HttpClient\Exception\JsonException;
31+
use Symfony\Contracts\HttpClient\ResponseInterface as HttpResponse;
2032

2133
/**
2234
* @author Mathieu Santostefano <msantostefano@proton.me>
2335
*/
2436
final class ResultConverter implements ResultConverterInterface
2537
{
26-
public function __construct(
27-
private readonly OpenAiResponseConverter $gptResponseConverter = new OpenAiResponseConverter(),
28-
) {
29-
}
30-
3138
public function supports(Model $model): bool
3239
{
3340
return $model instanceof Completions;
3441
}
3542

36-
public function convert(RawResultInterface $result, array $options = []): ResultInterface
43+
public function convert(RawResultInterface|RawHttpResult $result, array $options = []): ResultInterface
44+
{
45+
if ($options['stream'] ?? false) {
46+
return new StreamResult($this->convertStream($result->getObject()));
47+
}
48+
49+
$data = $result->getData();
50+
51+
if (isset($data['error']['type']) && 'exceed_context_size_error' === $data['error']['type']) {
52+
throw new ExceedContextSizeException($data['error']['message']);
53+
}
54+
55+
if (isset($data['error']['code']) && 'content_filter' === $data['error']['code']) {
56+
throw new ContentFilterException($data['error']['message']);
57+
}
58+
59+
if (!isset($data['choices'])) {
60+
throw new RuntimeException('Response does not contain choices.');
61+
}
62+
63+
$choices = array_map($this->convertChoice(...), $data['choices']);
64+
65+
return 1 === \count($choices) ? $choices[0] : new ChoiceResult(...$choices);
66+
}
67+
68+
private function convertStream(HttpResponse $result): \Generator
69+
{
70+
$toolCalls = [];
71+
foreach ((new EventSourceHttpClient())->stream($result) as $chunk) {
72+
if (!$chunk instanceof ServerSentEvent || '[DONE]' === $chunk->getData()) {
73+
continue;
74+
}
75+
76+
try {
77+
$data = $chunk->getArrayData();
78+
} catch (JsonException) {
79+
// try catch only needed for Symfony 6.4
80+
continue;
81+
}
82+
83+
if ($this->streamIsToolCall($data)) {
84+
$toolCalls = $this->convertStreamToToolCalls($toolCalls, $data);
85+
}
86+
87+
if ([] !== $toolCalls && $this->isToolCallsStreamFinished($data)) {
88+
yield new ToolCallResult(...array_map($this->convertToolCall(...), $toolCalls));
89+
}
90+
91+
if (!isset($data['choices'][0]['delta']['content'])) {
92+
continue;
93+
}
94+
95+
yield $data['choices'][0]['delta']['content'];
96+
}
97+
}
98+
99+
/**
100+
* @param array<string, mixed> $toolCalls
101+
* @param array<string, mixed> $data
102+
*
103+
* @return array<string, mixed>
104+
*/
105+
private function convertStreamToToolCalls(array $toolCalls, array $data): array
37106
{
38-
return $this->gptResponseConverter->convert($result, $options);
107+
if (!isset($data['choices'][0]['delta']['tool_calls'])) {
108+
return $toolCalls;
109+
}
110+
111+
foreach ($data['choices'][0]['delta']['tool_calls'] as $i => $toolCall) {
112+
if (isset($toolCall['id'])) {
113+
// initialize tool call
114+
$toolCalls[$i] = [
115+
'id' => $toolCall['id'],
116+
'function' => $toolCall['function'],
117+
];
118+
continue;
119+
}
120+
121+
// add arguments delta to tool call
122+
$toolCalls[$i]['function']['arguments'] .= $toolCall['function']['arguments'];
123+
}
124+
125+
return $toolCalls;
126+
}
127+
128+
/**
129+
* @param array<string, mixed> $data
130+
*/
131+
private function streamIsToolCall(array $data): bool
132+
{
133+
return isset($data['choices'][0]['delta']['tool_calls']);
134+
}
135+
136+
/**
137+
* @param array<string, mixed> $data
138+
*/
139+
private function isToolCallsStreamFinished(array $data): bool
140+
{
141+
return isset($data['choices'][0]['finish_reason']) && 'tool_calls' === $data['choices'][0]['finish_reason'];
142+
}
143+
144+
/**
145+
* @param array{
146+
* index: int,
147+
* message: array{
148+
* role: 'assistant',
149+
* content: ?string,
150+
* tool_calls: array{
151+
* id: string,
152+
* type: 'function',
153+
* function: array{
154+
* name: string,
155+
* arguments: string
156+
* },
157+
* },
158+
* refusal: ?mixed
159+
* },
160+
* logprobs: string,
161+
* finish_reason: 'stop'|'length'|'tool_calls'|'content_filter',
162+
* } $choice
163+
*/
164+
private function convertChoice(array $choice): ToolCallResult|TextResult
165+
{
166+
if ('tool_calls' === $choice['finish_reason']) {
167+
return new ToolCallResult(...array_map([$this, 'convertToolCall'], $choice['message']['tool_calls']));
168+
}
169+
170+
if (\in_array($choice['finish_reason'], ['stop', 'length'], true)) {
171+
return new TextResult($choice['message']['content']);
172+
}
173+
174+
throw new RuntimeException(\sprintf('Unsupported finish reason "%s".', $choice['finish_reason']));
175+
}
176+
177+
/**
178+
* @param array{
179+
* id: string,
180+
* type: 'function',
181+
* function: array{
182+
* name: string,
183+
* arguments: string
184+
* }
185+
* } $toolCall
186+
*/
187+
private function convertToolCall(array $toolCall): ToolCall
188+
{
189+
$arguments = json_decode($toolCall['function']['arguments'], true, \JSON_THROW_ON_ERROR);
190+
191+
return new ToolCall($toolCall['id'], $toolCall['function']['name'], $arguments);
39192
}
40193
}

‎src/platform/src/Bridge/DockerModelRunner/Embeddings.php‎

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -24,31 +24,12 @@ class Embeddings extends Model
2424
public const EMBEDDING_GEMMA = 'ai/embeddinggemma';
2525
public const GRANITE_EMBEDDING_MULTI = 'ai/granite-embedding-multilingual';
2626

27-
private const TOOL_PATTERNS = [
28-
'/./' => [
29-
Capability::INPUT_MESSAGES,
30-
Capability::OUTPUT_TEXT,
31-
Capability::OUTPUT_STRUCTURED,
32-
],
33-
'/^(nomic).*/' => [
34-
Capability::INPUT_MULTIPLE,
35-
],
36-
];
37-
3827
public function __construct(
3928
string $name = self::NOMIC_EMBED_TEXT,
4029
array $options = [],
4130
) {
42-
$capabilities = [];
43-
44-
foreach (self::TOOL_PATTERNS as $pattern => $possibleCapabilities) {
45-
if (1 === preg_match($pattern, $name)) {
46-
foreach ($possibleCapabilities as $capability) {
47-
$capabilities[] = $capability;
48-
}
49-
}
50-
}
51-
52-
parent::__construct($name, $capabilities, $options);
31+
// All capabilities are assumed to be supported since we cannot know in advance
32+
// whether Docker Model Runner and/or each model allows for a particular capability.
33+
parent::__construct($name, Capability::cases(), $options);
5334
}
5435
}

‎src/platform/src/Exception/ExceedContextSizeException.php‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\AI\Platform\Exception;
13+
14+
/**
15+
* @author Mathieu Santostefano <msantostefano@proton.me>
16+
*/
17+
class ExceedContextSizeException extends InvalidArgumentException
18+
{
19+
}

‎src/platform/tests/Bridge/DockerModelRunner/Completions/ModelClientTest.php‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,7 @@ public function testItIsExecutingTheCorrectRequest()
3939
$resultCallback = static function (string $method, string $url, array $options): MockResponse {
4040
self::assertSame('POST', $method);
4141
self::assertSame('http://localhost:1234/engines/v1/chat/completions', $url);
42-
self::assertSame(
43-
'{"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}',
44-
$options['body']
45-
);
42+
self::assertSame('{"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}', $options['body']);
4643

4744
return new MockResponse();
4845
};
@@ -65,10 +62,7 @@ public function testItMergesOptionsWithPayload()
6562
$resultCallback = static function (string $method, string $url, array $options): MockResponse {
6663
self::assertSame('POST', $method);
6764
self::assertSame('http://localhost:1234/engines/v1/chat/completions', $url);
68-
self::assertSame(
69-
'{"temperature":0.7,"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}',
70-
$options['body']
71-
);
65+
self::assertSame('{"temperature":0.7,"model":"test-model","messages":[{"role":"user","content":"Hello, world!"}]}', $options['body']);
7266

7367
return new MockResponse();
7468
};

‎src/platform/tests/Bridge/DockerModelRunner/Embeddings/ModelClientTest.php‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,7 @@ public function testItMergesOptionsWithPayload()
5555
$resultCallback = static function (string $method, string $url, array $options): MockResponse {
5656
self::assertSame('POST', $method);
5757
self::assertSame('http://localhost:1234/engines/v1/embeddings', $url);
58-
self::assertSame(
59-
'{"custom_option":"value","model":"test-model","input":"Hello, world!"}',
60-
$options['body']
61-
);
58+
self::assertSame('{"custom_option":"value","model":"test-model","input":"Hello, world!"}', $options['body']);
6259

6360
return new MockResponse();
6461
};

0 commit comments

Comments
 (0)