Skip to content

Commit dab22b1

Browse files
daniel-lxsdaniel-lxs
committed
feat: implement streaming for native tool calls
This adds streaming support for native tool calls (OpenAI-style function calling) in both the Roo Code Cloud (roo.ts) and OpenRouter providers. Changes: - Add tool_call_start, tool_call_delta, and tool_call_end events to ApiStreamChunk - Implement streaming tool call tracking in roo.ts and openrouter.ts providers - Add NativeToolCallParser.processStreamingChunk() for incremental JSON parsing - Add NativeToolCallParser.startStreamingToolCall() and finalizeStreamingToolCall() - Use partial-json-parser to extract partial values from incomplete JSON - Handle streaming tool calls in Task.ts by calling tool.handlePartial() - Update BaseTool to handle partial native tool calls The streaming implementation allows the UI to show tool parameters as they stream in, providing the same experience as XML tools during LLM streaming.
1 parent 3ac5bec commit dab22b1

File tree

8 files changed

+749
-101
lines changed

8 files changed

+749
-101
lines changed

‎src/api/providers/__tests__/roo.spec.ts‎

Lines changed: 288 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -636,7 +636,7 @@ describe("RooHandler", () => {
636636
handler = new RooHandler(mockOptions)
637637
})
638638

639-
it("should yield tool calls when finish_reason is tool_calls", async () => {
639+
it("should yield streaming tool call chunks when finish_reason is tool_calls", async () => {
640640
mockCreate.mockResolvedValueOnce({
641641
[Symbol.asyncIterator]: async function* () {
642642
yield {
@@ -689,14 +689,24 @@ describe("RooHandler", () => {
689689
chunks.push(chunk)
690690
}
691691

692-
const toolCallChunks = chunks.filter((chunk) => chunk.type === "tool_call")
693-
expect(toolCallChunks).toHaveLength(1)
694-
expect(toolCallChunks[0].id).toBe("call_123")
695-
expect(toolCallChunks[0].name).toBe("read_file")
696-
expect(toolCallChunks[0].arguments).toBe('{"path":"test.ts"}')
692+
// Verify we get streaming chunks
693+
const startChunks = chunks.filter((chunk) => chunk.type === "tool_call_start")
694+
const deltaChunks = chunks.filter((chunk) => chunk.type === "tool_call_delta")
695+
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
696+
697+
expect(startChunks).toHaveLength(1)
698+
expect(startChunks[0].id).toBe("call_123")
699+
expect(startChunks[0].name).toBe("read_file")
700+
701+
expect(deltaChunks).toHaveLength(2)
702+
expect(deltaChunks[0].delta).toBe('{"path":"')
703+
expect(deltaChunks[1].delta).toBe('test.ts"}')
704+
705+
expect(endChunks).toHaveLength(1)
706+
expect(endChunks[0].id).toBe("call_123")
697707
})
698708

699-
it("should yield tool calls even when finish_reason is not set (fallback behavior)", async () => {
709+
it("should yield streaming tool calls even when finish_reason is not set (fallback behavior)", async () => {
700710
mockCreate.mockResolvedValueOnce({
701711
[Symbol.asyncIterator]: async function* () {
702712
yield {
@@ -738,15 +748,23 @@ describe("RooHandler", () => {
738748
chunks.push(chunk)
739749
}
740750

741-
// Tool calls should still be yielded via the fallback mechanism
742-
const toolCallChunks = chunks.filter((chunk) => chunk.type === "tool_call")
743-
expect(toolCallChunks).toHaveLength(1)
744-
expect(toolCallChunks[0].id).toBe("call_456")
745-
expect(toolCallChunks[0].name).toBe("write_to_file")
746-
expect(toolCallChunks[0].arguments).toBe('{"path":"test.ts","content":"hello"}')
751+
// Tool calls should still be yielded via the fallback mechanism as streaming chunks
752+
const startChunks = chunks.filter((chunk) => chunk.type === "tool_call_start")
753+
const deltaChunks = chunks.filter((chunk) => chunk.type === "tool_call_delta")
754+
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
755+
756+
expect(startChunks).toHaveLength(1)
757+
expect(startChunks[0].id).toBe("call_456")
758+
expect(startChunks[0].name).toBe("write_to_file")
759+
760+
expect(deltaChunks).toHaveLength(1)
761+
expect(deltaChunks[0].delta).toBe('{"path":"test.ts","content":"hello"}')
762+
763+
expect(endChunks).toHaveLength(1)
764+
expect(endChunks[0].id).toBe("call_456")
747765
})
748766

749-
it("should handle multiple tool calls", async () => {
767+
it("should handle multiple streaming tool calls", async () => {
750768
mockCreate.mockResolvedValueOnce({
751769
[Symbol.asyncIterator]: async function* () {
752770
yield {
@@ -800,15 +818,21 @@ describe("RooHandler", () => {
800818
chunks.push(chunk)
801819
}
802820

803-
const toolCallChunks = chunks.filter((chunk) => chunk.type === "tool_call")
804-
expect(toolCallChunks).toHaveLength(2)
805-
expect(toolCallChunks[0].id).toBe("call_1")
806-
expect(toolCallChunks[0].name).toBe("read_file")
807-
expect(toolCallChunks[1].id).toBe("call_2")
808-
expect(toolCallChunks[1].name).toBe("read_file")
821+
const startChunks = chunks.filter((chunk) => chunk.type === "tool_call_start")
822+
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
823+
824+
expect(startChunks).toHaveLength(2)
825+
expect(startChunks[0].id).toBe("call_1")
826+
expect(startChunks[0].name).toBe("read_file")
827+
expect(startChunks[1].id).toBe("call_2")
828+
expect(startChunks[1].name).toBe("read_file")
829+
830+
expect(endChunks).toHaveLength(2)
831+
expect(endChunks[0].id).toBe("call_1")
832+
expect(endChunks[1].id).toBe("call_2")
809833
})
810834

811-
it("should accumulate tool call arguments across multiple chunks", async () => {
835+
it("should accumulate tool call arguments across multiple streaming chunks", async () => {
812836
mockCreate.mockResolvedValueOnce({
813837
[Symbol.asyncIterator]: async function* () {
814838
yield {
@@ -876,11 +900,21 @@ describe("RooHandler", () => {
876900
chunks.push(chunk)
877901
}
878902

879-
const toolCallChunks = chunks.filter((chunk) => chunk.type === "tool_call")
880-
expect(toolCallChunks).toHaveLength(1)
881-
expect(toolCallChunks[0].id).toBe("call_789")
882-
expect(toolCallChunks[0].name).toBe("execute_command")
883-
expect(toolCallChunks[0].arguments).toBe('{"command":"npm install"}')
903+
const startChunks = chunks.filter((chunk) => chunk.type === "tool_call_start")
904+
const deltaChunks = chunks.filter((chunk) => chunk.type === "tool_call_delta")
905+
const endChunks = chunks.filter((chunk) => chunk.type === "tool_call_end")
906+
907+
expect(startChunks).toHaveLength(1)
908+
expect(startChunks[0].id).toBe("call_789")
909+
expect(startChunks[0].name).toBe("execute_command")
910+
911+
expect(deltaChunks).toHaveLength(3)
912+
expect(deltaChunks[0].delta).toBe('{"command":"')
913+
expect(deltaChunks[1].delta).toBe("npm install")
914+
expect(deltaChunks[2].delta).toBe('"}')
915+
916+
expect(endChunks).toHaveLength(1)
917+
expect(endChunks[0].id).toBe("call_789")
884918
})
885919

886920
it("should not yield empty tool calls when no tool calls present", async () => {
@@ -906,4 +940,232 @@ describe("RooHandler", () => {
906940
expect(toolCallChunks).toHaveLength(0)
907941
})
908942
})
943+
944+
describe("streaming tool calls", () => {
945+
beforeEach(() => {
946+
handler = new RooHandler(mockOptions)
947+
})
948+
949+
it("should emit tool_call_start, tool_call_delta, and tool_call_end chunks", async () => {
950+
mockCreate.mockResolvedValueOnce({
951+
[Symbol.asyncIterator]: async function* () {
952+
// First chunk: tool call starts with ID and name
953+
yield {
954+
choices: [
955+
{
956+
delta: {
957+
tool_calls: [
958+
{
959+
index: 0,
960+
id: "call_streaming_123",
961+
function: { name: "read_file", arguments: "" },
962+
},
963+
],
964+
},
965+
index: 0,
966+
},
967+
],
968+
}
969+
// Second chunk: first part of arguments
970+
yield {
971+
choices: [
972+
{
973+
delta: {
974+
tool_calls: [
975+
{
976+
index: 0,
977+
function: { arguments: '{"files":[{"p' },
978+
},
979+
],
980+
},
981+
index: 0,
982+
},
983+
],
984+
}
985+
// Third chunk: more arguments
986+
yield {
987+
choices: [
988+
{
989+
delta: {
990+
tool_calls: [
991+
{
992+
index: 0,
993+
function: { arguments: 'ath":"test.ts"}]}' },
994+
},
995+
],
996+
},
997+
index: 0,
998+
},
999+
],
1000+
}
1001+
// Final chunk: finish
1002+
yield {
1003+
choices: [
1004+
{
1005+
delta: {},
1006+
finish_reason: "tool_calls",
1007+
index: 0,
1008+
},
1009+
],
1010+
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
1011+
}
1012+
},
1013+
})
1014+
1015+
const stream = handler.createMessage(systemPrompt, messages)
1016+
const chunks: any[] = []
1017+
for await (const chunk of stream) {
1018+
chunks.push(chunk)
1019+
}
1020+
1021+
// Verify we get start, delta, and end chunks
1022+
const startChunks = chunks.filter((c) => c.type === "tool_call_start")
1023+
const deltaChunks = chunks.filter((c) => c.type === "tool_call_delta")
1024+
const endChunks = chunks.filter((c) => c.type === "tool_call_end")
1025+
1026+
expect(startChunks).toHaveLength(1)
1027+
expect(startChunks[0]).toEqual({
1028+
type: "tool_call_start",
1029+
id: "call_streaming_123",
1030+
name: "read_file",
1031+
})
1032+
1033+
expect(deltaChunks).toHaveLength(2)
1034+
expect(deltaChunks[0]).toEqual({
1035+
type: "tool_call_delta",
1036+
id: "call_streaming_123",
1037+
delta: '{"files":[{"p',
1038+
})
1039+
expect(deltaChunks[1]).toEqual({
1040+
type: "tool_call_delta",
1041+
id: "call_streaming_123",
1042+
delta: 'ath":"test.ts"}]}',
1043+
})
1044+
1045+
expect(endChunks).toHaveLength(1)
1046+
expect(endChunks[0]).toEqual({
1047+
type: "tool_call_end",
1048+
id: "call_streaming_123",
1049+
})
1050+
})
1051+
1052+
it("should handle multiple streaming tool calls", async () => {
1053+
mockCreate.mockResolvedValueOnce({
1054+
[Symbol.asyncIterator]: async function* () {
1055+
// First tool call starts
1056+
yield {
1057+
choices: [
1058+
{
1059+
delta: {
1060+
tool_calls: [
1061+
{
1062+
index: 0,
1063+
id: "call_1",
1064+
function: {
1065+
name: "read_file",
1066+
arguments: '{"files":[{"path":"file1.ts"}]}',
1067+
},
1068+
},
1069+
],
1070+
},
1071+
index: 0,
1072+
},
1073+
],
1074+
}
1075+
// Second tool call starts
1076+
yield {
1077+
choices: [
1078+
{
1079+
delta: {
1080+
tool_calls: [
1081+
{
1082+
index: 1,
1083+
id: "call_2",
1084+
function: { name: "list_files", arguments: '{"path":"src"}' },
1085+
},
1086+
],
1087+
},
1088+
index: 0,
1089+
},
1090+
],
1091+
}
1092+
// Finish
1093+
yield {
1094+
choices: [
1095+
{
1096+
delta: {},
1097+
finish_reason: "tool_calls",
1098+
index: 0,
1099+
},
1100+
],
1101+
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
1102+
}
1103+
},
1104+
})
1105+
1106+
const stream = handler.createMessage(systemPrompt, messages)
1107+
const chunks: any[] = []
1108+
for await (const chunk of stream) {
1109+
chunks.push(chunk)
1110+
}
1111+
1112+
const startChunks = chunks.filter((c) => c.type === "tool_call_start")
1113+
const endChunks = chunks.filter((c) => c.type === "tool_call_end")
1114+
1115+
expect(startChunks).toHaveLength(2)
1116+
expect(startChunks[0].id).toBe("call_1")
1117+
expect(startChunks[0].name).toBe("read_file")
1118+
expect(startChunks[1].id).toBe("call_2")
1119+
expect(startChunks[1].name).toBe("list_files")
1120+
1121+
expect(endChunks).toHaveLength(2)
1122+
expect(endChunks[0].id).toBe("call_1")
1123+
expect(endChunks[1].id).toBe("call_2")
1124+
})
1125+
1126+
it("should emit end chunks even when finish_reason is not tool_calls (fallback)", async () => {
1127+
mockCreate.mockResolvedValueOnce({
1128+
[Symbol.asyncIterator]: async function* () {
1129+
yield {
1130+
choices: [
1131+
{
1132+
delta: {
1133+
tool_calls: [
1134+
{
1135+
index: 0,
1136+
id: "call_fallback",
1137+
function: {
1138+
name: "read_file",
1139+
arguments: '{"files":[{"path":"test.ts"}]}',
1140+
},
1141+
},
1142+
],
1143+
},
1144+
index: 0,
1145+
},
1146+
],
1147+
}
1148+
// Stream ends with different finish_reason
1149+
yield {
1150+
choices: [{ delta: {}, finish_reason: "stop", index: 0 }],
1151+
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
1152+
}
1153+
},
1154+
})
1155+
1156+
const stream = handler.createMessage(systemPrompt, messages)
1157+
const chunks: any[] = []
1158+
for await (const chunk of stream) {
1159+
chunks.push(chunk)
1160+
}
1161+
1162+
const startChunks = chunks.filter((c) => c.type === "tool_call_start")
1163+
const endChunks = chunks.filter((c) => c.type === "tool_call_end")
1164+
1165+
// Should still emit start/end chunks via fallback
1166+
expect(startChunks).toHaveLength(1)
1167+
expect(endChunks).toHaveLength(1)
1168+
expect(endChunks[0].id).toBe("call_fallback")
1169+
})
1170+
})
9091171
})

0 commit comments

Comments
 (0)