diff --git a/bun.lock b/bun.lock index 5e50ce311526..0cfbfb2766ab 100644 --- a/bun.lock +++ b/bun.lock @@ -319,6 +319,7 @@ "hono": "catalog:", "hono-openapi": "catalog:", "ignore": "7.0.5", + "jimp": "1.6.0", "jsonc-parser": "3.3.1", "minimatch": "10.0.3", "open": "10.1.2", @@ -3246,7 +3247,7 @@ "pagefind": ["pagefind@1.4.0", "", { "optionalDependencies": { "@pagefind/darwin-arm64": "1.4.0", "@pagefind/darwin-x64": "1.4.0", "@pagefind/freebsd-x64": "1.4.0", "@pagefind/linux-arm64": "1.4.0", "@pagefind/linux-x64": "1.4.0", "@pagefind/windows-x64": "1.4.0" }, "bin": { "pagefind": "lib/runner/bin.cjs" } }, "sha512-z2kY1mQlL4J8q5EIsQkLzQjilovKzfNVhX8De6oyE6uHpfFtyBaqUpcl/XzJC/4fjD8vBDyh1zolimIcVrCn9g=="], - "pako": ["pako@0.2.9", "", {}, "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA=="], + "pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], "param-case": ["param-case@3.0.4", "", { "dependencies": { "dot-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A=="], @@ -4496,9 +4497,9 @@ "type-is/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "unifont/ofetch": ["ofetch@1.5.1", "", { "dependencies": { "destr": "^2.0.5", "node-fetch-native": "^1.6.7", "ufo": "^1.6.1" } }, "sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA=="], + "unicode-trie/pako": ["pako@0.2.9", "", {}, "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA=="], - "utif2/pako": ["pako@1.0.11", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="], + "unifont/ofetch": ["ofetch@1.5.1", "", { "dependencies": { "destr": "^2.0.5", "node-fetch-native": "^1.6.7", "ufo": "^1.6.1" } }, "sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA=="], "vitest/tinyexec": ["tinyexec@1.0.2", "", {}, "sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg=="], diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 953cd2f45f2a..3336655c2091 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -104,6 +104,7 @@ "hono": "catalog:", "hono-openapi": "catalog:", "ignore": "7.0.5", + "jimp": "1.6.0", "jsonc-parser": "3.3.1", "minimatch": "10.0.3", "open": "10.1.2", diff --git a/packages/opencode/src/cli/cmd/tui/util/clipboard.ts b/packages/opencode/src/cli/cmd/tui/util/clipboard.ts index ad1f86e3070c..dc81276a5622 100644 --- a/packages/opencode/src/cli/cmd/tui/util/clipboard.ts +++ b/packages/opencode/src/cli/cmd/tui/util/clipboard.ts @@ -3,6 +3,7 @@ import type { CliRenderer } from "@opentui/core" import { platform, release } from "os" import clipboardy from "clipboardy" import { lazy } from "../../../../util/lazy.js" +import { processImage } from "../../../../util/image.js" import { tmpdir } from "os" import path from "path" @@ -29,7 +30,7 @@ export namespace Clipboard { .quiet() const file = Bun.file(tmpfile) const buffer = await file.arrayBuffer() - return { data: Buffer.from(buffer).toString("base64"), mime: "image/png" } + return processImage(Buffer.from(buffer), "image/png") } catch { } finally { await $`rm -f "${tmpfile}"`.nothrow().quiet() @@ -43,7 +44,7 @@ export namespace Clipboard { if (base64) { const imageBuffer = Buffer.from(base64.trim(), "base64") if (imageBuffer.length > 0) { - return { data: imageBuffer.toString("base64"), mime: "image/png" } + return processImage(imageBuffer, "image/png") } } } @@ -51,11 +52,11 @@ export namespace Clipboard { if (os === "linux") { const wayland = await $`wl-paste -t image/png`.nothrow().arrayBuffer() if (wayland && wayland.byteLength > 0) { - return { data: Buffer.from(wayland).toString("base64"), mime: "image/png" } + return processImage(Buffer.from(wayland), "image/png") } const x11 = await $`xclip -selection clipboard -t image/png -o`.nothrow().arrayBuffer() if (x11 && x11.byteLength > 0) { - return { data: Buffer.from(x11).toString("base64"), mime: "image/png" } + return processImage(Buffer.from(x11), "image/png") } } diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts index fb3825302918..43ec7c23abcf 100644 --- a/packages/opencode/src/session/compaction.ts +++ b/packages/opencode/src/session/compaction.ts @@ -149,7 +149,7 @@ export namespace SessionCompaction { tools: {}, system: [], messages: [ - ...MessageV2.toModelMessages(input.messages, model), + ...(await MessageV2.toModelMessages(input.messages, model)), { role: "user", content: [ diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index 6358c6c5e9b0..6a2ba5f50d74 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -12,6 +12,16 @@ import { STATUS_CODES } from "http" import { iife } from "@/util/iife" import { type SystemError } from "bun" import type { Provider } from "@/provider/provider" +import { processImage } from "@/util/image" +import { appendFileSync } from "fs" + +const DEBUG_LOG = "/tmp/opencode-image-debug.log" +function dbg(msg: string) { + const line = `[${new Date().toISOString()}] [msg-v2] ${msg}\n` + try { + appendFileSync(DEBUG_LOG, line) + } catch {} +} export namespace MessageV2 { export const OutputLengthError = NamedError.create("MessageOutputLengthError", z.object({})) @@ -435,7 +445,8 @@ export namespace MessageV2 { }) export type WithParts = z.infer - export function toModelMessages(input: WithParts[], model: Provider.Model): ModelMessage[] { + export async function toModelMessages(input: WithParts[], model: Provider.Model): Promise { + dbg(`=== toModelMessages called: ${input.length} messages, model=${model.id} ===`) const result: UIMessage[] = [] const toolNames = new Set() @@ -452,6 +463,10 @@ export namespace MessageV2 { const attachments = (outputObject.attachments ?? []).filter((attachment) => { return attachment.url.startsWith("data:") && attachment.url.includes(",") }) + if (attachments.length > 0) + dbg( + `toModelOutput: ${attachments.length} attachment(s): ${attachments.map((a) => `${a.mime} urlLen=${a.url.length}`).join(", ")}`, + ) return { type: "content", @@ -462,7 +477,9 @@ export namespace MessageV2 { mediaType: attachment.mime, data: iife(() => { const commaIndex = attachment.url.indexOf(",") - return commaIndex === -1 ? attachment.url : attachment.url.slice(commaIndex + 1) + const b64 = commaIndex === -1 ? attachment.url : attachment.url.slice(commaIndex + 1) + dbg(` media part: mime=${attachment.mime} base64Len=${b64.length}`) + return b64 }), })), ], @@ -489,13 +506,31 @@ export namespace MessageV2 { text: part.text, }) // text/plain and directory files are converted into text parts, ignore them - if (part.type === "file" && part.mime !== "text/plain" && part.mime !== "application/x-directory") + if (part.type === "file" && part.mime !== "text/plain" && part.mime !== "application/x-directory") { + let url = part.url + let mime = part.mime + dbg(`file part: mime=${mime} filename=${part.filename} urlLen=${url.length}`) + if (mime.startsWith("image/") && mime !== "image/svg+xml") { + const match = url.match(/^data:([^;]+);base64,(.*)$/) + dbg(` image match: ${match ? `yes, base64Len=${match[2].length}` : "no match on data URL"}`) + if (match) { + try { + const processed = await processImage(Buffer.from(match[2], "base64"), mime) + url = `data:${processed.mime};base64,${processed.data}` + mime = processed.mime + dbg(` processed: mime=${mime} newUrlLen=${url.length}`) + } catch (e) { + dbg(` processImage THREW: ${e}`) + } + } + } userMessage.parts.push({ type: "file", - url: part.url, - mediaType: part.mime, + url, + mediaType: mime, filename: part.filename, }) + } if (part.type === "compaction") { userMessage.parts.push({ @@ -544,7 +579,20 @@ export namespace MessageV2 { toolNames.add(part.tool) if (part.state.status === "completed") { const outputText = part.state.time.compacted ? "[Old tool result content cleared]" : part.state.output - const attachments = part.state.time.compacted ? [] : (part.state.attachments ?? []) + const raw = part.state.time.compacted ? [] : (part.state.attachments ?? []) + const attachments = await Promise.all( + raw.map(async (att) => { + if (att.mime.startsWith("image/") && att.mime !== "image/svg+xml") { + const match = att.url.match(/^data:([^;]+);base64,(.*)$/) + if (match) { + dbg(`tool attachment: processing ${att.mime} base64Len=${match[2].length}`) + const processed = await processImage(Buffer.from(match[2], "base64"), att.mime) + return { mime: processed.mime, url: `data:${processed.mime};base64,${processed.data}` } + } + } + return att + }), + ) const output = attachments.length > 0 ? { diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index ab1eba5beb30..c6ab889bb84e 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -607,7 +607,7 @@ export namespace SessionPrompt { sessionID, system: [...(await SystemPrompt.environment(model)), ...(await InstructionPrompt.system())], messages: [ - ...MessageV2.toModelMessages(sessionMessages, model), + ...(await MessageV2.toModelMessages(sessionMessages, model)), ...(isLastStep ? [ { @@ -1819,7 +1819,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the }, ...(hasOnlySubtaskParts ? [{ role: "user" as const, content: subtaskParts.map((p) => p.prompt).join("\n") }] - : MessageV2.toModelMessages(contextMessages, model)), + : await MessageV2.toModelMessages(contextMessages, model)), ], }) const text = await result.text.catch((err) => log.error("failed to generate title", { error: err })) diff --git a/packages/opencode/src/tool/read.ts b/packages/opencode/src/tool/read.ts index f230cdf44cbb..5b2dd61930b5 100644 --- a/packages/opencode/src/tool/read.ts +++ b/packages/opencode/src/tool/read.ts @@ -9,6 +9,7 @@ import { Instance } from "../project/instance" import { Identifier } from "../id/id" import { assertExternalDirectory } from "./external-directory" import { InstructionPrompt } from "../session/instruction" +import { processImage } from "../util/image" const DEFAULT_READ_LIMIT = 2000 const MAX_LINE_LENGTH = 2000 @@ -67,7 +68,10 @@ export const ReadTool = Tool.define("read", { file.type.startsWith("image/") && file.type !== "image/svg+xml" && file.type !== "image/vnd.fastbidsheet" const isPdf = file.type === "application/pdf" if (isImage || isPdf) { - const mime = file.type + const raw = Buffer.from(await file.bytes()) + const processed = isImage ? await processImage(raw, file.type) : undefined + const mime = processed?.mime ?? file.type + const data = processed?.data ?? raw.toString("base64") const msg = `${isImage ? "Image" : "PDF"} read successfully` return { title, @@ -84,7 +88,7 @@ export const ReadTool = Tool.define("read", { messageID: ctx.messageID, type: "file", mime, - url: `data:${mime};base64,${Buffer.from(await file.bytes()).toString("base64")}`, + url: `data:${mime};base64,${data}`, }, ], } diff --git a/packages/opencode/src/util/image.ts b/packages/opencode/src/util/image.ts new file mode 100644 index 000000000000..b675b8976f94 --- /dev/null +++ b/packages/opencode/src/util/image.ts @@ -0,0 +1,93 @@ +import { Jimp } from "jimp" +import { Log } from "./log" +import { appendFileSync } from "fs" + +const log = Log.create({ service: "image" }) +const DEBUG_LOG = "/tmp/opencode-image-debug.log" + +function dbg(msg: string) { + const line = `[${new Date().toISOString()}] ${msg}\n` + try { + appendFileSync(DEBUG_LOG, line) + } catch {} +} + +// Anthropic rejects images >2000px per dimension in many-image requests +const MAX_DIMENSION = 2000 +const MAX_BUFFER = 3_932_160 +const API_LIMIT = 5_242_880 + +export async function processImage(buffer: Buffer, mime: string): Promise<{ data: string; mime: string }> { + dbg(`processImage called: mime=${mime} bufLen=${buffer.length}`) + + try { + const img = await Jimp.fromBuffer(buffer) + const w = img.width + const h = img.height + dbg(`processImage: dimensions=${w}x${h} bufLen=${buffer.length}`) + + if (buffer.length <= MAX_BUFFER && w <= MAX_DIMENSION && h <= MAX_DIMENSION) { + dbg(`processImage: within limits, passthrough`) + return { data: buffer.toString("base64"), mime } + } + + let tw = w + let th = h + if (tw > MAX_DIMENSION) { + th = Math.round((th * MAX_DIMENSION) / tw) + tw = MAX_DIMENSION + } + if (th > MAX_DIMENSION) { + tw = Math.round((tw * MAX_DIMENSION) / th) + th = MAX_DIMENSION + } + + const needsResize = tw !== w || th !== h + + // If only file size is too large (no dimension issue), try compression + if (!needsResize && buffer.length > MAX_BUFFER) { + for (const q of [80, 60, 40, 20]) { + const compressed = await img.getBuffer("image/jpeg", { quality: q }) + dbg(`processImage: jpeg q=${q} → ${compressed.length}`) + if (compressed.length <= MAX_BUFFER) + return { data: Buffer.from(compressed).toString("base64"), mime: "image/jpeg" } + } + } + + if (needsResize) { + dbg(`processImage: resizing ${w}x${h} → ${tw}x${th}`) + log.info("resizing image", { from: `${w}x${h}`, to: `${tw}x${th}` }) + img.scaleToFit({ w: tw, h: th }) + } + + // Try original format first + const resized = await img.getBuffer("image/jpeg", { quality: 85 }) + dbg(`processImage: resized → ${resized.length} bytes`) + if (resized.length <= MAX_BUFFER) return { data: Buffer.from(resized).toString("base64"), mime: "image/jpeg" } + + // Progressive quality reduction + for (const q of [80, 60, 40, 20]) { + const compressed = await img.getBuffer("image/jpeg", { quality: q }) + dbg(`processImage: jpeg q=${q} → ${compressed.length}`) + if (compressed.length <= MAX_BUFFER) + return { data: Buffer.from(compressed).toString("base64"), mime: "image/jpeg" } + } + + // Aggressive downscale as last resort + const sw = Math.min(tw, 1000) + const sh = Math.round((th * sw) / Math.max(tw, 1)) + dbg(`processImage: aggressive downscale → ${sw}x${sh}`) + log.info("aggressive downscale", { to: `${sw}x${sh}` }) + img.scaleToFit({ w: sw, h: sh }) + const tiny = await img.getBuffer("image/jpeg", { quality: 20 }) + dbg(`processImage: tiny → ${tiny.length} bytes`) + return { data: Buffer.from(tiny).toString("base64"), mime: "image/jpeg" } + } catch (err) { + dbg(`processImage: ERROR: ${err}`) + log.warn("image processing failed, sending raw", { error: err }) + if (buffer.length <= API_LIMIT) return { data: buffer.toString("base64"), mime } + throw new Error( + `Unable to process image (${(buffer.length / 1048576).toFixed(1)} MB). The image exceeds the 5MB API limit and processing failed. Please use a smaller image.`, + ) + } +} diff --git a/packages/opencode/test/session/message-v2.test.ts b/packages/opencode/test/session/message-v2.test.ts index 2f632ad1cf2b..14ae012751a5 100644 --- a/packages/opencode/test/session/message-v2.test.ts +++ b/packages/opencode/test/session/message-v2.test.ts @@ -103,684 +103,658 @@ function basePart(messageID: string, id: string) { } describe("session.message-v2.toModelMessage", () => { - test("filters out messages with no parts", () => { - const input: MessageV2.WithParts[] = [ - { - info: userInfo("m-empty"), - parts: [], - }, - { - info: userInfo("m-user"), - parts: [ - { - ...basePart("m-user", "p1"), - type: "text", - text: "hello", - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "hello" }], - }, - ]) - }) - - test("filters out messages with only ignored parts", () => { - const messageID = "m-user" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(messageID), - parts: [ - { - ...basePart(messageID, "p1"), - type: "text", - text: "ignored", - ignored: true, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([]) - }) - - test("includes synthetic text parts", () => { - const messageID = "m-user" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(messageID), - parts: [ - { - ...basePart(messageID, "p1"), - type: "text", - text: "hello", - synthetic: true, - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo("m-assistant", messageID), - parts: [ - { - ...basePart("m-assistant", "a1"), - type: "text", - text: "assistant", - synthetic: true, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "hello" }], - }, - { - role: "assistant", - content: [{ type: "text", text: "assistant" }], - }, - ]) - }) - - test("converts user text/file parts and injects compaction/subtask prompts", () => { - const messageID = "m-user" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(messageID), - parts: [ - { - ...basePart(messageID, "p1"), - type: "text", - text: "hello", - }, - { - ...basePart(messageID, "p2"), - type: "text", - text: "ignored", - ignored: true, - }, - { - ...basePart(messageID, "p3"), - type: "file", - mime: "image/png", - filename: "img.png", - url: "https://example.com/img.png", - }, - { - ...basePart(messageID, "p4"), - type: "file", - mime: "text/plain", - filename: "note.txt", - url: "https://example.com/note.txt", - }, - { - ...basePart(messageID, "p5"), - type: "file", - mime: "application/x-directory", - filename: "dir", - url: "https://example.com/dir", - }, - { - ...basePart(messageID, "p6"), - type: "compaction", - auto: true, - }, - { - ...basePart(messageID, "p7"), - type: "subtask", - prompt: "prompt", - description: "desc", - agent: "agent", - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [ - { type: "text", text: "hello" }, - { - type: "file", - mediaType: "image/png", - filename: "img.png", - data: "https://example.com/img.png", - }, - { type: "text", text: "What did we do so far?" }, - { type: "text", text: "The following tool was executed by the user" }, - ], - }, - ]) - }) - - test("converts assistant tool completion into tool-call + tool-result messages with attachments", () => { - const userID = "m-user" - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(userID), - parts: [ - { - ...basePart(userID, "u1"), - type: "text", - text: "run tool", - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo(assistantID, userID), - parts: [ - { - ...basePart(assistantID, "a1"), - type: "text", - text: "done", - metadata: { openai: { assistant: "meta" } }, - }, - { - ...basePart(assistantID, "a2"), - type: "tool", - callID: "call-1", - tool: "bash", - state: { - status: "completed", - input: { cmd: "ls" }, - output: "ok", - title: "Bash", - metadata: {}, - time: { start: 0, end: 1 }, - attachments: [ - { - ...basePart(assistantID, "file-1"), - type: "file", - mime: "image/png", - filename: "attachment.png", - url: "data:image/png;base64,Zm9v", - }, - ], - }, - metadata: { openai: { tool: "meta" } }, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "run tool" }], - }, - { - role: "assistant", - content: [ - { type: "text", text: "done", providerOptions: { openai: { assistant: "meta" } } }, - { - type: "tool-call", - toolCallId: "call-1", - toolName: "bash", + test("filters out messages with no parts", async () => {const input: MessageV2.WithParts[] = [ + { + info: userInfo("m-empty"), + parts: [], + }, + { + info: userInfo("m-user"), + parts: [ + { + ...basePart("m-user", "p1"), + type: "text", + text: "hello", + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "hello" }], + }, + ])}) + + test("filters out messages with only ignored parts", async () => {const messageID = "m-user" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(messageID), + parts: [ + { + ...basePart(messageID, "p1"), + type: "text", + text: "ignored", + ignored: true, + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([])}) + + test("includes synthetic text parts", async () => {const messageID = "m-user" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(messageID), + parts: [ + { + ...basePart(messageID, "p1"), + type: "text", + text: "hello", + synthetic: true, + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo("m-assistant", messageID), + parts: [ + { + ...basePart("m-assistant", "a1"), + type: "text", + text: "assistant", + synthetic: true, + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "hello" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "assistant" }], + }, + ])}) + + test("converts user text/file parts and injects compaction/subtask prompts", async () => {const messageID = "m-user" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(messageID), + parts: [ + { + ...basePart(messageID, "p1"), + type: "text", + text: "hello", + }, + { + ...basePart(messageID, "p2"), + type: "text", + text: "ignored", + ignored: true, + }, + { + ...basePart(messageID, "p3"), + type: "file", + mime: "image/png", + filename: "img.png", + url: "https://example.com/img.png", + }, + { + ...basePart(messageID, "p4"), + type: "file", + mime: "text/plain", + filename: "note.txt", + url: "https://example.com/note.txt", + }, + { + ...basePart(messageID, "p5"), + type: "file", + mime: "application/x-directory", + filename: "dir", + url: "https://example.com/dir", + }, + { + ...basePart(messageID, "p6"), + type: "compaction", + auto: true, + }, + { + ...basePart(messageID, "p7"), + type: "subtask", + prompt: "prompt", + description: "desc", + agent: "agent", + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [ + { type: "text", text: "hello" }, + { + type: "file", + mediaType: "image/png", + filename: "img.png", + data: "https://example.com/img.png", + }, + { type: "text", text: "What did we do so far?" }, + { type: "text", text: "The following tool was executed by the user" }, + ], + }, + ])}) + + test("converts assistant tool completion into tool-call + tool-result messages with attachments", async () => {const userID = "m-user" + const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(userID), + parts: [ + { + ...basePart(userID, "u1"), + type: "text", + text: "run tool", + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo(assistantID, userID), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "text", + text: "done", + metadata: { openai: { assistant: "meta" } }, + }, + { + ...basePart(assistantID, "a2"), + type: "tool", + callID: "call-1", + tool: "bash", + state: { + status: "completed", input: { cmd: "ls" }, - providerExecuted: undefined, - providerOptions: { openai: { tool: "meta" } }, - }, - ], - }, - { - role: "tool", - content: [ - { - type: "tool-result", - toolCallId: "call-1", - toolName: "bash", - output: { - type: "content", - value: [ - { type: "text", text: "ok" }, - { type: "media", mediaType: "image/png", data: "Zm9v" }, - ], - }, - providerOptions: { openai: { tool: "meta" } }, - }, - ], - }, - ]) - }) - - test("omits provider metadata when assistant model differs", () => { - const userID = "m-user" - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(userID), - parts: [ - { - ...basePart(userID, "u1"), - type: "text", - text: "run tool", - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo(assistantID, userID, undefined, { providerID: "other", modelID: "other" }), - parts: [ - { - ...basePart(assistantID, "a1"), - type: "text", - text: "done", - metadata: { openai: { assistant: "meta" } }, - }, - { - ...basePart(assistantID, "a2"), - type: "tool", - callID: "call-1", - tool: "bash", - state: { - status: "completed", - input: { cmd: "ls" }, - output: "ok", - title: "Bash", - metadata: {}, - time: { start: 0, end: 1 }, - }, - metadata: { openai: { tool: "meta" } }, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "run tool" }], - }, - { - role: "assistant", - content: [ - { type: "text", text: "done" }, - { - type: "tool-call", - toolCallId: "call-1", - toolName: "bash", + output: "ok", + title: "Bash", + metadata: {}, + time: { start: 0, end: 1 }, + attachments: [ + { + ...basePart(assistantID, "file-1"), + type: "file", + mime: "image/png", + filename: "attachment.png", + url: "data:image/png;base64,Zm9v", + }, + ], + }, + metadata: { openai: { tool: "meta" } }, + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "run tool" }], + }, + { + role: "assistant", + content: [ + { type: "text", text: "done", providerOptions: { openai: { assistant: "meta" } } }, + { + type: "tool-call", + toolCallId: "call-1", + toolName: "bash", + input: { cmd: "ls" }, + providerExecuted: undefined, + providerOptions: { openai: { tool: "meta" } }, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "bash", + output: { + type: "content", + value: [ + { type: "text", text: "ok" }, + { type: "media", mediaType: "image/png", data: "Zm9v" }, + ], + }, + providerOptions: { openai: { tool: "meta" } }, + }, + ], + }, + ])}) + + test("omits provider metadata when assistant model differs", async () => {const userID = "m-user" + const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(userID), + parts: [ + { + ...basePart(userID, "u1"), + type: "text", + text: "run tool", + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo(assistantID, userID, undefined, { providerID: "other", modelID: "other" }), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "text", + text: "done", + metadata: { openai: { assistant: "meta" } }, + }, + { + ...basePart(assistantID, "a2"), + type: "tool", + callID: "call-1", + tool: "bash", + state: { + status: "completed", input: { cmd: "ls" }, - providerExecuted: undefined, - }, - ], - }, - { - role: "tool", - content: [ - { - type: "tool-result", - toolCallId: "call-1", - toolName: "bash", - output: { type: "text", value: "ok" }, - }, - ], - }, - ]) - }) - - test("replaces compacted tool output with placeholder", () => { - const userID = "m-user" - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(userID), - parts: [ - { - ...basePart(userID, "u1"), - type: "text", - text: "run tool", - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo(assistantID, userID), - parts: [ - { - ...basePart(assistantID, "a1"), - type: "tool", - callID: "call-1", - tool: "bash", - state: { - status: "completed", - input: { cmd: "ls" }, - output: "this should be cleared", - title: "Bash", - metadata: {}, - time: { start: 0, end: 1, compacted: 1 }, - }, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "run tool" }], - }, - { - role: "assistant", - content: [ - { - type: "tool-call", - toolCallId: "call-1", - toolName: "bash", + output: "ok", + title: "Bash", + metadata: {}, + time: { start: 0, end: 1 }, + }, + metadata: { openai: { tool: "meta" } }, + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "run tool" }], + }, + { + role: "assistant", + content: [ + { type: "text", text: "done" }, + { + type: "tool-call", + toolCallId: "call-1", + toolName: "bash", + input: { cmd: "ls" }, + providerExecuted: undefined, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "bash", + output: { type: "text", value: "ok" }, + }, + ], + }, + ])}) + + test("replaces compacted tool output with placeholder", async () => {const userID = "m-user" + const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(userID), + parts: [ + { + ...basePart(userID, "u1"), + type: "text", + text: "run tool", + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo(assistantID, userID), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "tool", + callID: "call-1", + tool: "bash", + state: { + status: "completed", input: { cmd: "ls" }, - providerExecuted: undefined, + output: "this should be cleared", + title: "Bash", + metadata: {}, + time: { start: 0, end: 1, compacted: 1 }, }, - ], - }, - { - role: "tool", - content: [ - { - type: "tool-result", - toolCallId: "call-1", - toolName: "bash", - output: { type: "text", value: "[Old tool result content cleared]" }, - }, - ], - }, - ]) - }) - - test("converts assistant tool error into error-text tool result", () => { - const userID = "m-user" - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(userID), - parts: [ - { - ...basePart(userID, "u1"), - type: "text", - text: "run tool", - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo(assistantID, userID), - parts: [ - { - ...basePart(assistantID, "a1"), - type: "tool", - callID: "call-1", - tool: "bash", - state: { - status: "error", - input: { cmd: "ls" }, - error: "nope", - time: { start: 0, end: 1 }, - metadata: {}, - }, - metadata: { openai: { tool: "meta" } }, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "run tool" }], - }, - { - role: "assistant", - content: [ - { - type: "tool-call", - toolCallId: "call-1", - toolName: "bash", + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "run tool" }], + }, + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-1", + toolName: "bash", + input: { cmd: "ls" }, + providerExecuted: undefined, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "bash", + output: { type: "text", value: "[Old tool result content cleared]" }, + }, + ], + }, + ])}) + + test("converts assistant tool error into error-text tool result", async () => {const userID = "m-user" + const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(userID), + parts: [ + { + ...basePart(userID, "u1"), + type: "text", + text: "run tool", + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo(assistantID, userID), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "tool", + callID: "call-1", + tool: "bash", + state: { + status: "error", input: { cmd: "ls" }, - providerExecuted: undefined, - providerOptions: { openai: { tool: "meta" } }, - }, - ], - }, - { - role: "tool", - content: [ - { - type: "tool-result", - toolCallId: "call-1", - toolName: "bash", - output: { type: "error-text", value: "nope" }, - providerOptions: { openai: { tool: "meta" } }, - }, - ], - }, - ]) - }) - - test("filters assistant messages with non-abort errors", () => { - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: assistantInfo( - assistantID, - "m-parent", - new MessageV2.APIError({ message: "boom", isRetryable: true }).toObject() as MessageV2.APIError, - ), - parts: [ - { - ...basePart(assistantID, "a1"), - type: "text", - text: "should not render", - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([]) - }) - - test("includes aborted assistant messages only when they have non-step-start/reasoning content", () => { - const assistantID1 = "m-assistant-1" - const assistantID2 = "m-assistant-2" - - const aborted = new MessageV2.AbortedError({ message: "aborted" }).toObject() as MessageV2.Assistant["error"] - - const input: MessageV2.WithParts[] = [ - { - info: assistantInfo(assistantID1, "m-parent", aborted), - parts: [ - { - ...basePart(assistantID1, "a1"), - type: "reasoning", - text: "thinking", - time: { start: 0 }, - }, - { - ...basePart(assistantID1, "a2"), - type: "text", - text: "partial answer", - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo(assistantID2, "m-parent", aborted), - parts: [ - { - ...basePart(assistantID2, "b1"), - type: "step-start", - }, - { - ...basePart(assistantID2, "b2"), - type: "reasoning", - text: "thinking", - time: { start: 0 }, - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "assistant", - content: [ - { type: "reasoning", text: "thinking", providerOptions: undefined }, - { type: "text", text: "partial answer" }, - ], - }, - ]) - }) - - test("splits assistant messages on step-start boundaries", () => { - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: assistantInfo(assistantID, "m-parent"), - parts: [ - { - ...basePart(assistantID, "p1"), - type: "text", - text: "first", - }, - { - ...basePart(assistantID, "p2"), - type: "step-start", - }, - { - ...basePart(assistantID, "p3"), - type: "text", - text: "second", - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([ - { - role: "assistant", - content: [{ type: "text", text: "first" }], - }, - { - role: "assistant", - content: [{ type: "text", text: "second" }], - }, - ]) - }) - - test("drops messages that only contain step-start parts", () => { - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: assistantInfo(assistantID, "m-parent"), - parts: [ - { - ...basePart(assistantID, "p1"), - type: "step-start", - }, - ] as MessageV2.Part[], - }, - ] - - expect(MessageV2.toModelMessages(input, model)).toStrictEqual([]) - }) - - test("converts pending/running tool calls to error results to prevent dangling tool_use", () => { - const userID = "m-user" - const assistantID = "m-assistant" - - const input: MessageV2.WithParts[] = [ - { - info: userInfo(userID), - parts: [ - { - ...basePart(userID, "u1"), - type: "text", - text: "run tool", - }, - ] as MessageV2.Part[], - }, - { - info: assistantInfo(assistantID, userID), - parts: [ - { - ...basePart(assistantID, "a1"), - type: "tool", - callID: "call-pending", - tool: "bash", - state: { - status: "pending", - input: { cmd: "ls" }, - raw: "", - }, + error: "nope", + time: { start: 0, end: 1 }, + metadata: {}, }, - { - ...basePart(assistantID, "a2"), - type: "tool", - callID: "call-running", - tool: "read", - state: { - status: "running", - input: { path: "/tmp" }, - time: { start: 0 }, - }, - }, - ] as MessageV2.Part[], - }, - ] - - const result = MessageV2.toModelMessages(input, model) - - expect(result).toStrictEqual([ - { - role: "user", - content: [{ type: "text", text: "run tool" }], - }, - { - role: "assistant", - content: [ - { - type: "tool-call", - toolCallId: "call-pending", - toolName: "bash", + metadata: { openai: { tool: "meta" } }, + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "run tool" }], + }, + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-1", + toolName: "bash", + input: { cmd: "ls" }, + providerExecuted: undefined, + providerOptions: { openai: { tool: "meta" } }, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-1", + toolName: "bash", + output: { type: "error-text", value: "nope" }, + providerOptions: { openai: { tool: "meta" } }, + }, + ], + }, + ])}) + + test("filters assistant messages with non-abort errors", async () => {const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: assistantInfo( + assistantID, + "m-parent", + new MessageV2.APIError({ message: "boom", isRetryable: true }).toObject() as MessageV2.APIError, + ), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "text", + text: "should not render", + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([])}) + + test("includes aborted assistant messages only when they have non-step-start/reasoning content", async () => {const assistantID1 = "m-assistant-1" + const assistantID2 = "m-assistant-2" + + const aborted = new MessageV2.AbortedError({ message: "aborted" }).toObject() as MessageV2.Assistant["error"] + + const input: MessageV2.WithParts[] = [ + { + info: assistantInfo(assistantID1, "m-parent", aborted), + parts: [ + { + ...basePart(assistantID1, "a1"), + type: "reasoning", + text: "thinking", + time: { start: 0 }, + }, + { + ...basePart(assistantID1, "a2"), + type: "text", + text: "partial answer", + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo(assistantID2, "m-parent", aborted), + parts: [ + { + ...basePart(assistantID2, "b1"), + type: "step-start", + }, + { + ...basePart(assistantID2, "b2"), + type: "reasoning", + text: "thinking", + time: { start: 0 }, + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "assistant", + content: [ + { type: "reasoning", text: "thinking", providerOptions: undefined }, + { type: "text", text: "partial answer" }, + ], + }, + ])}) + + test("splits assistant messages on step-start boundaries", async () => {const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: assistantInfo(assistantID, "m-parent"), + parts: [ + { + ...basePart(assistantID, "p1"), + type: "text", + text: "first", + }, + { + ...basePart(assistantID, "p2"), + type: "step-start", + }, + { + ...basePart(assistantID, "p3"), + type: "text", + text: "second", + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([ + { + role: "assistant", + content: [{ type: "text", text: "first" }], + }, + { + role: "assistant", + content: [{ type: "text", text: "second" }], + }, + ])}) + + test("drops messages that only contain step-start parts", async () => {const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: assistantInfo(assistantID, "m-parent"), + parts: [ + { + ...basePart(assistantID, "p1"), + type: "step-start", + }, + ] as MessageV2.Part[], + }, + ] + + expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([])}) + + test("converts pending/running tool calls to error results to prevent dangling tool_use", async () => {const userID = "m-user" + const assistantID = "m-assistant" + + const input: MessageV2.WithParts[] = [ + { + info: userInfo(userID), + parts: [ + { + ...basePart(userID, "u1"), + type: "text", + text: "run tool", + }, + ] as MessageV2.Part[], + }, + { + info: assistantInfo(assistantID, userID), + parts: [ + { + ...basePart(assistantID, "a1"), + type: "tool", + callID: "call-pending", + tool: "bash", + state: { + status: "pending", input: { cmd: "ls" }, - providerExecuted: undefined, - }, - { - type: "tool-call", - toolCallId: "call-running", - toolName: "read", + raw: "", + }, + }, + { + ...basePart(assistantID, "a2"), + type: "tool", + callID: "call-running", + tool: "read", + state: { + status: "running", input: { path: "/tmp" }, - providerExecuted: undefined, - }, - ], - }, - { - role: "tool", - content: [ - { - type: "tool-result", - toolCallId: "call-pending", - toolName: "bash", - output: { type: "error-text", value: "[Tool execution was interrupted]" }, - }, - { - type: "tool-result", - toolCallId: "call-running", - toolName: "read", - output: { type: "error-text", value: "[Tool execution was interrupted]" }, + time: { start: 0 }, }, - ], - }, - ]) - }) + }, + ] as MessageV2.Part[], + }, + ] + + const result = await MessageV2.toModelMessages(input, model) + + expect(result).toStrictEqual([ + { + role: "user", + content: [{ type: "text", text: "run tool" }], + }, + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId: "call-pending", + toolName: "bash", + input: { cmd: "ls" }, + providerExecuted: undefined, + }, + { + type: "tool-call", + toolCallId: "call-running", + toolName: "read", + input: { path: "/tmp" }, + providerExecuted: undefined, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: "call-pending", + toolName: "bash", + output: { type: "error-text", value: "[Tool execution was interrupted]" }, + }, + { + type: "tool-result", + toolCallId: "call-running", + toolName: "read", + output: { type: "error-text", value: "[Tool execution was interrupted]" }, + }, + ], + }, + ])}) })