Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
298 changes: 298 additions & 0 deletions src/api/providers/__tests__/anthropic.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -741,4 +741,302 @@ describe("AnthropicHandler", () => {
})
})
})

describe("extended thinking with signature capture", () => {
const systemPrompt = "You are a helpful assistant."
const messages: Anthropic.Messages.MessageParam[] = [
{
role: "user",
content: [{ type: "text" as const, text: "Think about this carefully" }],
},
]

it("should capture signature_delta and make it available via getThoughtSignature()", async () => {
mockCreate.mockImplementationOnce(async () => ({
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: {
usage: {
input_tokens: 100,
output_tokens: 50,
},
},
}
yield {
type: "content_block_start",
index: 0,
content_block: {
type: "thinking",
thinking: "Let me think...",
},
}
yield {
type: "content_block_delta",
index: 0,
delta: {
type: "thinking_delta",
thinking: " about this carefully",
},
}
yield {
type: "content_block_delta",
index: 0,
delta: {
type: "signature_delta",
signature: "test_signature_123",
},
}
yield {
type: "content_block_stop",
index: 0,
}
yield {
type: "content_block_start",
index: 1,
content_block: {
type: "text",
text: "Here is my response",
},
}
yield {
type: "content_block_stop",
index: 1,
}
},
}))

const stream = handler.createMessage(systemPrompt, messages, {
taskId: "test-task",
})

const chunks: any[] = []
for await (const chunk of stream) {
chunks.push(chunk)
}

// Verify reasoning chunks were emitted
const reasoningChunks = chunks.filter((chunk) => chunk.type === "reasoning")
expect(reasoningChunks).toHaveLength(2)
expect(reasoningChunks[0].text).toBe("Let me think...")
expect(reasoningChunks[1].text).toBe(" about this carefully")

// Verify thinking_complete chunk was emitted with signature
const thinkingCompleteChunk = chunks.find((chunk) => chunk.type === "thinking_complete")
expect(thinkingCompleteChunk).toBeDefined()
expect(thinkingCompleteChunk.signature).toBe("test_signature_123")

// Verify getThoughtSignature() returns the captured signature
expect(handler.getThoughtSignature()).toBe("test_signature_123")
})

it("should reset signature for each new request", async () => {
// First request with signature
mockCreate.mockImplementationOnce(async () => ({
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: { usage: { input_tokens: 100, output_tokens: 50 } },
}
yield {
type: "content_block_start",
index: 0,
content_block: { type: "thinking", thinking: "First thinking" },
}
yield {
type: "content_block_delta",
index: 0,
delta: { type: "signature_delta", signature: "first_signature" },
}
yield { type: "content_block_stop", index: 0 }
},
}))

const stream1 = handler.createMessage(systemPrompt, messages, { taskId: "test-task-1" })
for await (const _chunk of stream1) {
// consume
}
expect(handler.getThoughtSignature()).toBe("first_signature")

// Second request without signature
mockCreate.mockImplementationOnce(async () => ({
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: { usage: { input_tokens: 100, output_tokens: 50 } },
}
yield {
type: "content_block_start",
index: 0,
content_block: { type: "text", text: "Just text, no thinking" },
}
yield { type: "content_block_stop", index: 0 }
},
}))

const stream2 = handler.createMessage(systemPrompt, messages, { taskId: "test-task-2" })
for await (const _chunk of stream2) {
// consume
}

// Signature should be reset (undefined) for the new request
expect(handler.getThoughtSignature()).toBeUndefined()
})

it("should accumulate signature_delta chunks (incremental signature)", async () => {
mockCreate.mockImplementationOnce(async () => ({
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: { usage: { input_tokens: 100, output_tokens: 50 } },
}
yield {
type: "content_block_start",
index: 0,
content_block: { type: "thinking", thinking: "Thinking..." },
}
yield {
type: "content_block_delta",
index: 0,
delta: { type: "signature_delta", signature: "sig_part1" },
}
yield {
type: "content_block_delta",
index: 0,
delta: { type: "signature_delta", signature: "_part2" },
}
yield {
type: "content_block_delta",
index: 0,
delta: { type: "signature_delta", signature: "_part3" },
}
yield { type: "content_block_stop", index: 0 }
},
}))

const stream = handler.createMessage(systemPrompt, messages, { taskId: "test-task" })

const chunks: any[] = []
for await (const chunk of stream) {
chunks.push(chunk)
}

// Verify the accumulated signature
expect(handler.getThoughtSignature()).toBe("sig_part1_part2_part3")

// Verify thinking_complete has the accumulated signature
const thinkingCompleteChunk = chunks.find((chunk) => chunk.type === "thinking_complete")
expect(thinkingCompleteChunk?.signature).toBe("sig_part1_part2_part3")
})

it("should not emit thinking_complete if no signature is captured", async () => {
mockCreate.mockImplementationOnce(async () => ({
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: { usage: { input_tokens: 100, output_tokens: 50 } },
}
yield {
type: "content_block_start",
index: 0,
content_block: { type: "thinking", thinking: "Thinking without signature" },
}
yield {
type: "content_block_delta",
index: 0,
delta: { type: "thinking_delta", thinking: "More thinking" },
}
// No signature_delta event
yield { type: "content_block_stop", index: 0 }
},
}))

const stream = handler.createMessage(systemPrompt, messages, { taskId: "test-task" })

const chunks: any[] = []
for await (const chunk of stream) {
chunks.push(chunk)
}

// Verify thinking_complete was NOT emitted (no signature)
const thinkingCompleteChunk = chunks.find((chunk) => chunk.type === "thinking_complete")
expect(thinkingCompleteChunk).toBeUndefined()

// Verify getThoughtSignature() returns undefined
expect(handler.getThoughtSignature()).toBeUndefined()
})

it("should handle interleaved thinking with tool use", async () => {
mockCreate.mockImplementationOnce(async () => ({
async *[Symbol.asyncIterator]() {
yield {
type: "message_start",
message: { usage: { input_tokens: 100, output_tokens: 50 } },
}
// First: thinking block
yield {
type: "content_block_start",
index: 0,
content_block: { type: "thinking", thinking: "Let me think about what tool to use" },
}
yield {
type: "content_block_delta",
index: 0,
delta: { type: "signature_delta", signature: "thinking_signature_abc" },
}
yield { type: "content_block_stop", index: 0 }
// Second: tool use block
yield {
type: "content_block_start",
index: 1,
content_block: {
type: "tool_use",
id: "toolu_456",
name: "get_weather",
},
}
yield {
type: "content_block_delta",
index: 1,
delta: {
type: "input_json_delta",
partial_json: '{"location":"Paris"}',
},
}
yield { type: "content_block_stop", index: 1 }
},
}))

const stream = handler.createMessage(systemPrompt, messages, {
taskId: "test-task",
tools: [
{
type: "function" as const,
function: {
name: "get_weather",
description: "Get weather",
parameters: { type: "object", properties: { location: { type: "string" } } },
},
},
],
})

const chunks: any[] = []
for await (const chunk of stream) {
chunks.push(chunk)
}

// Verify thinking_complete was emitted for the thinking block
const thinkingCompleteChunk = chunks.find((chunk) => chunk.type === "thinking_complete")
expect(thinkingCompleteChunk).toBeDefined()
expect(thinkingCompleteChunk.signature).toBe("thinking_signature_abc")

// Verify signature is available for tool use continuation
expect(handler.getThoughtSignature()).toBe("thinking_signature_abc")

// Verify tool_call_partial was also emitted
const toolChunks = chunks.filter((chunk) => chunk.type === "tool_call_partial")
expect(toolChunks.length).toBeGreaterThan(0)
})
})
})
Loading
Loading