Anthropic: Full support for Claude-3 models. Closes #443, #450

Thanks to @slapglif in #450 for a reference implementation.

src/common/util/modelUtils.ts

@@ -15,6 +15,9 @@ export function prettyBaseModel(model: string | undefined): string {
   if (model.includes('gpt-3.5-turbo-16k')) return '3.5 Turbo 16k';
   if (model.includes('gpt-3.5-turbo')) return '3.5 Turbo';
   if (model.endsWith('.bin')) return model.slice(0, -4);
+  // [Anthropic]
+  if (model.includes('claude-3-opus')) return 'Claude 3 Opus';
+  if (model.includes('claude-3-sonnet')) return 'Claude 3 Sonnet';
   // [LM Studio]
   if (model.startsWith('C:\\') || model.startsWith('D:\\'))
     return getModelFromFile(model).replace('.gguf', '');

src/modules/llms/server/llm.server.streaming.ts

@@ -2,12 +2,12 @@ import { z } from 'zod';
 import { NextRequest, NextResponse } from 'next/server';
 import { createParser as createEventsourceParser, EventSourceParseCallback, EventSourceParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
 
-import { createEmptyReadableStream, debugGenerateCurlCommand, nonTrpcServerFetchOrThrow, safeErrorString, SERVER_DEBUG_WIRE } from '~/server/wire';
+import { createEmptyReadableStream, debugGenerateCurlCommand, nonTrpcServerFetchOrThrow, safeErrorString, SERVER_DEBUG_WIRE, serverCapitalizeFirstLetter } from '~/server/wire';
 
 
 // Anthropic server imports
-import type { AnthropicWire } from './anthropic/anthropic.wiretypes';
-import { anthropicAccess, anthropicAccessSchema, anthropicChatCompletionPayload } from './anthropic/anthropic.router';
+import { AnthropicWireMessagesResponse, anthropicWireMessagesResponseSchema } from './anthropic/anthropic.wiretypes';
+import { anthropicAccess, anthropicAccessSchema, anthropicMessagesPayloadOrThrow } from './anthropic/anthropic.router';
 
 // Gemini server imports
 import { geminiAccess, geminiAccessSchema, geminiGenerateContentTextPayload } from './gemini/gemini.router';
@@ -38,7 +38,7 @@ type MuxingFormat = 'sse' | 'json-nl';
  * The peculiarity of our parser is the injection of a JSON structure at the beginning of the stream, to
  * communicate parameters before the text starts flowing to the client.
  */
-type AIStreamParser = (data: string) => { text: string, close: boolean };
+type AIStreamParser = (data: string, eventType?: string) => { text: string, close: boolean };
 
 
 const chatStreamingInputSchema = z.object({
@@ -74,9 +74,9 @@ export async function llmStreamingRelayHandler(req: NextRequest): Promise<Respon
     let body: object;
     switch (access.dialect) {
       case 'anthropic':
-        requestAccess = anthropicAccess(access, '/v1/complete');
-        body = anthropicChatCompletionPayload(model, history, true);
-        vendorStreamParser = createStreamParserAnthropic();
+        requestAccess = anthropicAccess(access, '/v1/messages');
+        body = anthropicMessagesPayloadOrThrow(model, history, true);
+        vendorStreamParser = createStreamParserAnthropicMessages();
         break;
 
       case 'gemini':
@@ -217,7 +217,7 @@ function createEventStreamTransformer(muxingFormat: MuxingFormat, vendorTextPars
         }
 
         try {
-          const { text, close } = vendorTextParser(event.data);
+          const { text, close } = vendorTextParser(event.data, event.event);
           if (text)
             controller.enqueue(textEncoder.encode(text));
           if (close)
@@ -246,19 +246,95 @@ function createEventStreamTransformer(muxingFormat: MuxingFormat, vendorTextPars
 
 /// Stream Parsers
 
-function createStreamParserAnthropic(): AIStreamParser {
-  let hasBegun = false;
+function createStreamParserAnthropicMessages(): AIStreamParser {
+  let responseMessage: AnthropicWireMessagesResponse | null = null;
+  let hasErrored = false;
 
-  return (data: string) => {
+  // Note: at this stage, the parser only returns the text content as text, which is streamed as text
+  //       to the client. It is however building in parallel the responseMessage object, which is not
+  //       yet used, but contains token counts, for instance.
+  return (data: string, eventName?: string) => {
+    let text = '';
 
-    const json: AnthropicWire.Complete.Response = JSON.parse(data);
-    let text = json.completion;
+    // if we've errored, we should not be receiving more data
+    if (hasErrored)
+      console.log('Anthropic stream has errored already, but received more data:', data);
 
-    // hack: prepend the model name to the first packet
-    if (!hasBegun) {
-      hasBegun = true;
-      const firstPacket: ChatStreamingFirstOutputPacketSchema = { model: json.model };
-      text = JSON.stringify(firstPacket) + text;
+    switch (eventName) {
+      // Ignore pings
+      case 'ping':
+        break;
+
+      // Initialize the message content for a new message
+      case 'message_start':
+        const firstMessage = !responseMessage;
+        const { message } = JSON.parse(data);
+        responseMessage = anthropicWireMessagesResponseSchema.parse(message);
+        // hack: prepend the model name to the first packet
+        if (firstMessage) {
+          const firstPacket: ChatStreamingFirstOutputPacketSchema = { model: responseMessage.model };
+          text = JSON.stringify(firstPacket);
+        }
+        break;
+
+      // Initialize content block if needed
+      case 'content_block_start':
+        if (responseMessage) {
+          const { index, content_block } = JSON.parse(data);
+          if (responseMessage.content[index] === undefined)
+            responseMessage.content[index] = content_block;
+          text = responseMessage.content[index].text;
+        } else
+          throw new Error('Unexpected content block start');
+        break;
+
+      // Append delta text to the current message content
+      case 'content_block_delta':
+        if (responseMessage) {
+          const { index, delta } = JSON.parse(data);
+          if (delta.type !== 'text_delta')
+            throw new Error(`Unexpected content block non-text delta (${delta.type})`);
+          if (responseMessage.content[index] === undefined)
+            throw new Error(`Unexpected content block delta location (${index})`);
+          responseMessage.content[index].text += delta.text;
+          text = delta.text;
+        } else
+          throw new Error('Unexpected content block delta');
+        break;
+
+      // Finalize content block if needed.
+      case 'content_block_stop':
+        if (responseMessage) {
+          const { index } = JSON.parse(data);
+          if (responseMessage.content[index] === undefined)
+            throw new Error(`Unexpected content block end location (${index})`);
+        } else
+          throw new Error('Unexpected content block stop');
+        break;
+
+      // Optionally handle top-level message changes. Example: updating stop_reason
+      case 'message_delta':
+        if (responseMessage) {
+          const { delta } = JSON.parse(data);
+          Object.assign(responseMessage, delta);
+          console.log('Anthropic message delta:', delta);
+        } else
+          throw new Error('Unexpected message delta');
+        break;
+
+      // We can now close the message
+      case 'message_stop':
+        return { text: '', close: true };
+
+      // Occasionaly, the server will send errors, such as {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
+      case 'error':
+        hasErrored = true;
+        const { error } = JSON.parse(data);
+        const errorText = (error.type && error.message) ? `${error.type}: ${error.message}` : safeErrorString(error);
+        return { text: `[Anthropic Server Error] ${errorText}`, close: true };
+
+      default:
+        throw new Error(`Unexpected event name: ${eventName}`);
     }
 
     return { text, close: false };

src/modules/llms/vendors/anthropic/AnthropicSourceSetup.tsx

@@ -1,7 +1,6 @@
 import * as React from 'react';
 
 import { Alert } from '@mui/joy';
-import WarningRoundedIcon from '@mui/icons-material/WarningRounded';
 
 import { FormInputKey } from '~/common/components/forms/FormInputKey';
 import { FormTextField } from '~/common/components/forms/FormTextField';
@@ -40,11 +39,9 @@ export function AnthropicSourceSetup(props: { sourceId: DModelSourceId }) {
 
   return <>
 
-    <Alert variant='soft' color='warning' startDecorator={<WarningRoundedIcon color='warning' />}>
+    <Alert variant='soft' color='success'>
       <div>
-        Note: <strong>Claude-3</strong> API support is being added as the Anthropic API has changed. Please refer to <Link
-        level='body-sm' href='https://github.com/enricoros/big-AGI/issues/443' target='_blank'>issue #443</Link> for
-        updates.
+        Note: <strong>Claude-3</strong> models are now supported.
       </div>
     </Alert>
 
@@ -86,4 +83,5 @@ export function AnthropicSourceSetup(props: { sourceId: DModelSourceId }) {
     {isError && <InlineError error={error} />}
 
   </>;
+  ;
 }