feat (core): support stopSequences setting. (#2371)

.changeset/giant-shirts-relate.md

@@ -0,0 +1,13 @@
+---
+'@ai-sdk/amazon-bedrock': patch
+'@ai-sdk/google-vertex': patch
+'@ai-sdk/anthropic': patch
+'@ai-sdk/provider': patch
+'@ai-sdk/mistral': patch
+'@ai-sdk/cohere': patch
+'@ai-sdk/google': patch
+'@ai-sdk/openai': patch
+'ai': patch
+---
+
+feat (core): support stopSequences setting.

content/docs/03-ai-sdk-core/25-settings.mdx

@@ -21,8 +21,9 @@ const result = await generateText({
 
 <Note>
   Some providers do not support all common settings. If you use a setting with a
-  provider that does not support it, a warning will be included in the AI
-  function result object.
+  provider that does not support it, a warning will be generated. You can check
+  the `warnings` property in the result object to see if any warnings were
+  generated.
 </Note>
 
 ### `maxTokens`
@@ -62,6 +63,13 @@ The frequency penalty affects the likelihood of the model to repeatedly use the
 The value is passed through to the provider. The range depends on the provider and model.
 For most providers, `0` means no penalty.
 
+### `stopSequences`
+
+The stop sequences to use for stopping the text generation.
+
+If set, the model will stop generating text when one of the stop sequences is generated.
+Providers may have limits on the number of stop sequences.
+
 ### `seed`
 
 It is the seed (integer) to use for random sampling.

content/docs/07-reference/ai-sdk-core/01-generate-text.mdx

@@ -302,6 +302,13 @@ console.log(text);
       description:
         'Frequency penalty setting. It affects the likelihood of the model to repeatedly use the same words or phrases. The value is passed through to the provider. The range depends on the provider and model.',
     },
+    {
+      name: 'stopSequences',
+      type: 'string[]',
+      isOptional: true,
+      description:
+        'Sequences that will stop the generation of the text. If the model generates any of these sequences, it will stop generating further text.',
+    },
     {
       name: 'seed',
       type: 'number',

content/docs/07-reference/ai-sdk-core/02-stream-text.mdx

@@ -304,6 +304,13 @@ for await (const textPart of textStream) {
       description:
         'Frequency penalty setting. It affects the likelihood of the model to repeatedly use the same words or phrases. The value is passed through to the provider. The range depends on the provider and model.',
     },
+    {
+      name: 'stopSequences',
+      type: 'string[]',
+      isOptional: true,
+      description:
+        'Sequences that will stop the generation of the text. If the model generates any of these sequences, it will stop generating further text.',
+    },
     {
       name: 'seed',
       type: 'number',

content/docs/07-reference/ai-sdk-rsc/01-stream-ui.mdx

@@ -252,6 +252,13 @@ A helper function to create a streamable UI from LLM providers. This function is
       description:
         'Frequency penalty setting. It affects the likelihood of the model to repeatedly use the same words or phrases. The value is passed through to the provider. The range depends on the provider and model.',
     },
+    {
+      name: 'stopSequences',
+      type: 'string[]',
+      isOptional: true,
+      description:
+        'Sequences that will stop the generation of the text. If the model generates any of these sequences, it will stop generating further text.',
+    },
     {
       name: 'seed',
       type: 'number',

packages/amazon-bedrock/src/bedrock-chat-language-model.ts

@@ -57,6 +57,7 @@ export class BedrockChatLanguageModel implements LanguageModelV1 {
     frequencyPenalty,
     presencePenalty,
     seed,
+    stopSequences,
     headers,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
@@ -101,6 +102,7 @@ export class BedrockChatLanguageModel implements LanguageModelV1 {
         maxTokens,
         temperature,
         topP,
+        stopSequences,
       },
       messages,
     };

packages/anthropic/src/anthropic-messages-language-model.ts

@@ -61,6 +61,7 @@ export class AnthropicMessagesLanguageModel implements LanguageModelV1 {
     frequencyPenalty,
     presencePenalty,
     seed,
+    stopSequences,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
 
@@ -100,6 +101,7 @@ export class AnthropicMessagesLanguageModel implements LanguageModelV1 {
       max_tokens: maxTokens ?? 4096, // 4096: max model output tokens
       temperature,
       top_p: topP,
+      stop_sequences: stopSequences,
 
       // prompt:
       system: messagesPrompt.system,

packages/cohere/src/cohere-chat-language-model.ts

@@ -56,6 +56,7 @@ export class CohereChatLanguageModel implements LanguageModelV1 {
     topP,
     frequencyPenalty,
     presencePenalty,
+    stopSequences,
     seed,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
@@ -78,6 +79,7 @@ export class CohereChatLanguageModel implements LanguageModelV1 {
       temperature,
       p: topP,
       seed,
+      stop_sequences: stopSequences,
 
       // messages:
       chat_history: history,

packages/core/core/generate-object/generate-object.ts

@@ -69,7 +69,7 @@ export async function generateObject<T>({
   headers,
   experimental_telemetry: telemetry,
   ...settings
-}: CallSettings &
+}: Omit<CallSettings, 'stopSequences'> &
   Prompt & {
     /**
 The language model to use.

packages/core/core/generate-object/stream-object.ts

@@ -79,7 +79,7 @@ export async function streamObject<T>({
   headers,
   onFinish,
   ...settings
-}: CallSettings &
+}: Omit<CallSettings, 'stopSequences'> &
   Prompt & {
     /**
 The language model to use.

packages/core/core/generate-text/generate-text.ts

@@ -56,6 +56,8 @@ The value is passed through to the provider. The range depends on the provider a
 @param frequencyPenalty - Frequency penalty setting.
 It affects the likelihood of the model to repeatedly use the same words or phrases.
 The value is passed through to the provider. The range depends on the provider and model.
+@param stopSequences - Stop sequences.
+If set, the model will stop generating text when one of the stop sequences is generated.
 @param seed - The seed (integer) to use for random sampling.
 If set and supported by the model, calls will generate deterministic results.
 

packages/core/core/generate-text/stream-text.ts

@@ -60,6 +60,8 @@ The value is passed through to the provider. The range depends on the provider a
 @param frequencyPenalty - Frequency penalty setting.
 It affects the likelihood of the model to repeatedly use the same words or phrases.
 The value is passed through to the provider. The range depends on the provider and model.
+@param stopSequences - Stop sequences.
+If set, the model will stop generating text when one of the stop sequences is generated.
 @param seed - The seed (integer) to use for random sampling.
 If set and supported by the model, calls will generate deterministic results.
 

packages/core/core/prompt/call-settings.ts

@@ -46,6 +46,13 @@ and 1 (maximum penalty, decrease repetition). 0 means no penalty.
    */
   frequencyPenalty?: number;
 
+  /**
+Stop sequences.
+If set, the model will stop generating text when one of the stop sequences is generated.
+Providers may have limits on the number of stop sequences.
+   */
+  stopSequences?: string[];
+
   /**
 The seed (integer) to use for random sampling. If set and supported
 by the model, calls will generate deterministic results.

packages/core/core/prompt/prepare-call-settings.ts

@@ -10,6 +10,7 @@ export function prepareCallSettings({
   topP,
   presencePenalty,
   frequencyPenalty,
+  stopSequences,
   seed,
   maxRetries,
 }: CallSettings): CallSettings {
@@ -105,6 +106,10 @@ export function prepareCallSettings({
     topP,
     presencePenalty,
     frequencyPenalty,
+    stopSequences:
+      stopSequences != null && stopSequences.length > 0
+        ? stopSequences
+        : undefined,
     seed,
     maxRetries: maxRetries ?? 2,
   };

packages/google-vertex/src/google-vertex-language-model.test.ts

@@ -191,6 +191,7 @@ describe('doGenerate', () => {
       temperature: 0.5,
       maxTokens: 100,
       topP: 0.9,
+      stopSequences: ['abc', 'def'],
     });
 
     expect(mockVertexAI.lastModelParams).toStrictEqual({
@@ -200,6 +201,7 @@ describe('doGenerate', () => {
         temperature: 0.5,
         topK: 0.1,
         topP: 0.9,
+        stopSequences: ['abc', 'def'],
       },
       tools: undefined,
       safetySettings: undefined,

packages/google-vertex/src/google-vertex-language-model.ts

@@ -49,14 +49,15 @@ export class GoogleVertexLanguageModel implements LanguageModelV1 {
   }
 
   private async getArgs({
-    prompt,
     mode,
-    frequencyPenalty,
-    presencePenalty,
-    seed,
+    prompt,
     maxTokens,
     temperature,
     topP,
+    frequencyPenalty,
+    presencePenalty,
+    stopSequences,
+    seed,
     headers,
   }: LanguageModelV1CallOptions) {
     const warnings: LanguageModelV1CallWarning[] = [];
@@ -97,6 +98,7 @@ export class GoogleVertexLanguageModel implements LanguageModelV1 {
       maxOutputTokens: maxTokens,
       temperature,
       topP,
+      stopSequences,
     };
 
     const type = mode.type;

packages/google/src/google-generative-ai-language-model.ts

@@ -61,6 +61,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV1 {
     topP,
     frequencyPenalty,
     presencePenalty,
+    stopSequences,
     seed,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
@@ -96,6 +97,7 @@ export class GoogleGenerativeAILanguageModel implements LanguageModelV1 {
       maxOutputTokens: maxTokens,
       temperature,
       topP,
+      stopSequences,
     };
 
     const { contents, systemInstruction } =

packages/mistral/src/mistral-chat-language-model.ts

@@ -60,6 +60,7 @@ export class MistralChatLanguageModel implements LanguageModelV1 {
     topP,
     frequencyPenalty,
     presencePenalty,
+    stopSequences,
     seed,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
@@ -80,6 +81,13 @@ export class MistralChatLanguageModel implements LanguageModelV1 {
       });
     }
 
+    if (stopSequences != null) {
+      warnings.push({
+        type: 'unsupported-setting',
+        setting: 'stopSequences',
+      });
+    }
+
     const baseArgs = {
       // model id:
       model: this.modelId,

packages/openai/src/openai-chat-language-model.ts

@@ -64,6 +64,7 @@ export class OpenAIChatLanguageModel implements LanguageModelV1 {
     topP,
     frequencyPenalty,
     presencePenalty,
+    stopSequences,
     seed,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
@@ -96,6 +97,7 @@ export class OpenAIChatLanguageModel implements LanguageModelV1 {
       top_p: topP,
       frequency_penalty: frequencyPenalty,
       presence_penalty: presencePenalty,
+      stop: stopSequences,
       seed,
 
       // messages:

packages/openai/src/openai-completion-language-model.ts

@@ -65,13 +65,16 @@ export class OpenAICompletionLanguageModel implements LanguageModelV1 {
     topP,
     frequencyPenalty,
     presencePenalty,
+    stopSequences: userStopSequences,
     seed,
   }: Parameters<LanguageModelV1['doGenerate']>[0]) {
     const type = mode.type;
 
     const { prompt: completionPrompt, stopSequences } =
       convertToOpenAICompletionPrompt({ prompt, inputFormat });
 
+    const stop = [...(stopSequences ?? []), ...(userStopSequences ?? [])];
+
     const baseArgs = {
       // model id:
       model: this.modelId,
@@ -102,7 +105,7 @@ export class OpenAICompletionLanguageModel implements LanguageModelV1 {
       prompt: completionPrompt,
 
       // stop sequences:
-      stop: stopSequences,
+      stop: stop.length > 0 ? stop : undefined,
     };
 
     switch (type) {

packages/provider/src/language-model/v1/language-model-v1-call-settings.ts

@@ -11,6 +11,13 @@ It is recommended to set either `temperature` or `topP`, but not both.
    */
   temperature?: number;
 
+  /**
+Stop sequences. 
+If set, the model will stop generating text when one of the stop sequences is generated.
+Providers may have limits on the number of stop sequences.
+   */
+  stopSequences?: string[];
+
   /**
 Nucleus sampling.