openai[minor],core[minor]: Add support for passing strict in openai tools

langchain-core/src/language_models/base.ts

@@ -233,6 +233,15 @@ export interface FunctionDefinition {
    * how to call the function.
    */
   description?: string;
+
+  /**
+   * Whether to enable strict schema adherence when generating the function call. If
+   * set to true, the model will follow the exact schema defined in the `parameters`
+   * field. Only a subset of JSON Schema is supported when `strict` is `true`. Learn
+   * more about Structured Outputs in the
+   * [function calling guide](https://platform.openai.com/docs/guides/function-calling).
+   */
+  strict?: boolean;
 }
 
 export interface ToolDefinition {

langchain-core/src/utils/function_calling.ts

@@ -34,14 +34,29 @@ export function convertToOpenAIFunction(
  */
 export function convertToOpenAITool(
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  tool: StructuredToolInterface | Record<string, any> | RunnableToolLike
+  tool: StructuredToolInterface | Record<string, any> | RunnableToolLike,
+  fields?: {
+    /**
+     * If `true`, model output is guaranteed to exactly match the JSON Schema
+     * provided in the function definition.
+     */
+    strict?: boolean;
+  }
 ): ToolDefinition {
+  let toolDef: ToolDefinition | undefined;
   if (isStructuredTool(tool) || isRunnableToolLike(tool)) {
-    return {
+    toolDef = {
       type: "function",
       function: convertToOpenAIFunction(tool),
     };
+  } else {
+    toolDef = tool as ToolDefinition;
+  }
+
+  if (fields?.strict !== undefined) {
+    toolDef.function.strict = fields.strict;
   }
+
   return tool as ToolDefinition;
 }
 

libs/langchain-openai/package.json

@@ -37,7 +37,7 @@
   "dependencies": {
     "@langchain/core": ">=0.2.16 <0.3.0",
     "js-tiktoken": "^1.0.12",
-    "openai": "^4.49.1",
+    "openai": "^4.55.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.3"
   },

libs/langchain-openai/src/chat_models.ts

@@ -1,4 +1,4 @@
-import { type ClientOptions, OpenAI as OpenAIClient } from "openai";
+import { type ClientOptions, OpenAI as OpenAIClient, } from "openai";
 
 import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
 import {
@@ -299,6 +299,16 @@ export interface ChatOpenAICallOptions
    * call multiple tools in one response.
    */
   parallel_tool_calls?: boolean;
+  /**
+   * If `true`, model output is guaranteed to exactly match the JSON Schema
+   * provided in the tool definition.
+   * Enabled by default for `"gpt-"` models.
+   */
+  strict?: boolean;
+}
+
+export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<AzureOpenAIInput>, BaseChatModelParams {
+  configuration?: ClientOptions & LegacyOpenAIInput;
 }
 
 /**
@@ -441,12 +451,15 @@ export class ChatOpenAI<
 
   protected clientConfig: ClientOptions;
 
+  /**
+   * Whether the model supports the 'strict' argument when passing in tools.
+   * Defaults to `true` if `modelName`/`model` starts with 'gpt-' otherwise
+   * defaults to `false`.
+   */
+  supportsStrictToolCalling?: boolean;
+
   constructor(
-    fields?: Partial<OpenAIChatInput> &
-      Partial<AzureOpenAIInput> &
-      BaseChatModelParams & {
-        configuration?: ClientOptions & LegacyOpenAIInput;
-      },
+    fields?: ChatOpenAIFields,
     /** @deprecated */
     configuration?: ClientOptions & LegacyOpenAIInput
   ) {
@@ -541,6 +554,12 @@ export class ChatOpenAI<
       ...configuration,
       ...fields?.configuration,
     };
+
+    // Assume only "gpt-..." models support strict tool calling as of 08/06/24.
+    this.supportsStrictToolCalling = 
+    fields?.supportsStrictToolCalling !== undefined
+      ? fields.supportsStrictToolCalling
+      : this.modelName.startsWith("gpt-");
   }
 
   getLsParams(options: this["ParsedCallOptions"]): LangSmithParams {
@@ -563,8 +582,9 @@ export class ChatOpenAI<
     )[],
     kwargs?: Partial<CallOptions>
   ): Runnable<BaseLanguageModelInput, AIMessageChunk, CallOptions> {
+    const strict = kwargs?.strict !== undefined ? kwargs.strict : this.supportsStrictToolCalling;
     return this.bind({
-      tools: tools.map(convertToOpenAITool),
+      tools: tools.map((tool) => convertToOpenAITool(tool, { strict })),
       ...kwargs,
     } as Partial<CallOptions>);
   }
@@ -578,6 +598,7 @@ export class ChatOpenAI<
       streaming?: boolean;
     }
   ): Omit<OpenAIClient.Chat.ChatCompletionCreateParams, "messages"> {
+    const strict = options?.strict !== undefined ? options.strict : this.supportsStrictToolCalling;
     function isStructuredToolArray(
       tools?: unknown[]
     ): tools is StructuredToolInterface[] {
@@ -615,7 +636,7 @@ export class ChatOpenAI<
       functions: options?.functions,
       function_call: options?.function_call,
       tools: isStructuredToolArray(options?.tools)
-        ? options?.tools.map(convertToOpenAITool)
+        ? options?.tools.map((tool) => convertToOpenAITool(tool, { strict }))
         : options?.tools,
       tool_choice: formatToOpenAIToolChoice(options?.tool_choice),
       response_format: options?.response_format,

libs/langchain-openai/src/tests/chat_models.test.ts

@@ -0,0 +1,212 @@
+import { z } from "zod";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { it, expect, describe, beforeAll, afterAll, jest } from "@jest/globals";
+import { ChatOpenAI } from "../chat_models.js";
+
+
+describe("strict tool calling", () => {
+  const weatherTool = {
+    type: "function" as const,
+    function: {
+      name: "get_current_weather",
+      description: "Get the current weather in a location",
+      parameters: zodToJsonSchema(z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }))
+    }
+  }
+
+  // Store the original value of LANGCHAIN_TRACING_V2
+  let oldLangChainTracingValue: string | undefined;
+  // Before all tests, save the current LANGCHAIN_TRACING_V2 value
+  beforeAll(() => {
+    oldLangChainTracingValue = process.env.LANGCHAIN_TRACING_V2;
+  })
+  // After all tests, restore the original LANGCHAIN_TRACING_V2 value
+  afterAll(() => {
+    if (oldLangChainTracingValue !== undefined) {
+      process.env.LANGCHAIN_TRACING_V2 = oldLangChainTracingValue;
+    } else {
+      // If it was undefined, remove the environment variable
+      delete process.env.LANGCHAIN_TRACING_V2;
+    }
+  })
+
+  it("Can accept strict as a call arg via .bindTools", async () => {
+    const mockFetch = jest.fn<(url: any, init?: any) => Promise<any>>();
+    mockFetch.mockImplementation((url, options): Promise<any> => {
+      // Store the request details for later inspection
+      mockFetch.mock.calls.push({ url, options } as any);
+
+      // Return a mock response
+      return Promise.resolve({
+        ok: true,
+        json: () => Promise.resolve({}),
+      }) as Promise<any>;
+    });
+
+    const model = new ChatOpenAI({
+      model: "gpt-4",
+      configuration: {
+        fetch: mockFetch,
+      },
+      maxRetries: 0,
+    });
+
+    const modelWithTools = model.bindTools([weatherTool], { strict: true });
+
+    // This will fail since we're not returning a valid response in our mocked fetch function.
+    await expect(modelWithTools.invoke("What's the weather like?")).rejects.toThrow();
+
+    expect(mockFetch).toHaveBeenCalled();
+    const [_url, options] = mockFetch.mock.calls[0];
+
+    if (options && options.body) {
+      expect(JSON.parse(options.body).tools).toEqual([expect.objectContaining({
+        type: "function",
+        function: {
+          ...weatherTool.function,
+          // This should be added to the function call because `strict` was passed to `bindTools`
+          strict: true,
+        }
+      })]);
+    } else {
+      throw new Error("Body not found in request.")
+    }
+  });
+
+  it("Can accept strict as a call arg via .bind", async () => {
+    const mockFetch = jest.fn<(url: any, init?: any) => Promise<any>>();
+    mockFetch.mockImplementation((url, options): Promise<any> => {
+      // Store the request details for later inspection
+      mockFetch.mock.calls.push({ url, options } as any);
+
+      // Return a mock response
+      return Promise.resolve({
+        ok: true,
+        json: () => Promise.resolve({}),
+      }) as Promise<any>;
+    });
+
+    const model = new ChatOpenAI({
+      model: "gpt-4",
+      configuration: {
+        fetch: mockFetch,
+      },
+      maxRetries: 0,
+    });
+
+    const modelWithTools = model.bind({
+      tools: [weatherTool],
+      strict: true
+    });
+
+    // This will fail since we're not returning a valid response in our mocked fetch function.
+    await expect(modelWithTools.invoke("What's the weather like?")).rejects.toThrow();
+
+    expect(mockFetch).toHaveBeenCalled();
+    const [_url, options] = mockFetch.mock.calls[0];
+
+    if (options && options.body) {
+      expect(JSON.parse(options.body).tools).toEqual([expect.objectContaining({
+        type: "function",
+        function: {
+          ...weatherTool.function,
+          // This should be added to the function call because `strict` was passed to `bind`
+          strict: true,
+        }
+      })]);
+    } else {
+      throw new Error("Body not found in request.")
+    }
+  });
+
+  it("Sets strict to true if the model name starts with 'gpt-'", async () => {
+    const mockFetch = jest.fn<(url: any, init?: any) => Promise<any>>();
+    mockFetch.mockImplementation((url, options): Promise<any> => {
+      // Store the request details for later inspection
+      mockFetch.mock.calls.push({ url, options } as any);
+
+      // Return a mock response
+      return Promise.resolve({
+        ok: true,
+        json: () => Promise.resolve({}),
+      }) as Promise<any>;
+    });
+
+    const model = new ChatOpenAI({
+      model: "gpt-4",
+      configuration: {
+        fetch: mockFetch,
+      },
+      maxRetries: 0,
+    });
+
+    // Do NOT pass `strict` here since we're checking that it's set to true by default
+    const modelWithTools = model.bindTools([weatherTool]);
+
+    // This will fail since we're not returning a valid response in our mocked fetch function.
+    await expect(modelWithTools.invoke("What's the weather like?")).rejects.toThrow();
+
+    expect(mockFetch).toHaveBeenCalled();
+    const [_url, options] = mockFetch.mock.calls[0];
+
+    if (options && options.body) {
+      expect(JSON.parse(options.body).tools).toEqual([expect.objectContaining({
+        type: "function",
+        function: {
+          ...weatherTool.function,
+          // This should be added to the function call because `strict` was passed to `bind`
+          strict: true,
+        }
+      })]);
+    } else {
+      throw new Error("Body not found in request.")
+    }
+  });
+
+  it("Strict is false if supportsStrictToolCalling is false", async () => {
+    const mockFetch = jest.fn<(url: any, init?: any) => Promise<any>>();
+    mockFetch.mockImplementation((url, options): Promise<any> => {
+      // Store the request details for later inspection
+      mockFetch.mock.calls.push({ url, options } as any);
+
+      // Return a mock response
+      return Promise.resolve({
+        ok: true,
+        json: () => Promise.resolve({}),
+      }) as Promise<any>;
+    });
+
+    const model = new ChatOpenAI({
+      model: "gpt-4",
+      configuration: {
+        fetch: mockFetch,
+      },
+      maxRetries: 0,
+      supportsStrictToolCalling: false,
+    });
+
+    // Do NOT pass `strict` here since we're checking that it's set to true by default
+    const modelWithTools = model.bindTools([weatherTool]);
+
+    // This will fail since we're not returning a valid response in our mocked fetch function.
+    await expect(modelWithTools.invoke("What's the weather like?")).rejects.toThrow();
+
+    expect(mockFetch).toHaveBeenCalled();
+    const [_url, options] = mockFetch.mock.calls[0];
+
+    if (options && options.body) {
+      expect(JSON.parse(options.body).tools).toEqual([expect.objectContaining({
+        type: "function",
+        function: {
+          ...weatherTool.function,
+          // This should be added to the function call because `strict` was passed to `bind`
+          strict: false,
+        }
+      })]);
+    } else {
+      throw new Error("Body not found in request.")
+    }
+  });
+})

libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts

@@ -3,6 +3,7 @@ import { zodToJsonSchema } from "zod-to-json-schema";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { AIMessage } from "@langchain/core/messages";
 import { ChatOpenAI } from "../chat_models.js";
+import { test, expect } from "@jest/globals";
 
 test("withStructuredOutput zod schema function calling", async () => {
   const model = new ChatOpenAI({

libs/langchain-openai/src/types.ts

@@ -155,6 +155,13 @@ export interface OpenAIChatInput extends OpenAIBaseInput {
    * Currently in experimental beta.
    */
   __includeRawResponse?: boolean;
+
+  /**
+   * Whether the model supports the 'strict' argument when passing in tools.
+   * Defaults to `true` if `modelName`/`model` starts with 'gpt-' otherwise
+   * defaults to `false`.
+   */
+  supportsStrictToolCalling?: boolean;
 }
 
 export declare interface AzureOpenAIInput {