langchain[patch]: Allow custom handling of tool runtime errors in leg…

…acy agent executor (#6687)
langchain-ai · Sep 4, 2024 · 9c131b3 · 9c131b3
1 parent 40b85da
commit 9c131b3
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 4 deletions.
diff --git a/langchain/package.json b/langchain/package.json
@@ -584,9 +584,9 @@
     "prepack": "yarn build",
     "release": "release-it --only-version --config .release-it.json",
     "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000 --maxWorkers=50%",
-    "test:watch": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
-    "test:integration": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%",
-    "test:single": "yarn run build:deps && NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
+    "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
+    "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 100000 --maxWorkers=50%",
+    "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 100000",
     "format": "prettier --config .prettierrc --write \"src\"",
     "format:check": "prettier --config .prettierrc --check \"src\""
   },

diff --git a/langchain/src/agents/executor.ts b/langchain/src/agents/executor.ts
@@ -311,6 +311,7 @@ export interface AgentExecutorInput extends ChainInputs {
     | boolean
     | string
     | ((e: OutputParserException | ToolInputParsingException) => string);
+  handleToolRuntimeErrors?: (e: Error) => string;
 }
 
 // TODO: Type properly with { intermediateSteps?: AgentStep[] };
@@ -386,6 +387,8 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
     | ((e: OutputParserException | ToolInputParsingException) => string) =
     false;
 
+  handleToolRuntimeErrors?: (e: Error) => string;
+
   get inputKeys() {
     return this.agent.inputKeys;
   }
@@ -427,6 +430,7 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
     this.tools = input.tools;
     this.handleParsingErrors =
       input.handleParsingErrors ?? this.handleParsingErrors;
+    this.handleToolRuntimeErrors = input.handleToolRuntimeErrors;
     this.returnOnlyOutputs = returnOnlyOutputs;
     if (this.agent._agentActionType() === "multi") {
       for (const tool of this.tools) {
@@ -565,7 +569,8 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
                 "Received unsupported non-string response from tool call."
               );
             }
-          } catch (e) {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          } catch (e: any) {
             // eslint-disable-next-line no-instanceof/no-instanceof
             if (e instanceof ToolInputParsingException) {
               if (this.handleParsingErrors === true) {
@@ -583,6 +588,8 @@ export class AgentExecutor extends BaseChain<ChainValues, AgentExecutorOutput> {
                 runManager?.getChild()
               );
               return { action, observation: observation ?? "" };
+            } else if (this.handleToolRuntimeErrors !== undefined) {
+              observation = this.handleToolRuntimeErrors(e);
             }
           }
 

diff --git a/langchain/src/agents/tests/create_openai_tools_agent.int.test.ts b/langchain/src/agents/tests/create_openai_tools_agent.int.test.ts
@@ -4,6 +4,8 @@ import type { ChatPromptTemplate } from "@langchain/core/prompts";
 import { RunnableLambda } from "@langchain/core/runnables";
 import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
 import { AsyncLocalStorageProviderSingleton } from "@langchain/core/singletons";
+import { tool } from "@langchain/core/tools";
+import { z } from "zod";
 import { AsyncLocalStorage } from "async_hooks";
 import { TavilySearchResults } from "../../util/testing/tools/tavily_search.js";
 import { pull } from "../../hub.js";
@@ -40,6 +42,45 @@ test("createOpenAIToolsAgent works", async () => {
   expect(result.output.length).toBeGreaterThan(10);
 });
 
+test("createOpenAIToolsAgent handles errors", async () => {
+  const errorTools = [
+    tool(
+      async () => {
+        const error = new Error("Error getting search results");
+        throw error;
+      },
+      {
+        name: "search-results",
+        schema: z.object({
+          query: z.string(),
+        }),
+        description: "Searches the web",
+      }
+    ),
+  ];
+  const prompt = await pull<ChatPromptTemplate>("hwchase17/openai-tools-agent");
+  const llm = new ChatOpenAI({
+    modelName: "gpt-3.5-turbo-1106",
+    temperature: 0,
+  });
+  const agent = await createOpenAIToolsAgent({
+    llm,
+    tools: errorTools,
+    prompt,
+  });
+  const agentExecutor = new AgentExecutor({
+    agent,
+    tools: errorTools,
+    handleToolRuntimeErrors: (e) => {
+      throw e;
+    },
+  });
+  const input = "what is LangChain?";
+  await expect(agentExecutor.invoke({ input })).rejects.toThrowError(
+    "Error getting search results"
+  );
+});
+
 test.skip("createOpenAIToolsAgent tracing works when it is nested in a lambda", async () => {
   AsyncLocalStorageProviderSingleton.initializeGlobalInstance(
     new AsyncLocalStorage()