feat(community): Stagehand tools integration

docs/core_docs/docs/integrations/tools/stagehand.mdx

@@ -0,0 +1,139 @@
+---
+sidebar_label: Stagehand AI Web Automation Toolkit
+hide_table_of_contents: true
+---
+
+import CodeBlock from "@theme/CodeBlock";
+import Example from "@examples/agents/stagehand_ai_web_browser.ts";
+import { StagehandToolkit } from "@langchain/community/agents/toolkits/stagehand";
+
+# Stagehand Toolkit
+
+The Stagehand Toolkit equips your AI agent with the following capabilities:
+
+- **navigate()**: Navigate to a specific URL.
+- **act()**: Perform browser automation actions like clicking, typing, and navigation.
+- **extract()**: Extract structured data from web pages using Zod schemas.
+- **observe()**: Get a list of possible actions and elements on the current page.
+
+## Setup
+
+1. Install the required packages:
+
+```bash
+npm install @langchain/langgraph @langchain/community @langchain/core
+```
+
+2. Create a Stagehand Instance
+   If you plan to run the browser locally, you'll also need to install Playwright's browser dependencies.
+
+```bash
+npx playwright install
+```
+
+3. Set up your model provider credentials:
+
+For OpenAI:
+
+```bash
+export OPENAI_API_KEY="your-openai-api-key"
+```
+
+For Anthropic:
+
+```bash
+export ANTHROPIC_API_KEY="your-anthropic-api-key"
+```
+
+## Usage, Standalone, Local Browser
+
+```typescript
+import { StagehandToolkit } from "langchain/community/agents/toolkits/stagehand";
+import { ChatOpenAI } from "@langchain/openai";
+import { Stagehand } from "@browserbasehq/stagehand";
+
+// Specify your Browserbase credentials.
+process.env.BROWSERBASE_API_KEY = "";
+process.env.BROWSERBASE_PROJECT_ID = "";
+
+// Specify OpenAI API key.
+process.env.OPENAI_API_KEY = "";
+
+const stagehand = new Stagehand({
+  env: "LOCAL",
+  headless: false,
+  verbose: 2,
+  debugDom: true,
+  enableCaching: false,
+});
+
+// Create a Stagehand Toolkit with all the available actions from the Stagehand.
+const stagehandToolkit = await StagehandToolkit.fromStagehand(stagehand);
+
+const navigateTool = stagehandToolkit.tools.find(
+  (t) => t.name === "stagehand_navigate"
+);
+if (!navigateTool) {
+  throw new Error("Navigate tool not found");
+}
+await navigateTool.invoke("https://www.google.com");
+
+const actionTool = stagehandToolkit.tools.find(
+  (t) => t.name === "stagehand_act"
+);
+if (!actionTool) {
+  throw new Error("Action tool not found");
+}
+await actionTool.invoke('Search for "OpenAI"');
+
+const observeTool = stagehandToolkit.tools.find(
+  (t) => t.name === "stagehand_observe"
+);
+if (!observeTool) {
+  throw new Error("Observe tool not found");
+}
+const result = await observeTool.invoke(
+  "What actions can be performed on the current page?"
+);
+const observations = JSON.parse(result);
+
+// Handle observations as needed
+console.log(observations);
+
+const currentUrl = stagehand.page.url();
+expect(currentUrl).toContain("google.com/search?q=OpenAI");
+```
+
+## Usage with LangGraph Agents
+
+<CodeBlock language="typescript">{Example}</CodeBlock>
+
+## Usage on Browserbase - remote headless browser
+
+If you want to run the browser remotely, you can use the Browserbase platform.
+
+You need to set the `BROWSERBASE_API_KEY` environment variable to your Browserbase API key.
+
+```bash
+export BROWSERBASE_API_KEY="your-browserbase-api-key"
+```
+
+You also need to set `BROWSERBASE_PROJECT_ID` to your Browserbase project ID.
+
+```bash
+export BROWSERBASE_PROJECT_ID="your-browserbase-project-id"
+```
+
+Then initialize the Stagehand instance with the `BROWSERBASE` environment.
+
+```typescript
+const stagehand = new Stagehand({
+  env: "BROWSERBASE",
+});
+```
+
+## Related
+
+- Tool [conceptual guide](/docs/concepts/tools)
+- Tool [how-to guides](/docs/how_to/#tools)
+- [Stagehand Documentation](https://github.com/browserbase/stagehand#readme)

examples/package.json

@@ -25,6 +25,7 @@
   "license": "MIT",
   "dependencies": {
     "@azure/identity": "^4.2.1",
+    "@browserbasehq/stagehand": "^1.3.0",
     "@clickhouse/client": "^0.2.5",
     "@elastic/elasticsearch": "^8.4.0",
     "@faker-js/faker": "^8.4.1",

examples/src/agents/stagehand_ai_web_browser.ts

@@ -0,0 +1,87 @@
+import { Stagehand } from "@browserbasehq/stagehand";
+import {
+  StagehandActTool,
+  StagehandNavigateTool,
+} from "@langchain/community/agents/toolkits/stagehand";
+import { ChatOpenAI } from "@langchain/openai";
+import { createReactAgent } from "@langchain/langgraph/prebuilt";
+
+async function main() {
+  // Initialize Stagehand once and pass it to the tools
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    enableCaching: true,
+  });
+
+  const actTool = new StagehandActTool(stagehand);
+  const navigateTool = new StagehandNavigateTool(stagehand);
+
+  // Initialize the model
+  const model = new ChatOpenAI({
+    modelName: "gpt-4",
+    temperature: 0,
+  });
+
+  // Create the agent using langgraph
+  const agent = createReactAgent({
+    llm: model,
+    tools: [actTool, navigateTool],
+  });
+
+  // Execute the agent using streams
+  const inputs1 = {
+    messages: [
+      {
+        role: "user",
+        content: "Navigate to https://www.google.com",
+      },
+    ],
+  };
+
+  const stream1 = await agent.stream(inputs1, {
+    streamMode: "values",
+  });
+
+  for await (const { messages } of stream1) {
+    const msg =
+      messages && messages.length > 0
+        ? messages[messages.length - 1]
+        : undefined;
+    if (msg?.content) {
+      console.log(msg.content);
+    } else if (msg?.tool_calls && msg.tool_calls.length > 0) {
+      console.log(msg.tool_calls);
+    } else {
+      console.log(msg);
+    }
+  }
+
+  const inputs2 = {
+    messages: [
+      {
+        role: "user",
+        content: "Search for 'OpenAI'",
+      },
+    ],
+  };
+
+  const stream2 = await agent.stream(inputs2, {
+    streamMode: "values",
+  });
+
+  for await (const { messages } of stream2) {
+    const msg =
+      messages && messages.length > 0
+        ? messages[messages.length - 1]
+        : undefined;
+    if (msg?.content) {
+      console.log(msg.content);
+    } else if (msg?.tool_calls && msg.tool_calls.length > 0) {
+      console.log(msg.tool_calls);
+    } else {
+      console.log(msg);
+    }
+  }
+}
+
+main();

libs/langchain-community/langchain.config.js

@@ -65,6 +65,7 @@ export const config = {
     "agents/toolkits/aws_sfn": "agents/toolkits/aws_sfn",
     "agents/toolkits/base": "agents/toolkits/base",
     "agents/toolkits/connery": "agents/toolkits/connery/index",
+    "agents/toolkits/stagehand": "agents/toolkits/stagehand",
     // embeddings
     "embeddings/alibaba_tongyi": "embeddings/alibaba_tongyi",
     "embeddings/baidu_qianfan": "embeddings/baidu_qianfan",
@@ -336,6 +337,7 @@ export const config = {
     "tools/gmail",
     "tools/google_calendar",
     "agents/toolkits/aws_sfn",
+    "agents/toolkits/stagehand",
     "callbacks/handlers/llmonitor",
     "callbacks/handlers/lunary",
     "callbacks/handlers/upstash_ratelimit",

libs/langchain-community/package.json

@@ -35,6 +35,7 @@
   "author": "LangChain",
   "license": "MIT",
   "dependencies": {
+    "@langchain/langgraph": "^0.2.22",
     "@langchain/openai": ">=0.2.0 <0.4.0",
     "binary-extensions": "^2.2.0",
     "expr-eval": "^2.0.2",
@@ -62,6 +63,7 @@
     "@azure/search-documents": "^12.0.0",
     "@azure/storage-blob": "^12.15.0",
     "@browserbasehq/sdk": "^1.1.5",
+    "@browserbasehq/stagehand": "^1.0.0",
     "@clickhouse/client": "^0.2.5",
     "@cloudflare/ai": "1.0.12",
     "@cloudflare/workers-types": "^4.20230922.0",
@@ -92,6 +94,7 @@
     "@notionhq/client": "^2.2.10",
     "@opensearch-project/opensearch": "^2.2.0",
     "@planetscale/database": "^1.8.0",
+    "@playwright/test": "^1.48.2",
     "@premai/prem-sdk": "^0.3.25",
     "@qdrant/js-client-rest": "^1.8.2",
     "@raycast/api": "^1.83.1",
@@ -190,6 +193,7 @@
     "node-llama-cpp": "3.1.1",
     "notion-to-md": "^3.1.0",
     "officeparser": "^4.0.4",
+    "openai": "^4.73.0",
     "pdf-parse": "1.1.1",
     "pg": "^8.11.0",
     "pg-copy-streams": "^6.0.5",
@@ -231,6 +235,7 @@
     "@azure/search-documents": "^12.0.0",
     "@azure/storage-blob": "^12.15.0",
     "@browserbasehq/sdk": "*",
+    "@browserbasehq/stagehand": "^1.0.0",
     "@clickhouse/client": "^0.2.5",
     "@cloudflare/ai": "*",
     "@datastax/astra-db-ts": "^1.0.0",
@@ -319,6 +324,7 @@
     "neo4j-driver": "*",
     "notion-to-md": "^3.1.0",
     "officeparser": "^4.0.4",
+    "openai": "*",
     "pdf-parse": "1.1.1",
     "pg": "^8.11.0",
     "pg-copy-streams": "^6.0.5",