[GRO-189] adding evals to mcp server, ci evals

.github/workflows/ci.yml

@@ -1,27 +1,46 @@
 name: CI
 
 on:
-  push:
-    branches: [main]
   pull_request:
     branches: [main]
+  push:
+    branches: [main]
 
 jobs:
-  lint:
+  test:
+    name: Test and Lint
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Install pnpm
         uses: pnpm/action-setup@v4
-      - name: Use Node.js 22
+
+      - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version: "22"
           cache: "pnpm"
+
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
-      - run: pnpm run build
-      - name: Run ESLint
-        run: pnpm run lint
-      - name: Ensure no changes
-        run: git diff --exit-code
+
+      - name: Run linting
+        run: pnpm lint
+
+      - name: Check formatting
+        run: pnpm format
+
+      - name: Build project
+        run: pnpm build
+
+      - name: Run evaluation tests
+        env:
+          BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
+          BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+        run: |
+          pnpm evals

.husky/pre-commit

@@ -1,2 +1,4 @@
 #!/bin/sh
+pnpm install
+pnpm build
 pnpm pre-commit

README.md

@@ -105,7 +105,7 @@ git clone https://github.com/browserbase/mcp-server-browserbase.git
 cd mcp-server-browserbase
 
 # Install the dependencies and build the project
-pnpm install && pnpm build
+npm install && npm run build
 ```
 
 Then in your MCP Config JSON run the server. To run locally we can use STDIO or self-host SHTTP.

evals/mcp-eval-basic.config.json

@@ -0,0 +1,142 @@
+{
+  "passThreshold": 0.7,
+  "server": {
+    "transport": "stdio",
+    "command": "node",
+    "args": ["./cli.js"],
+    "env": {
+      "BROWSERBASE_API_KEY": "${BROWSERBASE_API_KEY}",
+      "BROWSERBASE_PROJECT_ID": "${BROWSERBASE_PROJECT_ID}",
+      "GEMINI_API_KEY": "${GEMINI_API_KEY}"
+    }
+  },
+  "timeout": 180000,
+  "llmJudge": false,
+  "workflows": [
+    {
+      "name": "basic-navigation-test",
+      "description": "Test basic browser navigation functionality",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to https://example.com, and close the session",
+          "expectedState": "closed"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "search-and-extract-test",
+      "description": "Test navigation, search interaction, and data extraction",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to https://example.com, extract the page title, and close the session",
+          "expectedState": "Example Domain"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_stagehand_extract",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "observe-and-interact-test",
+      "description": "Test element observation and interaction capabilities",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to https://example.com, observe the page elements, and close the session",
+          "expectedState": "closed"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_stagehand_observe",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "screenshot-test",
+      "description": "Test screenshot functionality",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to https://example.com, take a screenshot, and close the session",
+          "expectedState": "closed"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_screenshot",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "multi-session-test",
+      "description": "Test multi-session browser management",
+      "steps": [
+        {
+          "user": "Create a multi-session browser named 'test-session', list all sessions, navigate to https://example.com in that session, and close the session",
+          "expectedState": "closed"
+        }
+      ],
+      "expectTools": [
+        "multi_browserbase_stagehand_session_create",
+        "multi_browserbase_stagehand_session_list",
+        "multi_browserbase_stagehand_navigate_session",
+        "multi_browserbase_stagehand_session_close"
+      ]
+    },
+    {
+      "name": "form-interaction-test",
+      "description": "Test form filling and submission capabilities",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to https://httpbin.org/forms/post, fill in the customer name field with 'TestUser', and close the session",
+          "expectedState": "closed"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_stagehand_act",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "error-handling-test",
+      "description": "Test error handling for invalid operations",
+      "steps": [
+        {
+          "user": "Create a browser session and try to navigate to an invalid URL like 'invalid-url-test'",
+          "expectedState": "error"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate"
+      ]
+    },
+    {
+      "name": "url-retrieval-test",
+      "description": "Test URL retrieval functionality",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to https://example.com, get the current URL to verify navigation, and close the session",
+          "expectedState": "https://example.com"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_stagehand_get_url",
+        "browserbase_session_close"
+      ]
+    }
+  ]
+}

evals/mcp-eval-minimal.config.json

@@ -0,0 +1,95 @@
+{
+  "passThreshold": 0.7,
+  "server": {
+    "transport": "stdio",
+    "command": "node",
+    "args": ["./cli.js"],
+    "env": {
+      "BROWSERBASE_API_KEY": "${BROWSERBASE_API_KEY}",
+      "BROWSERBASE_PROJECT_ID": "${BROWSERBASE_PROJECT_ID}",
+      "GEMINI_API_KEY": "${GEMINI_API_KEY}"
+    }
+  },
+  "timeout": 60000,
+  "llmJudge": false,
+  "workflows": [
+    {
+      "name": "smoke-test-navigation",
+      "description": "Quick test to verify basic navigation works",
+      "steps": [
+        {
+          "user": "Open a browser and go to example.org",
+          "expectedState": "session created"
+        },
+        {
+          "user": "Close the browser",
+          "expectedState": "closed successfully via Stagehand"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "smoke-test-extraction",
+      "description": "Quick test to verify data extraction works",
+      "steps": [
+        {
+          "user": "Navigate to example.org and extract the page title",
+          "expectedState": "Example Domain"
+        },
+        {
+          "user": "Close the session",
+          "expectedState": "closed successfully via Stagehand"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_stagehand_extract",
+        "browserbase_session_close"
+      ]
+    },
+    {
+      "name": "smoke-test-multi-session",
+      "description": "Quick test to verify multi-session functionality",
+      "steps": [
+        {
+          "user": "Create a browser session named 'test-session'",
+          "expectedState": "session created"
+        },
+        {
+          "user": "List active sessions",
+          "expectedState": "test-session"
+        },
+        {
+          "user": "Close the test session",
+          "expectedState": "closed session"
+        }
+      ],
+      "expectTools": [
+        "multi_browserbase_stagehand_session_create",
+        "multi_browserbase_stagehand_session_list",
+        "multi_browserbase_stagehand_session_close"
+      ]
+    },
+    {
+      "name": "smoke-test-url-tools",
+      "description": "Quick test to verify URL retrieval tools work",
+      "steps": [
+        {
+          "user": "Create a browser session, navigate to example.org, get the current URL, and close the session",
+          "expectedState": "example.org"
+        }
+      ],
+      "expectTools": [
+        "browserbase_session_create",
+        "browserbase_stagehand_navigate",
+        "browserbase_stagehand_get_url",
+        "browserbase_session_close"
+      ]
+    }
+  ]
+}