Skip to content

Commit ef38ccf

Browse files
monadoidmonadoid
authored
Add minimal contract test for public api (#1255)
# why We want to be able to see what the public API shape looks like - which top level functions are exported. # what changed Added vitest and a minimal test file that checks which functions are exported at the root level. If new functions are exported or removed, this test will catch that. I also added one example of drilling down and testing the parameters of StagehandMetrics — in the future we might want to do this for all exports. Curious if this surfaces and exports that don't need to be exported or not. # test plan Just added tests. They run in the CI pipeline or can be run with `pnpm --filter @browserbasehq/stagehand run test:vitest`
1 parent 632269c commit ef38ccf

File tree

7 files changed

+738
-44
lines changed

7 files changed

+738
-44
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,9 @@ jobs:
121121
- name: Run Build
122122
run: pnpm run build
123123

124+
- name: Run Vitest
125+
run: pnpm --filter @browserbasehq/stagehand run test:vitest
126+
124127
run-e2e-local-tests:
125128
needs: [run-lint, run-build]
126129
runs-on: ubuntu-latest

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,4 @@ packages/core/test-results/
2424
/examples/inference_summary
2525
/inference_summary
2626
.turbo
27+
.idea

‎packages/core/package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
"e2e:local": "playwright test --config=lib/v3/tests/v3.local.playwright.config.ts",
1919
"e2e:bb": "playwright test --config=lib/v3/tests/v3.bb.playwright.config.ts",
2020
"lint": "cd ../.. && prettier --check packages/core && cd packages/core && eslint .",
21-
"format": "prettier --write ."
21+
"format": "prettier --write .",
22+
"test:vitest": "pnpm run build-js && vitest run --config vitest.config.ts"
2223
},
2324
"files": [
2425
"dist/index.js",
@@ -83,7 +84,8 @@
8384
"prettier": "^3.2.5",
8485
"tsup": "^8.2.1",
8586
"tsx": "^4.10.5",
86-
"typescript": "^5.2.2"
87+
"typescript": "^5.2.2",
88+
"vitest": "^4.0.8"
8789
},
8890
"repository": {
8991
"type": "git",

‎packages/core/tests/public-types.test.ts‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import { describe, expect, expectTypeOf, it } from "vitest";
2+
import StagehandDefaultExport, * as Stagehand from "../dist/index.js";
3+
4+
const publicApiShape = {
5+
AISdkClient: Stagehand.AISdkClient,
6+
AVAILABLE_CUA_MODELS: Stagehand.AVAILABLE_CUA_MODELS,
7+
AgentProvider: Stagehand.AgentProvider,
8+
AgentScreenshotProviderError: Stagehand.AgentScreenshotProviderError,
9+
AnnotatedScreenshotText: Stagehand.AnnotatedScreenshotText,
10+
BrowserbaseSessionNotFoundError: Stagehand.BrowserbaseSessionNotFoundError,
11+
CaptchaTimeoutError: Stagehand.CaptchaTimeoutError,
12+
ConnectionTimeoutError: Stagehand.ConnectionTimeoutError,
13+
ConsoleMessage: Stagehand.ConsoleMessage,
14+
ContentFrameNotFoundError: Stagehand.ContentFrameNotFoundError,
15+
CreateChatCompletionResponseError:
16+
Stagehand.CreateChatCompletionResponseError,
17+
CuaModelRequiredError: Stagehand.CuaModelRequiredError,
18+
ElementNotVisibleError: Stagehand.ElementNotVisibleError,
19+
ExperimentalApiConflictError: Stagehand.ExperimentalApiConflictError,
20+
ExperimentalNotConfiguredError: Stagehand.ExperimentalNotConfiguredError,
21+
HandlerNotInitializedError: Stagehand.HandlerNotInitializedError,
22+
InvalidAISDKModelFormatError: Stagehand.InvalidAISDKModelFormatError,
23+
LLMClient: Stagehand.LLMClient,
24+
LLMResponseError: Stagehand.LLMResponseError,
25+
LOG_LEVEL_NAMES: Stagehand.LOG_LEVEL_NAMES,
26+
MCPConnectionError: Stagehand.MCPConnectionError,
27+
MissingEnvironmentVariableError: Stagehand.MissingEnvironmentVariableError,
28+
MissingLLMConfigurationError: Stagehand.MissingLLMConfigurationError,
29+
PageNotFoundError: Stagehand.PageNotFoundError,
30+
Response: Stagehand.Response,
31+
ResponseBodyError: Stagehand.ResponseBodyError,
32+
ResponseParseError: Stagehand.ResponseParseError,
33+
Stagehand: Stagehand.Stagehand,
34+
StagehandAPIError: Stagehand.StagehandAPIError,
35+
StagehandAPIUnauthorizedError: Stagehand.StagehandAPIUnauthorizedError,
36+
StagehandClickError: Stagehand.StagehandClickError,
37+
StagehandDefaultError: Stagehand.StagehandDefaultError,
38+
StagehandDomProcessError: Stagehand.StagehandDomProcessError,
39+
StagehandElementNotFoundError: Stagehand.StagehandElementNotFoundError,
40+
StagehandEnvironmentError: Stagehand.StagehandEnvironmentError,
41+
StagehandError: Stagehand.StagehandError,
42+
StagehandEvalError: Stagehand.StagehandEvalError,
43+
StagehandHttpError: Stagehand.StagehandHttpError,
44+
StagehandIframeError: Stagehand.StagehandIframeError,
45+
StagehandInitError: Stagehand.StagehandInitError,
46+
StagehandInvalidArgumentError: Stagehand.StagehandInvalidArgumentError,
47+
StagehandMissingArgumentError: Stagehand.StagehandMissingArgumentError,
48+
StagehandNotInitializedError: Stagehand.StagehandNotInitializedError,
49+
StagehandResponseBodyError: Stagehand.StagehandResponseBodyError,
50+
StagehandResponseParseError: Stagehand.StagehandResponseParseError,
51+
StagehandServerError: Stagehand.StagehandServerError,
52+
StagehandShadowRootMissingError: Stagehand.StagehandShadowRootMissingError,
53+
StagehandShadowSegmentEmptyError: Stagehand.StagehandShadowSegmentEmptyError,
54+
StagehandShadowSegmentNotFoundError:
55+
Stagehand.StagehandShadowSegmentNotFoundError,
56+
TimeoutError: Stagehand.TimeoutError,
57+
UnsupportedAISDKModelProviderError:
58+
Stagehand.UnsupportedAISDKModelProviderError,
59+
UnsupportedModelError: Stagehand.UnsupportedModelError,
60+
UnsupportedModelProviderError: Stagehand.UnsupportedModelProviderError,
61+
V3: Stagehand.V3,
62+
V3Evaluator: Stagehand.V3Evaluator,
63+
V3FunctionName: Stagehand.V3FunctionName,
64+
XPathResolutionError: Stagehand.XPathResolutionError,
65+
ZodSchemaValidationError: Stagehand.ZodSchemaValidationError,
66+
connectToMCPServer: Stagehand.connectToMCPServer,
67+
default: StagehandDefaultExport,
68+
defaultExtractSchema: Stagehand.defaultExtractSchema,
69+
getZodType: Stagehand.getZodType,
70+
injectUrls: Stagehand.injectUrls,
71+
isRunningInBun: Stagehand.isRunningInBun,
72+
jsonSchemaToZod: Stagehand.jsonSchemaToZod,
73+
loadApiKeyFromEnv: Stagehand.loadApiKeyFromEnv,
74+
modelToAgentProviderMap: Stagehand.modelToAgentProviderMap,
75+
pageTextSchema: Stagehand.pageTextSchema,
76+
providerEnvVarMap: Stagehand.providerEnvVarMap,
77+
toGeminiSchema: Stagehand.toGeminiSchema,
78+
transformSchema: Stagehand.transformSchema,
79+
trimTrailingTextNode: Stagehand.trimTrailingTextNode,
80+
validateZodSchema: Stagehand.validateZodSchema,
81+
} as const;
82+
83+
type StagehandExports = typeof Stagehand & {
84+
default: typeof StagehandDefaultExport;
85+
};
86+
87+
type PublicAPI = {
88+
[K in keyof typeof publicApiShape]: StagehandExports[K];
89+
};
90+
91+
describe("Stagehand public API types", () => {
92+
it("public API shape matches module exports", () => {
93+
const _check: PublicAPI = publicApiShape;
94+
void _check;
95+
});
96+
97+
it("does not expose unexpected top-level exports", () => {
98+
const expected = Object.keys(publicApiShape).sort();
99+
const actual = Object.keys(Stagehand).sort();
100+
expect(actual).toStrictEqual(expected);
101+
});
102+
103+
it("StagehandMetrics includes token counters", () => {
104+
expectTypeOf<Stagehand.StagehandMetrics>().toEqualTypeOf<{
105+
actPromptTokens: number;
106+
actCompletionTokens: number;
107+
actReasoningTokens: number;
108+
actCachedInputTokens: number;
109+
actInferenceTimeMs: number;
110+
extractPromptTokens: number;
111+
extractCompletionTokens: number;
112+
extractReasoningTokens: number;
113+
extractCachedInputTokens: number;
114+
extractInferenceTimeMs: number;
115+
observePromptTokens: number;
116+
observeCompletionTokens: number;
117+
observeReasoningTokens: number;
118+
observeCachedInputTokens: number;
119+
observeInferenceTimeMs: number;
120+
agentPromptTokens: number;
121+
agentCompletionTokens: number;
122+
agentReasoningTokens: number;
123+
agentCachedInputTokens: number;
124+
agentInferenceTimeMs: number;
125+
totalPromptTokens: number;
126+
totalCompletionTokens: number;
127+
totalReasoningTokens: number;
128+
totalCachedInputTokens: number;
129+
totalInferenceTimeMs: number;
130+
}>();
131+
});
132+
});

‎packages/core/tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@
1818
"declaration": true,
1919
"emitDeclarationOnly": true
2020
},
21-
"include": ["lib/**/*", "examples/**/*"],
21+
"include": ["lib/**/*", "examples/**/*", "tests/**/*"],
2222
"exclude": ["node_modules", "dist"]
2323
}

‎packages/core/vitest.config.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { defineConfig } from "vitest/config";
2+
3+
export default defineConfig({
4+
test: {
5+
environment: "node",
6+
include: ["tests/**/*.test.ts"],
7+
},
8+
});

0 commit comments

Comments
 (0)