diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 76e9d6b1f2476..19aa0d94f5d82 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -544,6 +544,17 @@ sequence of events is `request`, `response` and `requestfinished`. Emitted when [response] status and headers are received for a request. For a successful response, the sequence of events is `request`, `response` and `requestfinished`. +## event: Page.agentTurn +* since: v1.58 +- argument: <[Object]> + - `role` <[string]> + - `message` <[string]> + - `usage` ?<[Object]> + - `inputTokens` <[int]> + - `outputTokens` <[int]> + +Emitted when the agent makes a turn. + ## event: Page.webSocket * since: v1.9 - argument: <[WebSocket]> diff --git a/docs/src/api/params.md b/docs/src/api/params.md index babbdaf3b0a90..807670d29ac9d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -375,7 +375,7 @@ Emulates consistent window screen size available inside web page via `window.scr - `provider` <[string]> LLM provider to use. - `model` <[string]> Model identifier within provider. - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - - `cacheMode` ?<['force'|'ignore'|'auto']> Cache control, defaults to 'auto'. + - `cacheMode` ?<[CacheMode]<"force"|"ignore"|"update"|"auto">> Cache control, defaults to 'auto'. - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM. - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10. - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited. diff --git a/examples/todomvc/playwright.config.ts b/examples/todomvc/playwright.config.ts index e2fbf59bb8d82..e21dd0f655d31 100644 --- a/examples/todomvc/playwright.config.ts +++ b/examples/todomvc/playwright.config.ts @@ -1,6 +1,9 @@ /* eslint-disable notice/notice */ import { defineConfig, devices } from '@playwright/test'; +import dotenv from 'dotenv'; + +dotenv.config(); /** * See https://playwright.dev/docs/test-configuration. diff --git a/package-lock.json b/package-lock.json index 09707cfd578de..3b97f99851704 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@eslint/compat": "^1.3.2", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.34.0", - "@lowire/loop": "^0.0.8", + "@lowire/loop": "^0.0.11", "@modelcontextprotocol/sdk": "^1.17.5", "@octokit/graphql-schema": "^15.26.0", "@stylistic/eslint-plugin": "^5.2.3", @@ -1064,9 +1064,9 @@ } }, "node_modules/@lowire/loop": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.8.tgz", - "integrity": "sha512-DuyxRnPnu8Bwsfwt77tbZAgWGYKhEN/0xpsvHNin/62OEBiVx/nvSbj9NMFmeP83NoqCTG5n7JZCkA5GIw1pCA==", + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.11.tgz", + "integrity": "sha512-kDfrYp6g6W4Nm/Owu3pP04UM7jbqR9IO9ITNJ0p52huoE3f7KcyrYOyyi7ITrmzLm2h+7vUUMZEzhGNVIelpRg==", "dev": true, "license": "Apache-2.0", "engines": { diff --git a/package.json b/package.json index d95607cc2a990..b59d457fa0632 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "@eslint/compat": "^1.3.2", "@eslint/eslintrc": "^3.3.1", "@eslint/js": "^9.34.0", - "@lowire/loop": "^0.0.8", + "@lowire/loop": "^0.0.11", "@modelcontextprotocol/sdk": "^1.17.5", "@octokit/graphql-schema": "^15.26.0", "@stylistic/eslint-plugin": "^5.2.3", diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index d9917c1a1d0ff..6788988908458 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -1036,6 +1036,21 @@ export interface Page { * @param options */ extract(query: string, schema: Schema): Promise>; + /** + * Emitted when the agent makes a turn. + */ + on(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Emitted when the page closes. */ @@ -1243,6 +1258,21 @@ export interface Page { */ on(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. + */ + once(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. */ @@ -1338,6 +1368,21 @@ export interface Page { */ once(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Emitted when the agent makes a turn. + */ + addListener(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Emitted when the page closes. */ @@ -1545,6 +1590,21 @@ export interface Page { */ addListener(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Removes an event listener added by `on` or `addListener`. + */ + removeListener(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Removes an event listener added by `on` or `addListener`. */ @@ -1640,6 +1700,21 @@ export interface Page { */ removeListener(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Removes an event listener added by `on` or `addListener`. + */ + off(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Removes an event listener added by `on` or `addListener`. */ @@ -1735,6 +1810,21 @@ export interface Page { */ off(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Emitted when the agent makes a turn. + */ + prependListener(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Emitted when the page closes. */ @@ -4790,6 +4880,41 @@ export interface Page { height: number; }; + /** + * Emitted when the agent makes a turn. + */ + waitForEvent(event: 'agentturn', optionsOrPredicate?: { predicate?: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => boolean | Promise, timeout?: number } | ((data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => boolean | Promise)): Promise<{ + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }>; + /** * Emitted when the page closes. */ @@ -22118,7 +22243,7 @@ export interface BrowserContextOptions { /** * Cache control, defaults to 'auto'. */ - cacheMode?: 'force'|'ignore'|'auto'; + cacheMode?: "force"|"ignore"|"update"|"auto"; /** * Secrets to hide from the LLM. diff --git a/packages/playwright-core/ThirdPartyNotices.txt b/packages/playwright-core/ThirdPartyNotices.txt index c6d3cb672471b..7ed4413e0ba3c 100644 --- a/packages/playwright-core/ThirdPartyNotices.txt +++ b/packages/playwright-core/ThirdPartyNotices.txt @@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise. -- @lowire/loop@0.0.8 (https://github.com/pavelfeldman/lowire) +- @lowire/loop@0.0.11 (https://github.com/pavelfeldman/lowire) - @modelcontextprotocol/sdk@1.24.2 (https://github.com/modelcontextprotocol/typescript-sdk) - accepts@2.0.0 (https://github.com/jshttp/accepts) - agent-base@7.1.4 (https://github.com/TooTallNate/proxy-agents) @@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina - zod-to-json-schema@3.25.0 (https://github.com/StefanTerdell/zod-to-json-schema) - zod@3.25.76 (https://github.com/colinhacks/zod) -%% @lowire/loop@0.0.8 NOTICES AND INFORMATION BEGIN HERE +%% @lowire/loop@0.0.11 NOTICES AND INFORMATION BEGIN HERE ========================================= Apache License Version 2.0, January 2004 @@ -339,7 +339,7 @@ Apache License See the License for the specific language governing permissions and limitations under the License. ========================================= -END OF @lowire/loop@0.0.8 AND INFORMATION +END OF @lowire/loop@0.0.11 AND INFORMATION %% @modelcontextprotocol/sdk@1.24.2 NOTICES AND INFORMATION BEGIN HERE ========================================= diff --git a/packages/playwright-core/bundles/mcp/package-lock.json b/packages/playwright-core/bundles/mcp/package-lock.json index 0e18bcbefe5cd..2754245dd72fa 100644 --- a/packages/playwright-core/bundles/mcp/package-lock.json +++ b/packages/playwright-core/bundles/mcp/package-lock.json @@ -8,16 +8,16 @@ "name": "mcp-bundle", "version": "0.0.1", "dependencies": { - "@lowire/loop": "^0.0.8", + "@lowire/loop": "^0.0.11", "@modelcontextprotocol/sdk": "^1.24.0", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" } }, "node_modules/@lowire/loop": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.8.tgz", - "integrity": "sha512-DuyxRnPnu8Bwsfwt77tbZAgWGYKhEN/0xpsvHNin/62OEBiVx/nvSbj9NMFmeP83NoqCTG5n7JZCkA5GIw1pCA==", + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/@lowire/loop/-/loop-0.0.11.tgz", + "integrity": "sha512-kDfrYp6g6W4Nm/Owu3pP04UM7jbqR9IO9ITNJ0p52huoE3f7KcyrYOyyi7ITrmzLm2h+7vUUMZEzhGNVIelpRg==", "license": "Apache-2.0", "engines": { "node": ">=20" diff --git a/packages/playwright-core/bundles/mcp/package.json b/packages/playwright-core/bundles/mcp/package.json index 7e6f08a8e8213..2c6d12e6f0022 100644 --- a/packages/playwright-core/bundles/mcp/package.json +++ b/packages/playwright-core/bundles/mcp/package.json @@ -3,8 +3,8 @@ "version": "0.0.1", "private": true, "dependencies": { + "@lowire/loop": "^0.0.11", "@modelcontextprotocol/sdk": "^1.24.0", - "@lowire/loop": "^0.0.8", "zod": "^3.25.76", "zod-to-json-schema": "^3.24.6" } diff --git a/packages/playwright-core/src/client/events.ts b/packages/playwright-core/src/client/events.ts index cede73640cf3b..36a3c750933ad 100644 --- a/packages/playwright-core/src/client/events.ts +++ b/packages/playwright-core/src/client/events.ts @@ -55,6 +55,7 @@ export const Events = { }, Page: { + AgentTurn: 'agentturn', Close: 'close', Crash: 'crash', Console: 'console', diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 4f36c15869531..1c3cbe9e22b14 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -134,6 +134,7 @@ export class Page extends ChannelOwner implements api.Page this._closed = initializer.isClosed; this._opener = Page.fromNullable(initializer.opener); + this._channel.on('agentTurn', params => this.emit(Events.Page.AgentTurn, params)); this._channel.on('bindingCall', ({ binding }) => this._onBinding(BindingCall.from(binding))); this._channel.on('close', () => this._onClose()); this._channel.on('crash', () => this._onCrash()); diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 2cf6e69c39768..d75cb62722b0c 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -606,7 +606,7 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({ provider: tString, model: tString, cacheFile: tOptional(tString), - cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])), + cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])), secrets: tOptional(tArray(tType('NameValue'))), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), @@ -707,7 +707,7 @@ scheme.BrowserNewContextParams = tObject({ provider: tString, model: tString, cacheFile: tOptional(tString), - cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])), + cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])), secrets: tOptional(tArray(tType('NameValue'))), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), @@ -787,7 +787,7 @@ scheme.BrowserNewContextForReuseParams = tObject({ provider: tString, model: tString, cacheFile: tOptional(tString), - cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])), + cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])), secrets: tOptional(tArray(tType('NameValue'))), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), @@ -912,7 +912,7 @@ scheme.BrowserContextInitializer = tObject({ provider: tString, model: tString, cacheFile: tOptional(tString), - cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])), + cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])), secrets: tOptional(tArray(tType('NameValue'))), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), @@ -1180,6 +1180,14 @@ scheme.PageInitializer = tObject({ isClosed: tBoolean, opener: tOptional(tChannel(['Page'])), }); +scheme.PageAgentTurnEvent = tObject({ + role: tString, + message: tString, + usage: tOptional(tObject({ + inputTokens: tInt, + outputTokens: tInt, + })), +}); scheme.PageBindingCallEvent = tObject({ binding: tChannel(['BindingCall']), }); @@ -2832,7 +2840,7 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({ provider: tString, model: tString, cacheFile: tOptional(tString), - cacheMode: tOptional(tEnum(['ignore', 'force', 'auto'])), + cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])), secrets: tOptional(tArray(tType('NameValue'))), maxTurns: tOptional(tInt), maxTokens: tOptional(tInt), diff --git a/packages/playwright-core/src/server/agent/agent.ts b/packages/playwright-core/src/server/agent/agent.ts index 507d630e4c2a2..57de45dcd265b 100644 --- a/packages/playwright-core/src/server/agent/agent.ts +++ b/packages/playwright-core/src/server/agent/agent.ts @@ -21,10 +21,10 @@ import { debug } from '../../utilsBundle'; import { Loop } from '../../mcpBundle'; import { runAction } from './actionRunner'; import { Context } from './context'; +import { Page } from '../page'; import type { Progress } from '../progress'; import type * as channels from '@protocol/channels'; -import type { Page } from '../page'; import type * as loopTypes from '@lowire/loop'; import type * as actions from './actions'; @@ -72,6 +72,8 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp const { full } = await page.snapshotForAI(progress); const { tools, callTool } = toolsForLoop(context); + page.emit(Page.Events.AgentTurn, { role: 'user', message: userTask }); + const limits = context.limits(options); let turns = 0; const loop = new Loop(browserContext._options.agent.provider as any, { @@ -81,11 +83,23 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp callTool, tools, ...limits, - beforeTurn: params => { + onAfterTurn: ({ assistantMessage, totalUsage }) => { ++turns; - const lastReply = params.conversation.messages.findLast(m => m.role === 'assistant'); - const toolCall = lastReply?.content.find(c => c.type === 'tool_call'); - if (!resultSchema && toolCall && toolCall.arguments.thatShouldBeIt) + const usage = { inputTokens: totalUsage.input, outputTokens: totalUsage.output }; + const intent = assistantMessage.content.filter(c => c.type === 'text').map(c => c.text).join('\n'); + page.emit(Page.Events.AgentTurn, { role: 'assistant', message: intent, usage }); + if (!assistantMessage.content.filter(c => c.type === 'tool_call').length) + page.emit(Page.Events.AgentTurn, { role: 'assistant', message: `no tool calls`, usage }); + return 'continue'; + }, + onBeforeToolCall: ({ toolCall }) => { + page.emit(Page.Events.AgentTurn, { role: 'assistant', message: `call tool "${toolCall.name}"` }); + return 'continue'; + }, + onAfterToolCall: ({ toolCall }) => { + const suffix = toolCall.result?.isError ? 'failed' : 'succeeded'; + page.emit(Page.Events.AgentTurn, { role: 'user', message: `tool "${toolCall.name}" ${suffix}` }); + if (toolCall.arguments.thatShouldBeIt) return 'break'; return 'continue'; }, @@ -117,7 +131,7 @@ type CachedActions = Record(); async function cachedPerform(context: Context, options: channels.PagePerformParams): Promise { - if (!context.options?.cacheFile || context.options.cacheMode === 'ignore') + if (!context.options?.cacheFile || context.options.cacheMode === 'ignore' || context.options.cacheMode === 'update') return false; const cache = await cachedActions(context.options.cacheFile); diff --git a/packages/playwright-core/src/server/agent/tools.ts b/packages/playwright-core/src/server/agent/tools.ts index 4b85eda6fb2c3..f14671aef06ee 100644 --- a/packages/playwright-core/src/server/agent/tools.ts +++ b/packages/playwright-core/src/server/agent/tools.ts @@ -215,7 +215,7 @@ const fillForm = defineTool({ schema: { name: 'browser_fill_form', title: 'Fill form', - description: 'Fill multiple form fields', + description: 'Fill multiple form fields. Always use this tool when you can fill more than one field at a time.', inputSchema: baseSchema.extend({ fields: z.array(z.object({ name: z.string().describe('Human-readable field name'), diff --git a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts index dcfc35b0ecf71..1ba01607c50c2 100644 --- a/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/pageDispatcher.ts @@ -94,6 +94,7 @@ export class PageDispatcher extends Dispatcher this._dispatchEvent('agentTurn', params)); this.addObjectListener(Page.Events.Close, () => { this._dispatchEvent('close'); this._dispose(); diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 1759727992d74..2adb01e405971 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -120,6 +120,7 @@ type ExpectScreenshotOptions = ImageComparatorOptions & ScreenshotOptions & { }; const PageEvent = { + AgentTurn: 'agentturn', Close: 'close', Crash: 'crash', Download: 'download', @@ -136,6 +137,7 @@ const PageEvent = { } as const; export type PageEventMap = { + [PageEvent.AgentTurn]: [agentTurn: { role: string, message: string, usage?: { inputTokens: number, outputTokens: number } }]; [PageEvent.Close]: []; [PageEvent.Crash]: []; [PageEvent.Download]: [download: Download]; diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index d9917c1a1d0ff..6788988908458 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -1036,6 +1036,21 @@ export interface Page { * @param options */ extract(query: string, schema: Schema): Promise>; + /** + * Emitted when the agent makes a turn. + */ + on(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Emitted when the page closes. */ @@ -1243,6 +1258,21 @@ export interface Page { */ on(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. + */ + once(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. */ @@ -1338,6 +1368,21 @@ export interface Page { */ once(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Emitted when the agent makes a turn. + */ + addListener(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Emitted when the page closes. */ @@ -1545,6 +1590,21 @@ export interface Page { */ addListener(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Removes an event listener added by `on` or `addListener`. + */ + removeListener(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Removes an event listener added by `on` or `addListener`. */ @@ -1640,6 +1700,21 @@ export interface Page { */ removeListener(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Removes an event listener added by `on` or `addListener`. + */ + off(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Removes an event listener added by `on` or `addListener`. */ @@ -1735,6 +1810,21 @@ export interface Page { */ off(event: 'worker', listener: (worker: Worker) => any): this; + /** + * Emitted when the agent makes a turn. + */ + prependListener(event: 'agentturn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + /** * Emitted when the page closes. */ @@ -4790,6 +4880,41 @@ export interface Page { height: number; }; + /** + * Emitted when the agent makes a turn. + */ + waitForEvent(event: 'agentturn', optionsOrPredicate?: { predicate?: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => boolean | Promise, timeout?: number } | ((data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => boolean | Promise)): Promise<{ + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }>; + /** * Emitted when the page closes. */ @@ -22118,7 +22243,7 @@ export interface BrowserContextOptions { /** * Cache control, defaults to 'auto'. */ - cacheMode?: 'force'|'ignore'|'auto'; + cacheMode?: "force"|"ignore"|"update"|"auto"; /** * Secrets to hide from the LLM. diff --git a/packages/playwright/src/runner/storage.ts b/packages/playwright/src/runner/storage.ts index 46c897a6d28d8..90aace6faf9b6 100644 --- a/packages/playwright/src/runner/storage.ts +++ b/packages/playwright/src/runner/storage.ts @@ -87,7 +87,7 @@ export class Storage { } private _writeFile(entries: Record) { - this._writeChain = this._writeChain.then(() => fs.promises.writeFile(this._fileName, JSON.stringify(entries, null, 2))).catch(() => {}); + this._writeChain = this._writeChain.then(() => fs.promises.writeFile(this._fileName, JSON.stringify(entries, null, 2))); return this._writeChain; } diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index 3b2e83918ef68..190e39174e32c 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -1012,7 +1012,7 @@ export type BrowserTypeLaunchPersistentContextParams = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -1103,7 +1103,7 @@ export type BrowserTypeLaunchPersistentContextOptions = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -1233,7 +1233,7 @@ export type BrowserNewContextParams = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -1310,7 +1310,7 @@ export type BrowserNewContextOptions = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -1390,7 +1390,7 @@ export type BrowserNewContextForReuseParams = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -1467,7 +1467,7 @@ export type BrowserNewContextForReuseOptions = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -1611,7 +1611,7 @@ export type BrowserContextInitializer = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -2076,6 +2076,7 @@ export type PageInitializer = { opener?: PageChannel, }; export interface PageEventTarget { + on(event: 'agentTurn', callback: (params: PageAgentTurnEvent) => void): this; on(event: 'bindingCall', callback: (params: PageBindingCallEvent) => void): this; on(event: 'close', callback: (params: PageCloseEvent) => void): this; on(event: 'crash', callback: (params: PageCrashEvent) => void): this; @@ -2135,6 +2136,14 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel { perform(params: PagePerformParams, progress?: Progress): Promise; extract(params: PageExtractParams, progress?: Progress): Promise; } +export type PageAgentTurnEvent = { + role: string, + message: string, + usage?: { + inputTokens: number, + outputTokens: number, + }, +}; export type PageBindingCallEvent = { binding: BindingCallChannel, }; @@ -2669,6 +2678,7 @@ export type PageExtractResult = { }; export interface PageEvents { + 'agentTurn': PageAgentTurnEvent; 'bindingCall': PageBindingCallEvent; 'close': PageCloseEvent; 'crash': PageCrashEvent; @@ -4935,7 +4945,7 @@ export type AndroidDeviceLaunchBrowserParams = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, @@ -5010,7 +5020,7 @@ export type AndroidDeviceLaunchBrowserOptions = { provider: string, model: string, cacheFile?: string, - cacheMode?: 'ignore' | 'force' | 'auto', + cacheMode?: 'ignore' | 'force' | 'update' | 'auto', secrets?: NameValue[], maxTurns?: number, maxTokens?: number, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index 20dcdf07b0b59..e2a194629d6f2 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -600,6 +600,7 @@ ContextOptions: literals: - ignore - force + - update - auto secrets: type: array? @@ -2052,6 +2053,16 @@ Page: events: + agentTurn: + parameters: + role: string + message: string + usage: + type: object? + properties: + inputTokens: int + outputTokens: int + bindingCall: parameters: binding: BindingCall diff --git a/tests/library/perform-task.spec.ts b/tests/library/perform-task.spec.ts index 6ac69ea144cc2..0edc7c5d29262 100644 --- a/tests/library/perform-task.spec.ts +++ b/tests/library/perform-task.spec.ts @@ -21,9 +21,9 @@ import { browserTest as test, expect } from '../config/browserTest'; test.use({ agent: { provider: 'github', - model: 'claude-sonnet-4.5', - cachePathTemplate: '{testFilePath}-cache.json', - cacheMode: process.env.CI ? 'force' : 'auto', + model: 'gpt-4.1', + cachePathTemplate: '{testDir}/{testFilePath}-cache.json', + cacheMode: process.env.UPDATE_CACHE ? 'update' : process.env.CI ? 'force' : 'auto', secrets: { 'x-secret-email': 'secret-email@at-microsoft.com', } @@ -32,6 +32,9 @@ test.use({ test('page.perform', async ({ page, server }) => { await page.goto(server.PREFIX + '/evals/fill-form.html'); + page.on('agentturn', turn => { + console.log('agentturn', turn); + }); await page.perform('Fill out the form with the following details:\n' + 'Name: John Smith\n' + 'Address: 1045 La Avenida St, Mountain View, CA 94043\n' + diff --git a/tests/library/perform-task.spec.ts-cache.json b/tests/library/perform-task.spec.ts-cache.json index 528c77b73859a..ccd4d4a958557 100644 --- a/tests/library/perform-task.spec.ts-cache.json +++ b/tests/library/perform-task.spec.ts-cache.json @@ -1,72 +1,72 @@ { "Enter x-secret-email into the email field": { - "timestamp": 1765482830501, + "timestamp": 1765673294749, "actions": [ { "method": "fill", "selector": "internal:role=textbox[name=\"Email Address\"i]", "text": "x-secret-email", "code": "await page.getByRole('textbox', { name: 'Email Address' }).fill('x-secret-email');", - "intent": "Enter 'x-secret-email' into the email address field" + "intent": "Enter 'x-secret-email' into the 'Email Address' textbox." } ] }, "Fill out the form with the following details:\nName: John Smith\nAddress: 1045 La Avenida St, Mountain View, CA 94043\nEmail: john.smith@at-microsoft.com": { - "timestamp": 1765482831906, + "timestamp": 1765673328439, "actions": [ { "method": "fill", "selector": "internal:role=textbox[name=\"Full Name *\"i]", "text": "John Smith", "code": "await page.getByRole('textbox', { name: 'Full Name *' }).fill('John Smith');", - "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email (john.smith@at-microsoft.com)" + "intent": "Fill in the name, address, and email fields according to the provided user information." }, { "method": "fill", "selector": "internal:role=textbox[name=\"Email Address *\"i]", "text": "john.smith@at-microsoft.com", "code": "await page.getByRole('textbox', { name: 'Email Address *' }).fill('john.smith@at-microsoft.com');", - "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email (john.smith@at-microsoft.com)" + "intent": "Fill in the name, address, and email fields according to the provided user information." }, { "method": "fill", "selector": "internal:role=textbox[name=\"Street Address *\"i]", "text": "1045 La Avenida St", "code": "await page.getByRole('textbox', { name: 'Street Address *' }).fill('1045 La Avenida St');", - "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email (john.smith@at-microsoft.com)" + "intent": "Fill in the name, address, and email fields according to the provided user information." }, { "method": "fill", "selector": "internal:role=textbox[name=\"City *\"i]", "text": "Mountain View", "code": "await page.getByRole('textbox', { name: 'City *' }).fill('Mountain View');", - "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email (john.smith@at-microsoft.com)" + "intent": "Fill in the name, address, and email fields according to the provided user information." }, { "method": "fill", "selector": "internal:role=textbox[name=\"State/Province *\"i]", "text": "CA", "code": "await page.getByRole('textbox', { name: 'State/Province *' }).fill('CA');", - "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email (john.smith@at-microsoft.com)" + "intent": "Fill in the name, address, and email fields according to the provided user information." }, { "method": "fill", "selector": "internal:role=textbox[name=\"ZIP/Postal Code *\"i]", "text": "94043", "code": "await page.getByRole('textbox', { name: 'ZIP/Postal Code *' }).fill('94043');", - "intent": "Fill out the form with the provided details: Name (John Smith), Address (1045 La Avenida St, Mountain View, CA 94043), and Email (john.smith@at-microsoft.com)" + "intent": "Fill in the name, address, and email fields according to the provided user information." } ] }, "- Enter \"bogus\" into the email field\n - Check that the value is in fact \"bogus\"\n - Check that the error message is displayed": { - "timestamp": 1765482849524, + "timestamp": 1765673309068, "actions": [ { "method": "fill", "selector": "internal:role=textbox[name=\"Email Address\"i]", "text": "bogus", "code": "await page.getByRole('textbox', { name: 'Email Address' }).fill('bogus');", - "intent": "Enter \"bogus\" into the email field" + "intent": "Enter 'bogus' into the email field to simulate invalid input and test validation." }, { "method": "expectValue", @@ -74,14 +74,14 @@ "type": "textbox", "value": "bogus", "code": "await expect(page.getByRole('textbox', { name: 'Email Address' })).toHaveValue('bogus');", - "intent": "Check that the email field value is \"bogus\"" + "intent": "Check that the value in the email field is 'bogus'.\nCheck that the error message is displayed for the invalid email." }, { "method": "expectVisible", - "selector": "internal:text=\"Error: Invalid email address\"i", - "code": "await expect(page.getByText('Error: Invalid email address')).toBeVisible();", - "intent": "Check that the error message \"Error: Invalid email address\" is displayed" + "selector": "internal:text=\"error\"i", + "code": "await expect(page.getByText('error')).toBeVisible();", + "intent": "Check that the value in the email field is 'bogus'.\nCheck that the error message is displayed for the invalid email." } ] } -} \ No newline at end of file +} diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts index 874f1d1ecbd9e..6a3e609f09de9 100644 --- a/tests/mcp/fixtures.ts +++ b/tests/mcp/fixtures.ts @@ -202,6 +202,7 @@ export const test = serverTest.extend