diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 13299d4646cf7..55399a4d678f8 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -712,27 +712,23 @@ Raw CSS content to be injected into frame. ## async method: Page.agent * since: v1.58 * langs: js -* hidden - returns: <[PageAgent]> Initialize page agent with the llm provider and cache. ### option: Page.agent.cache * since: v1.58 -* hidden - `cache` <[Object]> - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. ### option: Page.agent.expect * since: v1.58 -* hidden - `expect` <[Object]> - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms. ### option: Page.agent.limits * since: v1.58 -* hidden - `limits` <[Object]> - `maxTokens` ?<[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. Defaults to unlimited. - `maxActions` ?<[int]> Maximum number of agentic actions to generate, defaults to 10. @@ -742,7 +738,6 @@ Limits to use for the agentic loop. ### option: Page.agent.provider * since: v1.58 -* hidden - `provider` <[Object]> - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use. - `apiEndpoint` ?<[string]> Endpoint to use if different from default. @@ -752,14 +747,12 @@ Limits to use for the agentic loop. ### option: Page.agent.secrets * since: v1.58 -* hidden - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM. ### option: Page.agent.systemPrompt * since: v1.58 -* hidden - `systemPrompt` <[string]> System prompt for the agent's loop. diff --git a/docs/src/api/class-pageagent.md b/docs/src/api/class-pageagent.md index e1a9f4e5201d7..0f04e2b04d851 100644 --- a/docs/src/api/class-pageagent.md +++ b/docs/src/api/class-pageagent.md @@ -1,11 +1,9 @@ # class: PageAgent * since: v1.58 * langs: js -* hidden ## event: PageAgent.turn * since: v1.58 -* hidden - argument: <[Object]> - `role` <[string]> - `message` <[string]> @@ -16,13 +14,11 @@ Emitted when the agent makes a turn. ## async method: PageAgent.dispose -* hidden * since: v1.58 Dispose this agent. ## async method: PageAgent.expect -* hidden * since: v1.58 Expect certain condition to be met. @@ -35,14 +31,12 @@ await agent.expect('"0 items" to be reported'); ### param: PageAgent.expect.expectation * since: v1.58 -* hidden - `expectation` <[string]> Expectation to assert. ### option: PageAgent.expect.timeout * since: v1.58 -* hidden - `timeout` <[float]> Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout. @@ -52,7 +46,6 @@ Expect timeout in milliseconds. Defaults to `5000`. The default value can be cha ## async method: PageAgent.extract * since: v1.58 -* hidden - returns: <[Object]> - `result` <[any]> - `usage` <[Object]> @@ -73,19 +66,16 @@ await agent.extract('List of items in the cart', z.object({ ### param: PageAgent.extract.query * since: v1.58 -* hidden - `query` <[string]> Task to perform using agentic loop. ### param: PageAgent.extract.schema * since: v1.58 -* hidden - `schema` <[z.ZodSchema]> ### option: PageAgent.extract.timeout * since: v1.58 -* hidden - `timeout` <[float]> Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or @@ -94,9 +84,9 @@ Extract timeout in milliseconds. Defaults to `5000`. The default value can be ch ### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 + ## async method: PageAgent.perform * since: v1.58 -* hidden - returns: <[Object]> - `usage` <[Object]> - `turns` <[int]> @@ -113,14 +103,12 @@ await agent.perform('Click submit button'); ### param: PageAgent.perform.task * since: v1.58 -* hidden - `task` <[string]> Task to perform using agentic loop. ### option: PageAgent.perform.timeout * since: v1.58 -* hidden - `timeout` <[float]> Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or @@ -128,11 +116,9 @@ Perform timeout in milliseconds. Defaults to `5000`. The default value can be ch ### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%% * since: v1.58 -* hidden ## async method: PageAgent.usage * since: v1.58 -* hidden - returns: <[Object]> - `turns` <[int]> - `inputTokens` <[int]> diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 90c3b9e3621f4..69257b93db23d 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -372,7 +372,6 @@ Emulates consistent window screen size available inside web page via `window.scr ## page-agent-cache-key * since: v1.58 -* hidden - `cacheKey` <[string]> All the agentic actions are converted to the Playwright calls and are cached. @@ -380,7 +379,6 @@ By default, they are cached globally with the `task` as a key. This option allow ## page-agent-max-tokens * since: v1.58 -* hidden - `maxTokens` <[int]> Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. @@ -388,14 +386,12 @@ Defaults to context-wide value specified in `agent` property. ## page-agent-max-actions * since: v1.58 -* hidden - `maxActions` <[int]> Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. ## page-agent-max-action-retries * since: v1.58 -* hidden - `maxActionRetries` <[int]> Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property. diff --git a/docs/src/test-api/class-fixtures.md b/docs/src/test-api/class-fixtures.md index 4704ff708a15d..9ec8e33f92cd3 100644 --- a/docs/src/test-api/class-fixtures.md +++ b/docs/src/test-api/class-fixtures.md @@ -20,7 +20,6 @@ Playwright Test comes with builtin fixtures listed below, and you can add your o ## property: Fixtures.agent * since: v1.58 -* hidden - type: <[PageAgent]> ## property: Fixtures.browser diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index 411ba615851fe..1f3ee92df7750 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -106,7 +106,6 @@ Base directory for all relative paths used in the reporters. ## property: FullConfig.runAgents * since: v1.58 -* hidden - type: <['RunAgentsMode]<"all"|"missing"|"none">> Whether to run LLM agent for [PageAgent]: diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index a7b417484e51c..a01babad1f23b 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -516,7 +516,6 @@ export default defineConfig({ ## property: TestConfig.runAgents * since: v1.58 -* hidden - type: ?<['RunAgentsMode]<"all"|"missing"|"none">> Whether to run LLM agent for [PageAgent]: diff --git a/docs/src/test-api/class-testoptions.md b/docs/src/test-api/class-testoptions.md index 759fa86e55715..4c60a5c89617d 100644 --- a/docs/src/test-api/class-testoptions.md +++ b/docs/src/test-api/class-testoptions.md @@ -48,7 +48,6 @@ export default defineConfig({ ## property: TestOptions.agentOptions * since: v1.58 -* hidden - type: <[Object]> - `provider` <[Object]> - `api` <[PageAgentAPI]<"openai"|"openai-compatible"|"anthropic"|"google">> API to use. diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index f6e6165885f0a..50dd490708e13 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -27,6 +27,13 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector state?: 'visible'|'attached'; }; +// @ts-ignore this will be any if zod is not installed +import { ZodTypeAny, z } from 'zod'; +// @ts-ignore this will be any if zod is not installed +import * as z3 from 'zod/v3'; +type ZodSchema = ZodTypeAny | z3.ZodTypeAny; +type InferZodSchema = T extends z3.ZodTypeAny ? z3.infer : T extends ZodTypeAny ? z.infer : never; + /** * Page provides methods to interact with a single tab in a [Browser](https://playwright.dev/docs/api/class-browser), * or an [extension background page](https://developer.chrome.com/extensions/background_pages) in Chromium. One @@ -2088,6 +2095,89 @@ export interface Page { url?: string; }): Promise; + /** + * Initialize page agent with the llm provider and cache. + * @param options + */ + agent(options?: { + cache?: { + /** + * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). + */ + cacheFile?: string; + + /** + * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. + */ + cacheOutFile?: string; + }; + + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + + /** + * Limits to use for the agentic loop. + */ + limits?: { + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to unlimited. + */ + maxTokens?: number; + + /** + * Maximum number of agentic actions to generate, defaults to 10. + */ + maxActions?: number; + + /** + * Maximum number retries per action, defaults to 3. + */ + maxActionRetries?: number; + }; + + provider?: { + /** + * API to use. + */ + api: "openai"|"openai-compatible"|"anthropic"|"google"; + + /** + * Endpoint to use if different from default. + */ + apiEndpoint?: string; + + /** + * API key for the LLM provider. + */ + apiKey: string; + + /** + * Amount of time to wait for the provider to respond to each request. + */ + apiTimeout?: number; + + /** + * Model identifier within the provider. Required in non-cache mode. + */ + model: string; + }; + + /** + * Secrets to hide from the LLM. + */ + secrets?: { [key: string]: string; }; + + /** + * System prompt for the agent's loop. + */ + systemPrompt?: string; + }): Promise; + /** * Brings page to front (activates tab). */ @@ -5204,6 +5294,243 @@ export interface Page { [Symbol.asyncDispose](): Promise; } +/** + * + */ +export interface PageAgent { + /** + * Extract information from the page using the agentic loop, return it in a given Zod format. + * + * **Usage** + * + * ```js + * await agent.extract('List of items in the cart', z.object({ + * title: z.string().describe('Item title to extract'), + * price: z.string().describe('Item price to extract'), + * }).array()); + * ``` + * + * @param query Task to perform using agentic loop. + * @param schema + * @param options + */ + extract(query: string, schema: Schema): Promise<{ result: InferZodSchema, usage: { turns: number, inputTokens: number, outputTokens: number } }>; + /** + * Emitted when the agent makes a turn. + */ + on(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. + */ + once(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + addListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + removeListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + off(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + prependListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Dispose this agent. + */ + dispose(): Promise; + + /** + * Expect certain condition to be met. + * + * **Usage** + * + * ```js + * await agent.expect('"0 items" to be reported'); + * ``` + * + * @param expectation Expectation to assert. + * @param options + */ + expect(expectation: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + cacheKey?: string; + + /** + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. + */ + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. + */ + timeout?: number; + }): Promise; + + /** + * Perform action using agentic loop. + * + * **Usage** + * + * ```js + * await agent.perform('Click submit button'); + * ``` + * + * @param task Task to perform using agentic loop. + * @param options + */ + perform(task: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + cacheKey?: string; + + /** + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. + */ + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. + */ + timeout?: number; + }): Promise<{ + usage: { + turns: number; + + inputTokens: number; + + outputTokens: number; + }; + }>; + + /** + * Returns the current token usage for this agent. + * + * **Usage** + * + * ```js + * const usage = await agent.usage(); + * console.log(`Tokens used: ${usage.inputTokens} in, ${usage.outputTokens} out`); + * ``` + * + */ + usage(): Promise<{ + turns: number; + + inputTokens: number; + + outputTokens: number; + }>; + + [Symbol.asyncDispose](): Promise; +} + /** * At every point of time, page exposes its current frame tree via the * [page.mainFrame()](https://playwright.dev/docs/api/class-page#page-main-frame) and diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index b8c2651ee9370..adc7654de9599 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -847,7 +847,6 @@ export class Page extends ChannelOwner implements api.Page return result.pdf; } - // @ts-expect-error agents are hidden async agent(options: Parameters[0] = {}) { const params: channels.PageAgentParams = { api: options.provider?.api, @@ -862,7 +861,6 @@ export class Page extends ChannelOwner implements api.Page maxTokens: options.limits?.maxTokens, maxActions: options.limits?.maxActions, maxActionRetries: options.limits?.maxActionRetries, - // @ts-expect-error runAgents is hidden secrets: options.secrets ? Object.entries(options.secrets).map(([name, value]) => ({ name, value })) : undefined, systemPrompt: options.systemPrompt, }; diff --git a/packages/playwright-core/src/client/pageAgent.ts b/packages/playwright-core/src/client/pageAgent.ts index 960867f758560..bcf0cdb839d98 100644 --- a/packages/playwright-core/src/client/pageAgent.ts +++ b/packages/playwright-core/src/client/pageAgent.ts @@ -22,7 +22,6 @@ import { Page } from './page'; import type * as api from '../../types/types'; import type * as channels from '@protocol/channels'; -// @ts-expect-error runAgents is hidden export class PageAgent extends ChannelOwner implements api.PageAgent { private _page: Page; _expectTimeout?: number; diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index f6e6165885f0a..50dd490708e13 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -27,6 +27,13 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector state?: 'visible'|'attached'; }; +// @ts-ignore this will be any if zod is not installed +import { ZodTypeAny, z } from 'zod'; +// @ts-ignore this will be any if zod is not installed +import * as z3 from 'zod/v3'; +type ZodSchema = ZodTypeAny | z3.ZodTypeAny; +type InferZodSchema = T extends z3.ZodTypeAny ? z3.infer : T extends ZodTypeAny ? z.infer : never; + /** * Page provides methods to interact with a single tab in a [Browser](https://playwright.dev/docs/api/class-browser), * or an [extension background page](https://developer.chrome.com/extensions/background_pages) in Chromium. One @@ -2088,6 +2095,89 @@ export interface Page { url?: string; }): Promise; + /** + * Initialize page agent with the llm provider and cache. + * @param options + */ + agent(options?: { + cache?: { + /** + * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default). + */ + cacheFile?: string; + + /** + * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`. + */ + cacheOutFile?: string; + }; + + expect?: { + /** + * Default timeout for expect calls in milliseconds, defaults to 5000ms. + */ + timeout?: number; + }; + + /** + * Limits to use for the agentic loop. + */ + limits?: { + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to unlimited. + */ + maxTokens?: number; + + /** + * Maximum number of agentic actions to generate, defaults to 10. + */ + maxActions?: number; + + /** + * Maximum number retries per action, defaults to 3. + */ + maxActionRetries?: number; + }; + + provider?: { + /** + * API to use. + */ + api: "openai"|"openai-compatible"|"anthropic"|"google"; + + /** + * Endpoint to use if different from default. + */ + apiEndpoint?: string; + + /** + * API key for the LLM provider. + */ + apiKey: string; + + /** + * Amount of time to wait for the provider to respond to each request. + */ + apiTimeout?: number; + + /** + * Model identifier within the provider. Required in non-cache mode. + */ + model: string; + }; + + /** + * Secrets to hide from the LLM. + */ + secrets?: { [key: string]: string; }; + + /** + * System prompt for the agent's loop. + */ + systemPrompt?: string; + }): Promise; + /** * Brings page to front (activates tab). */ @@ -5204,6 +5294,243 @@ export interface Page { [Symbol.asyncDispose](): Promise; } +/** + * + */ +export interface PageAgent { + /** + * Extract information from the page using the agentic loop, return it in a given Zod format. + * + * **Usage** + * + * ```js + * await agent.extract('List of items in the cart', z.object({ + * title: z.string().describe('Item title to extract'), + * price: z.string().describe('Item price to extract'), + * }).array()); + * ``` + * + * @param query Task to perform using agentic loop. + * @param schema + * @param options + */ + extract(query: string, schema: Schema): Promise<{ result: InferZodSchema, usage: { turns: number, inputTokens: number, outputTokens: number } }>; + /** + * Emitted when the agent makes a turn. + */ + on(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Adds an event listener that will be automatically removed after it is triggered once. See `addListener` for more information about this event. + */ + once(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + addListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + removeListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Removes an event listener added by `on` or `addListener`. + */ + off(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Emitted when the agent makes a turn. + */ + prependListener(event: 'turn', listener: (data: { + role: string; + + message: string; + + usage?: { + inputTokens: number; + + outputTokens: number; + }; + }) => any): this; + + /** + * Dispose this agent. + */ + dispose(): Promise; + + /** + * Expect certain condition to be met. + * + * **Usage** + * + * ```js + * await agent.expect('"0 items" to be reported'); + * ``` + * + * @param expectation Expectation to assert. + * @param options + */ + expect(expectation: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + cacheKey?: string; + + /** + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. + */ + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in + * the config, or by specifying the `expect` property of the + * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable + * timeout. + */ + timeout?: number; + }): Promise; + + /** + * Perform action using agentic loop. + * + * **Usage** + * + * ```js + * await agent.perform('Click submit button'); + * ``` + * + * @param task Task to perform using agentic loop. + * @param options + */ + perform(task: string, options?: { + /** + * All the agentic actions are converted to the Playwright calls and are cached. By default, they are cached globally + * with the `task` as a key. This option allows controlling the cache key explicitly. + */ + cacheKey?: string; + + /** + * Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` + * property. + */ + maxActionRetries?: number; + + /** + * Maximum number of agentic actions to generate, defaults to context-wide value specified in `agent` property. + */ + maxActions?: number; + + /** + * Maximum number of tokens to consume. The agentic loop will stop after input + output tokens exceed this value. + * Defaults to context-wide value specified in `agent` property. + */ + maxTokens?: number; + + /** + * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in + * the config, or by using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + * Pass `0` to disable timeout. + */ + timeout?: number; + }): Promise<{ + usage: { + turns: number; + + inputTokens: number; + + outputTokens: number; + }; + }>; + + /** + * Returns the current token usage for this agent. + * + * **Usage** + * + * ```js + * const usage = await agent.usage(); + * console.log(`Tokens used: ${usage.inputTokens} in, ${usage.outputTokens} out`); + * ``` + * + */ + usage(): Promise<{ + turns: number; + + inputTokens: number; + + outputTokens: number; + }>; + + [Symbol.asyncDispose](): Promise; +} + /** * At every point of time, page exposes its current frame tree via the * [page.mainFrame()](https://playwright.dev/docs/api/class-page#page-main-frame) and diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 1460935c07006..82d88aafed229 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -112,8 +112,7 @@ export class FullConfigInternal { quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false), reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]), reportSlowTests: takeFirst(userConfig.reportSlowTests, { max: 5, threshold: 300_000 /* 5 minutes */ }), - // @ts-expect-error runAgents is hidden - runAgents: takeFirst(configCLIOverrides.runAgents, 'none'), + runAgents: takeFirst(configCLIOverrides.runAgents, userConfig.runAgents, 'none'), shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null), tags: globalTags, updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'), @@ -197,7 +196,7 @@ export class FullProjectInternal { testDir, snapshotDir: takeFirst(pathResolve(configDir, projectConfig.snapshotDir), pathResolve(configDir, config.snapshotDir), testDir), testIgnore: takeFirst(projectConfig.testIgnore, config.testIgnore, []), - testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/*.@(spec|test).?(c|m)[jt]s?(x)'), + testMatch: takeFirst(projectConfig.testMatch, config.testMatch, '**/*.@(spec|test).{md,?(c|m)[jt]s?(x)}'), timeout: takeFirst(configCLIOverrides.debug ? 0 : undefined, configCLIOverrides.timeout, projectConfig.timeout, config.timeout, defaultTimeout), use: mergeObjects(config.use, projectConfig.use, configCLIOverrides.use), dependencies: projectConfig.dependencies || [], diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index e986e71b3374b..874ba50ef2d00 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -58,9 +58,6 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & { _setupContextOptions: void; _setupArtifacts: void; _contextFactory: (options?: BrowserContextOptions) => Promise<{ context: BrowserContext, close: () => Promise }>; - - agent: {}; - agentOptions?: any; }; type WorkerFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & { @@ -155,7 +152,7 @@ const playwrightFixtures: Fixtures = ({ }, { option: true, box: true }], serviceWorkers: [({ contextOptions }, use) => use(contextOptions.serviceWorkers ?? 'allow'), { option: true, box: true }], contextOptions: [{}, { option: true, box: true }], - agentOptions: [undefined, { option: true, box: true }], + agentOptions: [({}, use) => use(undefined), { option: true, box: true }], _combinedContextOptions: [async ({ acceptDownloads, @@ -462,11 +459,9 @@ const playwrightFixtures: Fixtures = ({ const testInfoImpl = testInfo as TestInfoImpl; const cachePathTemplate = agentOptions?.cachePathTemplate ?? '{testDir}/{testFilePath}-cache.json'; const resolvedCacheFile = testInfoImpl._applyPathTemplate(cachePathTemplate, '', '.json'); - // @ts-expect-error runAgents is hidden const cacheFile = testInfoImpl.config.runAgents === 'all' ? undefined : await testInfoImpl._cloneStorage(resolvedCacheFile); const cacheOutFile = path.join(testInfoImpl.artifactsDir(), 'agent-cache-' + createGuid() + '.json'); - // @ts-expect-error runAgents is hidden const provider = agentOptions?.provider && testInfo.config.runAgents !== 'none' ? agentOptions.provider : undefined; if (provider) testInfo.setTimeout(0); @@ -476,7 +471,6 @@ const playwrightFixtures: Fixtures = ({ cacheOutFile, }; - // @ts-expect-error agent is hidden const agent = await page.agent({ provider, cache, diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 458e177c162af..b4829257f84b4 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -777,7 +777,6 @@ export const baseFullConfig: reporterTypes.FullConfig = { tags: [], updateSnapshots: 'missing', updateSourceMethod: 'patch', - // @ts-expect-error runAgents is hidden runAgents: 'none', version: '', workers: 0, diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index e1fc9caf3c661..da4ad0f6483e3 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -424,6 +424,7 @@ const testOptions: [string, { description: string, choices?: string[], preset?: ['--repeat-each ', { description: `Run each test N times (default: 1)` }], ['--reporter ', { description: `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")` }], ['--retries ', { description: `Maximum retry count for flaky tests, zero for no retries (default: no retries)` }], + ['--run-agents ', { description: `Run agents to generate the code for page.perform`, choices: ['missing', 'all', 'none'], preset: 'none' }], ['--shard ', { description: `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"` }], ['--test-list ', { description: `Path to a file containing a list of tests to run. See https://playwright.dev/docs/test-cli for more details.` }], ['--test-list-invert ', { description: `Path to a file containing a list of tests to skip. See https://playwright.dev/docs/test-cli for more details.` }], diff --git a/packages/playwright/src/transform/transform.ts b/packages/playwright/src/transform/transform.ts index 3cab05d844399..862023cc5f9b8 100644 --- a/packages/playwright/src/transform/transform.ts +++ b/packages/playwright/src/transform/transform.ts @@ -224,7 +224,7 @@ export function transformHook(originalCode: string, filename: string, moduleUrl? // TODO: ideally, we would not transform before checking the cache. However, the source // currently depends on the seed.md, so "originalCode" is not enough to produce a cache key. let inputSourceMap: EncodedSourceMap | undefined; - if (filename.endsWith('.md') && false) { + if (filename.endsWith('.md')) { const transformed = transformMDToTS(originalCode, filename); originalCode = transformed.code; inputSourceMap = transformed.map; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 31caf199d7e3c..33409b8445e3a 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; +import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, PageAgent, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; @@ -1611,6 +1611,14 @@ interface TestConfig { */ retries?: number; + /** + * Whether to run LLM agent for [PageAgent](https://playwright.dev/docs/api/class-pageagent): + * - "all" disregards existing cache and performs all actions via LLM + * - "missing" only performs actions that don't have generated cache actions + * - "none" does not talk to LLM at all, relies on the cached actions (default) + */ + runAgents?: "all"|"missing"|"none"; + /** * Shard tests and execute only the selected shard. Specify in the one-based form like `{ total: 5, current: 2 }`. * @@ -2067,6 +2075,14 @@ export interface FullConfig { */ rootDir: string; + /** + * Whether to run LLM agent for [PageAgent](https://playwright.dev/docs/api/class-pageagent): + * - "all" disregards existing cache and performs all actions via LLM + * - "missing" only performs actions that don't have generated cache actions + * - "none" does not talk to LLM at all, relies on the cached actions (default) + */ + runAgents: "all"|"missing"|"none"; + /** * See [testConfig.shard](https://playwright.dev/docs/api/class-testconfig#test-config-shard). */ @@ -6934,6 +6950,24 @@ export interface PlaywrightWorkerOptions { export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type AgentOptions = { + provider?: { + api: 'openai' | 'openai-compatible' | 'anthropic' | 'google'; + apiEndpoint?: string; + apiKey: string; + apiTimeout?: number; + model: string; + }, + limits?: { + maxTokens?: number; + maxActions?: number; + maxActionRetries?: number; + }; + cachePathTemplate?: string; + runAgents?: 'all' | 'missing' | 'none'; + secrets?: { [key: string]: string }; + systemPrompt?: string; +}; /** * Playwright Test provides many options to configure test environment, @@ -6974,6 +7008,7 @@ export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; * */ export interface PlaywrightTestOptions { + agentOptions: AgentOptions | undefined; /** * Whether to automatically download all the attachments. Defaults to `true` where all the downloads are accepted. * @@ -7683,6 +7718,7 @@ export interface PlaywrightTestArgs { * */ request: APIRequestContext; + agent: PageAgent; } type ExcludeProps = { diff --git a/tests/library/agent-helpers.ts b/tests/library/agent-helpers.ts index a68988ed7f6c6..5a718082df48d 100644 --- a/tests/library/agent-helpers.ts +++ b/tests/library/agent-helpers.ts @@ -18,7 +18,6 @@ import fs from 'fs'; import path from 'path'; import { browserTest as test } from '../config/browserTest'; -// @ts-expect-error import type { BrowserContext, Page, PageAgent } from '@playwright/test'; export function cacheFile() { @@ -33,14 +32,12 @@ export async function setCacheObject(object: any) { await fs.promises.writeFile(cacheFile(), JSON.stringify(object, null, 2), 'utf8'); } -// @ts-expect-error type AgentOptions = Parameters[0]; export async function generateAgent(context: BrowserContext, options: AgentOptions = {}) { const apiCacheFile = path.join(__dirname, '__llm_cache__', sanitizeFileName(test.info().titlePath.join(' ')) + '.json'); const page = await context.newPage(); - // @ts-expect-error const agent = await page.agent({ provider: { api: 'anthropic' as const, @@ -60,7 +57,6 @@ export async function generateAgent(context: BrowserContext, options: AgentOptio export async function runAgent(context: BrowserContext, options: AgentOptions = {}) { const page = await context.newPage(); - // @ts-expect-error const agent = await page.agent({ ...options, cache: { cacheFile: cacheFile() }, diff --git a/tests/library/agent-perform.spec.ts b/tests/library/agent-perform.spec.ts index 2961178490ed5..915915cd80a05 100644 --- a/tests/library/agent-perform.spec.ts +++ b/tests/library/agent-perform.spec.ts @@ -230,14 +230,12 @@ test('empty cache file works', async ({ context }) => { }); test('missing apiKey throws a nice error', async ({ page }) => { - // @ts-expect-error const agent = await page.agent({ provider: { api: 'anthropic', model: 'some model' } as any }); const error = await agent.perform('click the Test button').catch(e => e); expect(error.message).toContain(`This action requires API key to be set on the page agent`); }); test('malformed apiEndpoint throws a nice error', async ({ page }) => { - // @ts-expect-error const agent = await page.agent({ provider: { api: 'anthropic', model: 'some model', apiKey: 'some key', apiEndpoint: 'foobar' } }); const error = await agent.perform('click the Test button').catch(e => e); expect(error.message).toContain(`Agent API endpoint "foobar" is not a valid URL`); diff --git a/utils/doclint/api_parser.js b/utils/doclint/api_parser.js index 28e67114509e5..a4d504ea48d6e 100644 --- a/utils/doclint/api_parser.js +++ b/utils/doclint/api_parser.js @@ -93,9 +93,6 @@ class ApiParser { if (!match) throw new Error('Invalid member: ' + spec.text); const metainfo = extractMetainfo(spec); - if (metainfo.hidden) - return; - const name = match[3]; let returnType = null; let optional = false; @@ -128,6 +125,8 @@ class ApiParser { const clazz = /** @type {docs.Class} */(this.classes.get(match[2])); if (!clazz) throw new Error(`Unknown class ${match[2]} for member: ` + spec.text); + if (metainfo.hidden) + return; const existingMember = clazz.membersArray.find(m => m.name === name && m.kind === member.kind); if (existingMember && isTypeOverride(existingMember, member)) { @@ -147,9 +146,6 @@ class ApiParser { const match = spec.text.match(/(param|option): (.*)/); if (!match) throw `Something went wrong with matching ${spec.text}`; - const metainfo = extractMetainfo(spec); - if (metainfo.hidden) - return null; // For "test.describe.only.title": // - className is "test" diff --git a/utils/doclint/missingDocs.js b/utils/doclint/missingDocs.js index 0eca3a2cbee89..856c4f02101b6 100644 --- a/utils/doclint/missingDocs.js +++ b/utils/doclint/missingDocs.js @@ -29,16 +29,12 @@ module.exports = function lint(documentation, jsSources, apiFileName) { documentation.copyDocsFromSuperclasses(errors); const apiMethods = listMethods(jsSources, apiFileName); for (const [className, methods] of apiMethods) { - if (className === 'PageAgent') - continue; const docClass = documentation.classes.get(className); if (!docClass) { errors.push(`Missing documentation for "${className}"`); continue; } for (const [methodName, params] of methods) { - if (className === 'Page' && methodName === 'agent') - continue; const members = docClass.membersArray.filter(m => m.alias === methodName && m.kind !== 'event'); if (!members.length) { errors.push(`Missing documentation for "${className}.${methodName}"`); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 7b46b3402b491..e71d6ebbf7790 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; +import type { APIRequestContext, Browser, BrowserContext, BrowserContextOptions, Page, PageAgent, LaunchOptions, ViewportSize, Geolocation, HTTPCredentials, Locator, APIResponse, PageScreenshotOptions } from 'playwright-core'; export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; @@ -265,8 +265,27 @@ export interface PlaywrightWorkerOptions { export type ScreenshotMode = 'off' | 'on' | 'only-on-failure' | 'on-first-failure'; export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure'; export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry'; +export type AgentOptions = { + provider?: { + api: 'openai' | 'openai-compatible' | 'anthropic' | 'google'; + apiEndpoint?: string; + apiKey: string; + apiTimeout?: number; + model: string; + }, + limits?: { + maxTokens?: number; + maxActions?: number; + maxActionRetries?: number; + }; + cachePathTemplate?: string; + runAgents?: 'all' | 'missing' | 'none'; + secrets?: { [key: string]: string }; + systemPrompt?: string; +}; export interface PlaywrightTestOptions { + agentOptions: AgentOptions | undefined; acceptDownloads: boolean; bypassCSP: boolean; colorScheme: ColorScheme; @@ -305,6 +324,7 @@ export interface PlaywrightTestArgs { context: BrowserContext; page: Page; request: APIRequestContext; + agent: PageAgent; } type ExcludeProps = { diff --git a/utils/generate_types/overrides.d.ts b/utils/generate_types/overrides.d.ts index a88c80ba6ecdf..09b54a6dd7cd9 100644 --- a/utils/generate_types/overrides.d.ts +++ b/utils/generate_types/overrides.d.ts @@ -26,6 +26,13 @@ type ElementHandleWaitForSelectorOptionsNotHidden = ElementHandleWaitForSelector state?: 'visible'|'attached'; }; +// @ts-ignore this will be any if zod is not installed +import { ZodTypeAny, z } from 'zod'; +// @ts-ignore this will be any if zod is not installed +import * as z3 from 'zod/v3'; +type ZodSchema = ZodTypeAny | z3.ZodTypeAny; +type InferZodSchema = T extends z3.ZodTypeAny ? z3.infer : T extends ZodTypeAny ? z.infer : never; + export interface Page { evaluate(pageFunction: PageFunction, arg: Arg): Promise; evaluate(pageFunction: PageFunction, arg?: any): Promise; @@ -74,6 +81,10 @@ export interface Page { }): Promise; } +export interface PageAgent { + extract(query: string, schema: Schema): Promise<{ result: InferZodSchema, usage: { turns: number, inputTokens: number, outputTokens: number } }>; +} + export interface Frame { evaluate(pageFunction: PageFunction, arg: Arg): Promise; evaluate(pageFunction: PageFunction, arg?: any): Promise;