chore: bring some clarity to aria snapshot options

Author: dgozman

packages/injected/src/ariaSnapshot.ts

@@ -48,9 +48,47 @@ type AriaRef = {
 
 let lastRef = 0;
 
-export type AriaTreeOptions = { forAI?: boolean, refPrefix?: string, refs?: boolean, visibleOnly?: boolean };
+export type AriaTreeOptions = {
+  mode: 'ai' | 'expect' | 'codegen' | 'autoexpect';
+  refPrefix?: string;
+};
+
+type InternalOptions = {
+  visibility: 'aria' | 'ariaOrVisible' | 'ariaAndVisible',
+  refs: 'all' | 'interactable' | 'none',
+  refPrefix?: string,
+  includeGenericRole?: boolean,
+  renderCursorPointer?: boolean,
+  renderActive?: boolean,
+  renderStringsAsRegex?: boolean,
+};
 
-export function generateAriaTree(rootElement: Element, options?: AriaTreeOptions): AriaSnapshot {
+function toInternalOptions(options: AriaTreeOptions): InternalOptions {
+  if (options.mode === 'ai') {
+    // For AI consumption.
+    return {
+      visibility: 'ariaOrVisible',
+      refs: 'interactable',
+      refPrefix: options.refPrefix,
+      includeGenericRole: true,
+      renderActive: true,
+      renderCursorPointer: true,
+    };
+  }
+  if (options.mode === 'autoexpect') {
+    // To auto-generate assertions on visible elements.
+    return { visibility: 'ariaAndVisible', refs: 'all' };
+  }
+  if (options.mode === 'codegen') {
+    // To generate aria assertion with regex heurisitcs.
+    return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true };
+  }
+  // To match aria snapshot.
+  return { visibility: 'aria', refs: 'none' };
+}
+
+export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot {
+  const options = toInternalOptions(publicOptions);
   const visited = new Set<Node>();
 
   const snapshot: AriaSnapshot = {
@@ -79,8 +117,16 @@ export function generateAriaTree(rootElement: Element, options?: AriaTreeOptions
       return;
 
     const element = node as Element;
-    const isElementHiddenForAria = roleUtils.isElementHiddenForAria(element);
-    if (isElementHiddenForAria && !options?.forAI)
+    const isElementVisibleForAria = !roleUtils.isElementHiddenForAria(element);
+    let visible = isElementVisibleForAria;
+    if (options.visibility === 'ariaOrVisible')
+      visible = isElementVisibleForAria || isElementVisible(element);
+    if (options.visibility === 'ariaAndVisible')
+      visible = isElementVisibleForAria && isElementVisible(element);
+
+    // Optimization: if we only consider aria visibility, we can skip child elements because
+    // they will not be visible for aria as well.
+    if (options.visibility === 'aria' && !visible)
       return;
 
     const ariaChildren: Element[] = [];
@@ -93,7 +139,6 @@ export function generateAriaTree(rootElement: Element, options?: AriaTreeOptions
       }
     }
 
-    const visible = options?.visibleOnly ? isElementVisible(element) : !isElementHiddenForAria || isElementVisible(element);
     const childAriaNode = visible ? toAriaNode(element, options) : null;
     if (childAriaNode) {
       if (childAriaNode.ref) {
@@ -157,36 +202,39 @@ export function generateAriaTree(rootElement: Element, options?: AriaTreeOptions
   return snapshot;
 }
 
-function ariaRef(element: Element, role: string, name: string, options?: AriaTreeOptions): string | undefined {
-  if (!options?.forAI && !options?.refs)
-    return undefined;
+function computeAriaRef(ariaNode: AriaNode, options: InternalOptions) {
+  if (options.refs === 'none')
+    return;
+  if (options.refs === 'interactable' && (!ariaNode.box.visible || !ariaNode.receivesPointerEvents))
+    return;
 
   let ariaRef: AriaRef | undefined;
-  ariaRef = (element as any)._ariaRef;
-  if (!ariaRef || ariaRef.role !== role || ariaRef.name !== name) {
-    ariaRef = { role, name, ref: (options?.refPrefix ?? '') + 'e' + (++lastRef) };
-    (element as any)._ariaRef = ariaRef;
+  ariaRef = (ariaNode.element as any)._ariaRef;
+  if (!ariaRef || ariaRef.role !== ariaNode.role || ariaRef.name !== ariaNode.name) {
+    ariaRef = { role: ariaNode.role, name: ariaNode.name, ref: (options.refPrefix ?? '') + 'e' + (++lastRef) };
+    (ariaNode.element as any)._ariaRef = ariaRef;
   }
-  return ariaRef.ref;
+  ariaNode.ref = ariaRef.ref;
 }
 
-function toAriaNode(element: Element, options?: AriaTreeOptions): AriaNode | null {
+function toAriaNode(element: Element, options: InternalOptions): AriaNode | null {
   const active = element.ownerDocument.activeElement === element;
   if (element.nodeName === 'IFRAME') {
-    return {
+    const ariaNode: AriaNode = {
       role: 'iframe',
       name: '',
-      ref: ariaRef(element, 'iframe', '', options),
       children: [],
       props: {},
       element,
       box: box(element),
       receivesPointerEvents: true,
       active
     };
+    computeAriaRef(ariaNode, options);
+    return ariaNode;
   }
 
-  const defaultRole = options?.forAI ? 'generic' : null;
+  const defaultRole = options.includeGenericRole ? 'generic' : null;
   const role = roleUtils.getAriaRole(element) ?? defaultRole;
   if (!role || role === 'presentation' || role === 'none')
     return null;
@@ -197,14 +245,14 @@ function toAriaNode(element: Element, options?: AriaTreeOptions): AriaNode | nul
   const result: AriaNode = {
     role,
     name,
-    ref: ariaRef(element, role, name, options),
     children: [],
     props: {},
     element,
     box: box(element),
     receivesPointerEvents,
     active
   };
+  computeAriaRef(result, options);
 
   if (roleUtils.kAriaCheckedRoles.includes(role))
     result.checked = roleUtils.getAriaChecked(element);
@@ -245,7 +293,7 @@ function normalizeGenericRoles(node: AriaNode) {
     }
 
     // Only remove generic that encloses one element, logical grouping still makes sense, even if it is not ref-able.
-    const removeSelf = node.role === 'generic' && result.length <= 1 && result.every(c => typeof c !== 'string' && receivesPointerEvents(c));
+    const removeSelf = node.role === 'generic' && result.length <= 1 && result.every(c => typeof c !== 'string' && !!c.ref);
     if (removeSelf)
       return result;
     node.children = result;
@@ -308,20 +356,20 @@ export type MatcherReceived = {
   regex: string;
 };
 
-export function matchesAriaTree(rootElement: Element, template: AriaTemplateNode): { matches: AriaNode[], received: MatcherReceived } {
-  const snapshot = generateAriaTree(rootElement);
+export function matchesExpectAriaTemplate(rootElement: Element, template: AriaTemplateNode): { matches: AriaNode[], received: MatcherReceived } {
+  const snapshot = generateAriaTree(rootElement, { mode: 'expect' });
   const matches = matchesNodeDeep(snapshot.root, template, false, false);
   return {
     matches,
     received: {
-      raw: renderAriaTree(snapshot, { mode: 'raw' }),
-      regex: renderAriaTree(snapshot, { mode: 'regex' }),
+      raw: renderAriaTree(snapshot, { mode: 'expect' }),
+      regex: renderAriaTree(snapshot, { mode: 'codegen' }),
     }
   };
 }
 
-export function getAllByAria(rootElement: Element, template: AriaTemplateNode): Element[] {
-  const root = generateAriaTree(rootElement).root;
+export function getAllElementsMatchingExpectAriaTemplate(rootElement: Element, template: AriaTemplateNode): Element[] {
+  const root = generateAriaTree(rootElement, { mode: 'expect' }).root;
   const matches = matchesNodeDeep(root, template, true, false);
   return matches.map(n => n.element);
 }
@@ -411,10 +459,11 @@ function matchesNodeDeep(root: AriaNode, template: AriaTemplateNode, collectAll:
   return results;
 }
 
-export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'raw' | 'regex', forAI?: boolean, refs?: boolean }): string {
+export function renderAriaTree(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions): string {
+  const options = toInternalOptions(publicOptions);
   const lines: string[] = [];
-  const includeText = options?.mode === 'regex' ? textContributesInfo : () => true;
-  const renderString = options?.mode === 'regex' ? convertToBestGuessRegex : (str: string) => str;
+  const includeText = options.renderStringsAsRegex ? textContributesInfo : () => true;
+  const renderString = options.renderStringsAsRegex ? convertToBestGuessRegex : (str: string) => str;
   const visit = (ariaNode: AriaNode | string, parentAriaNode: AriaNode | null, indent: string) => {
     if (typeof ariaNode === 'string') {
       if (parentAriaNode && !includeText(parentAriaNode, ariaNode))
@@ -442,7 +491,7 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'r
       key += ` [disabled]`;
     if (ariaNode.expanded)
       key += ` [expanded]`;
-    if (ariaNode.active && options?.forAI)
+    if (ariaNode.active && options.renderActive)
       key += ` [active]`;
     if (ariaNode.level)
       key += ` [level=${ariaNode.level}]`;
@@ -453,10 +502,9 @@ export function renderAriaTree(ariaSnapshot: AriaSnapshot, options?: { mode?: 'r
     if (ariaNode.selected === true)
       key += ` [selected]`;
 
-    const includeRef = (options?.forAI && receivesPointerEvents(ariaNode)) || options?.refs;
-    if (includeRef && ariaNode.ref) {
+    if (ariaNode.ref) {
       key += ` [ref=${ariaNode.ref}]`;
-      if (hasPointerCursor(ariaNode))
+      if (options.renderCursorPointer && hasPointerCursor(ariaNode))
         key += ' [cursor=pointer]';
     }
 
@@ -548,10 +596,6 @@ function textContributesInfo(node: AriaNode, text: string): boolean {
   return filtered.trim().length / text.length > 0.1;
 }
 
-function receivesPointerEvents(ariaNode: AriaNode): boolean {
-  return ariaNode.box.visible && ariaNode.receivesPointerEvents;
-}
-
 function hasPointerCursor(ariaNode: AriaNode): boolean {
   return ariaNode.box.style?.cursor === 'pointer';
 }

packages/injected/src/consoleApi.ts

@@ -91,8 +91,8 @@ export class ConsoleAPI {
       inspect: (selector: string) => this._inspect(selector),
       selector: (element: Element) => this._selector(element),
       generateLocator: (element: Element, language?: Language) => this._generateLocator(element, language),
-      ariaSnapshot: (element?: Element, options?: { forAI?: boolean }) => {
-        return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, options);
+      ariaSnapshot: (element?: Element) => {
+        return this._injectedScript.ariaSnapshot(element || this._injectedScript.document.body, { mode: 'expect' });
       },
       resume: () => this._resume(),
       ...new Locator(this._injectedScript, ''),

packages/injected/src/injectedScript.ts

@@ -19,7 +19,7 @@ import { asLocator } from '@isomorphic/locatorGenerators';
 import { parseAttributeSelector, parseSelector, stringifySelector, visitAllSelectorParts } from '@isomorphic/selectorParser';
 import { cacheNormalizedWhitespaces, normalizeWhiteSpace, trimStringWithEllipsis } from '@isomorphic/stringUtils';
 
-import { generateAriaTree, getAllByAria, matchesAriaTree, renderAriaTree } from './ariaSnapshot';
+import { generateAriaTree, getAllElementsMatchingExpectAriaTemplate, matchesExpectAriaTemplate, renderAriaTree } from './ariaSnapshot';
 import { enclosingShadowRootOrDocument, isElementVisible, isInsideScope, parentElementOrShadowHost, setGlobalOptions } from './domUtils';
 import { Highlight } from './highlight';
 import { kLayoutSelectorNames, layoutSelectorScore } from './layoutSelectorUtils';
@@ -297,21 +297,21 @@ export class InjectedScript {
     return new Set<Element>(result.map(r => r.element));
   }
 
-  ariaSnapshot(node: Node, options?: { mode?: 'raw' | 'regex' } & AriaTreeOptions): string {
+  ariaSnapshot(node: Node, options: AriaTreeOptions): string {
     if (node.nodeType !== Node.ELEMENT_NODE)
       throw this.createStacklessError('Can only capture aria snapshot of Element nodes.');
     this._lastAriaSnapshot = generateAriaTree(node as Element, options);
     return renderAriaTree(this._lastAriaSnapshot, options);
   }
 
   ariaSnapshotForRecorder(): { ariaSnapshot: string, refs: Map<Element, string> } {
-    const tree = generateAriaTree(this.document.body, { forAI: true });
-    const ariaSnapshot = renderAriaTree(tree, { forAI: true });
+    const tree = generateAriaTree(this.document.body, { mode: 'ai' });
+    const ariaSnapshot = renderAriaTree(tree, { mode: 'ai' });
     return { ariaSnapshot, refs: tree.refs };
   }
 
-  getAllByAria(document: Document, template: AriaTemplateNode): Element[] {
-    return getAllByAria(document.documentElement, template);
+  getAllElementsMatchingExpectAriaTemplate(document: Document, template: AriaTemplateNode): Element[] {
+    return getAllElementsMatchingExpectAriaTemplate(document.documentElement, template);
   }
 
   querySelectorAll(selector: ParsedSelector, root: Node): Element[] {
@@ -1469,7 +1469,7 @@ export class InjectedScript {
 
     {
       if (expression === 'to.match.aria') {
-        const result = matchesAriaTree(element, options.expectedValue);
+        const result = matchesExpectAriaTemplate(element, options.expectedValue);
         return {
           received: result.received,
           matches: !!result.matches.length,

packages/injected/src/recorder/recorder.ts

@@ -548,7 +548,7 @@ class RecordActionTool implements RecorderTool {
   private _captureAriaSnapshotForAction(action: actions.Action) {
     const documentElement = this._recorder.injectedScript.document.documentElement;
     if (documentElement)
-      action.ariaSnapshot = this._recorder.injectedScript.ariaSnapshot(documentElement, { mode: 'raw', refs: true, visibleOnly: true });
+      action.ariaSnapshot = this._recorder.injectedScript.ariaSnapshot(documentElement, { mode: 'autoexpect' });
   }
 
   private _recordAction(action: actions.Action) {
@@ -951,7 +951,7 @@ class TextAssertionTool implements RecorderTool {
         name: 'assertSnapshot',
         selector: this._hoverHighlight.selector,
         signals: [],
-        ariaSnapshot: this._recorder.injectedScript.ariaSnapshot(target, { mode: 'regex' }),
+        ariaSnapshot: this._recorder.injectedScript.ariaSnapshot(target, { mode: 'codegen' }),
       };
     } else {
       const generated = this._recorder.injectedScript.generateSelector(target, { testIdAttributeName: this._recorder.state.testIdAttributeName, forTextExpect: true });
@@ -1384,7 +1384,7 @@ export class Recorder {
 
     const ariaTemplateJSON = JSON.stringify(state.ariaTemplate);
     if (this._lastHighlightedAriaTemplateJSON !== ariaTemplateJSON) {
-      const elements = state.ariaTemplate ? this.injectedScript.getAllByAria(this.document, state.ariaTemplate) : [];
+      const elements = state.ariaTemplate ? this.injectedScript.getAllElementsMatchingExpectAriaTemplate(this.document, state.ariaTemplate) : [];
       if (elements.length) {
         const color = elements.length > 1 ? HighlightColors.multiple : HighlightColors.single;
         highlight = elements.map(element => ({ element, color }));
@@ -1591,7 +1591,7 @@ export class Recorder {
   }
 
   elementPicked(selector: string, model: HighlightModel) {
-    const ariaSnapshot = this.injectedScript.ariaSnapshot(model.elements[0]);
+    const ariaSnapshot = this.injectedScript.ariaSnapshot(model.elements[0], { mode: 'expect' });
     void this._delegate.elementPicked?.({ selector, ariaSnapshot });
   }
 }

packages/playwright-core/src/protocol/validator.ts

@@ -1559,7 +1559,6 @@ scheme.FrameAddStyleTagResult = tObject({
 });
 scheme.FrameAriaSnapshotParams = tObject({
   selector: tString,
-  forAI: tOptional(tBoolean),
   timeout: tNumber,
 });
 scheme.FrameAriaSnapshotResult = tObject({

packages/playwright-core/src/server/dispatchers/frameDispatcher.ts

@@ -273,6 +273,6 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Br
   }
 
   async ariaSnapshot(params: channels.FrameAriaSnapshotParams, progress: Progress): Promise<channels.FrameAriaSnapshotResult> {
-    return { snapshot: await this._frame.ariaSnapshot(progress, params.selector, params) };
+    return { snapshot: await this._frame.ariaSnapshot(progress, params.selector) };
   }
 }

packages/playwright-core/src/server/dom.ts

@@ -750,8 +750,8 @@ export class ElementHandle<T extends Node = Node> extends js.JSHandle<T> {
     return this._page.delegate.getBoundingBox(this);
   }
 
-  async ariaSnapshot(options?: { forAI?: boolean, refPrefix?: string }): Promise<string> {
-    return await this.evaluateInUtility(([injected, element, options]) => injected.ariaSnapshot(element, options), options);
+  async ariaSnapshot(): Promise<string> {
+    return await this.evaluateInUtility(([injected, element]) => injected.ariaSnapshot(element, { mode: 'expect' }), {});
   }
 
   async screenshot(progress: Progress, options: ScreenshotOptions): Promise<Buffer> {

packages/playwright-core/src/server/frames.ts

@@ -1341,8 +1341,8 @@ export class Frame extends SdkObject {
     return progress.wait(timeout);
   }
 
-  async ariaSnapshot(progress: Progress, selector: string, options: { forAI?: boolean }): Promise<string> {
-    return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => progress.race(handle.ariaSnapshot(options)));
+  async ariaSnapshot(progress: Progress, selector: string): Promise<string> {
+    return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, true /* performActionPreChecks */, handle => progress.race(handle.ariaSnapshot()));
   }
 
   async expect(progress: Progress, selector: string | undefined, options: FrameExpectParams, timeout?: number): Promise<ExpectResult> {

packages/playwright-core/src/server/page.ts

@@ -1005,7 +1005,7 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, frame
         const node = injected.document.body;
         if (!node)
           return true;
-        return injected.ariaSnapshot(node, { forAI: true, refPrefix });
+        return injected.ariaSnapshot(node, { mode: 'ai', refPrefix });
       }, frameOrdinal ? 'f' + frameOrdinal : ''));
       if (snapshotOrRetry === true)
         return continuePolling;

packages/protocol/src/channels.d.ts

@@ -2750,11 +2750,10 @@ export type FrameAddStyleTagResult = {
 };
 export type FrameAriaSnapshotParams = {
   selector: string,
-  forAI?: boolean,
   timeout: number,
 };
 export type FrameAriaSnapshotOptions = {
-  forAI?: boolean,
+
 };
 export type FrameAriaSnapshotResult = {
   snapshot: string,