fix(visibility): ignore z-index of common parent

README.md

@@ -42,7 +42,7 @@ The script will first query all elements on the page (and inside shadow roots) t
 
 #### 2. Elements filtering
 
-The script will then first proceed to filter out, in both lists, the elements that are not visible enough (see [src/constants.ts](src/constants.ts) for the threshold values, e.g. `0.7`). To do that, we first use an [Intersection Observer](https://developer.mozilla.org/en-US/docs/Web/API/Intersection_Observer_API) to check if the element is visible enough in the viewport, and if it is, we find the elements that are possibly intersecting with the element, using a QuadTree that we previously built with all elements on the page by their bounding boxes. We then query the QuadTree for elements that are possibly intersecting with the element, and we draw them on a canvas, after drawing the original element. We then calculate the pixel-by-pixel visibility ratio by counting the number of pixels that were not overlapped by other elements. If the ratio is above the threshold, we consider the element visible enough.
+The script will then first proceed to filter out, in both lists, the elements that are not visible enough (see [src/constants.ts](src/constants.ts) for the threshold values, e.g. `0.7`). To do that, we first use an [Intersection Observer](https://developer.mozilla.org/en-US/docs/Web/API/Intersection_Observer_API) to check if the element is visible enough in the viewport, and if it is, we find the elements that are possibly intersecting with the element, using [rbush](https://npmjs/package/rbush) as a tree that we previously built with all elements on the page through their bounding boxes. We then query the tree for elements that are possibly intersecting with the element, and draw them on a canvas, after drawing the original element. Finally, we calculate the pixel-by-pixel visibility ratio by counting the number of pixels that were not overlapped by other elements. If the ratio is above the threshold, we consider the element visible enough.
 
 After that, we take the elements in the second list (the ones that display a pointer cursor) and apply a nesting filter. This filter will remove all elements that are either inside a prioritized element (e.g. a button) or that have too many clickable children. Additionally, we consider elements disjoint if their size is different enough (see [src/constants.ts](src/constants.ts) for the threshold value, e.g. `0.7`).
 When applying this filter, we also consider the first list for reference, while not removing any element from that first list afterwards.

package.json

@@ -1,16 +1,6 @@
 {
  "name": "@brewcoua/web-som",
  "version": "1.2.0",
- "description": "Set-of-Marks script for web grounding, used for web agent automation",
- "homepage": "https://github.com/brewcoua/web-som#readme",
- "license": "(MIT OR Apache-2.0)",
- "keywords": [
- "web",
- "agent",
- "automation",
- "set-of-marks",
- "som"
- ],
  "author": {
  "name": "Brewen Couaran",
  "email": "contact@brewen.dev",
@@ -21,12 +11,6 @@
  "url": "git+ssh://git@github.com/brewcoua/web-som.git"
  },
  "main": "./SoM.min.js",
- "files": [
- "SoM.min.js",
- "SoM.min.js.map",
- "SoM.js",
- "SoM.js.map"
- ],
  "devDependencies": {
  "@rollup/plugin-commonjs": "^26.0.1",
  "@rollup/plugin-node-resolve": "^15.2.3",
@@ -39,6 +23,7 @@
  "@types/rbush": "^3.0.3",
  "conventional-changelog-conventionalcommits": "^8.0.0",
  "prettier": "^3.2.5",
+ "rbush": "^3.0.1",
  "rollup": "^4.18.0",
  "rollup-plugin-string": "^3.0.0",
  "semantic-release": "^24.0.0",
@@ -49,11 +34,24 @@
  "url": "https://github.com/brewcoua/web-som/issues",
  "email": "contact@brewen.dev"
  },
+ "description": "Set-of-Marks script for web grounding, used for web agent automation",
+ "files": [
+ "SoM.min.js",
+ "SoM.min.js.map",
+ "SoM.js",
+ "SoM.js.map"
+ ],
+ "homepage": "https://github.com/brewcoua/web-som#readme",
+ "keywords": [
+ "web",
+ "agent",
+ "automation",
+ "set-of-marks",
+ "som"
+ ],
+ "license": "(MIT OR Apache-2.0)",
  "scripts": {
  "build": "rollup -c",
  "build:watch": "rollup -c -w"
- },
- "dependencies": {
- "rbush": "^3.0.1"
  }
 }

src/filters/visibility/canvas.ts

@@ -48,6 +48,7 @@ export default class VisibilityCanvas {
  if (totalPixels === 0) return 0;
 
  const elements = this.getIntersectingElements(qt);
+
  for (const el of elements) {
  this.drawElement(el, 'black');
  }

src/filters/visibility/utils.ts

@@ -13,20 +13,29 @@ export function isAbove(
  referenceElement: HTMLElement
 ): boolean {
  // Helper function to get the effective z-index value
- function getEffectiveZIndex(element: HTMLElement): number {
+ function getEffectiveZIndex(
+ element: HTMLElement,
+ other: HTMLElement
+ ): number {
  while (element) {
  const zIndex = window.getComputedStyle(element).zIndex;
  if (zIndex !== 'auto') {
  const zIndexValue = parseInt(zIndex, 10);
+
+ // Do not count the z-index of a common parent
+ if (element.contains(other)) {
+ return 0;
+ }
+
  return isNaN(zIndexValue) ? 0 : zIndexValue;
  }
  element = element.parentElement as HTMLElement;
  }
  return 0;
  }
 
- const elementZIndex = getEffectiveZIndex(element);
- const referenceElementZIndex = getEffectiveZIndex(referenceElement);
+ const elementZIndex = getEffectiveZIndex(element, referenceElement);
+ const referenceElementZIndex = getEffectiveZIndex(referenceElement, element);
 
  const elementPosition = element.compareDocumentPosition(referenceElement);