iframe testing & fix (#593)

* fix: broken iframe exploration

* chore: ruff

* fix: prevent test errors

---------

Co-authored-by: Alexis Deprez <deprez.alexis@laposte.net>

lavague-core/lavague/core/base_driver.py

@@ -479,8 +479,11 @@ def js_wrap_function_call(fn: str):
 })();"""
 
 JS_GET_INTERACTIVES = """
+const windowHeight = (window.innerHeight || document.documentElement.clientHeight);
+const windowWidth = (window.innerWidth || document.documentElement.clientWidth);
+
 return (function() {
-    function getInteractions(e) {
+    function getInteractions(e, in_viewport, foreground_only) {
         const tag = e.tagName.toLowerCase();
         if (!e.checkVisibility() || e.hasAttribute('disabled') || e.hasAttribute('readonly')
           || (tag === 'input' && e.getAttribute('type') === 'hidden') || tag === 'body') {
@@ -520,13 +523,42 @@ def js_wrap_function_call(fn: str):
         if (hasEvent('scroll') || hasEvent('wheel')|| e.scrollHeight > e.clientHeight || e.scrollWidth > e.clientWidth) {
             //evts.push('SCROLL');
         }
+
+        if (in_viewport == true) {
+            const rect = e.getBoundingClientRect();
+            let iframe = e.ownerDocument.defaultView.frameElement;
+            while (iframe) {
+                const iframeRect = iframe.getBoundingClientRect();
+                rect.top += iframeRect.top;
+                rect.left += iframeRect.left;
+                rect.bottom += iframeRect.top;
+                rect.right += iframeRect.left;
+                iframe = iframe.ownerDocument.defaultView.frameElement;
+            }
+            const elemCenter = {
+                x: rect.left + rect.width / 2,
+                y: rect.top + rect.height / 2
+            };
+            if (elemCenter.x < 0) return [];
+            if (elemCenter.x > windowWidth) return [];
+            if (elemCenter.y < 0) return [];
+            if (elemCenter.y > windowHeight) return [];
+            if (foreground_only !== true) return evts; // whenever to check for elements above
+            let pointContainer = document.elementFromPoint(elemCenter.x, elemCenter.y);
+            do {
+                if (pointContainer === element) return evts;
+                if (pointContainer == null) return evts;
+            } while (pointContainer = pointContainer.parentNode);
+            return [];
+        }
+
         return evts;
     }
 
     const results = {};
     function traverse(node, xpath) {
         if (node.nodeType === Node.ELEMENT_NODE) {
-            const interactions = getInteractions(node);
+            const interactions = getInteractions(node, arguments?.[0], arguments?.[1]);
             if (interactions.length > 0) {
                 results[xpath] = interactions;
             }
@@ -560,44 +592,6 @@ def js_wrap_function_call(fn: str):
 })();
 """
 
-JS_GET_INTERACTIVES_IN_VIEWPORT = (
-    """
-const windowHeight = (window.innerHeight || document.documentElement.clientHeight);
-const windowWidth = (window.innerWidth || document.documentElement.clientWidth);
-return Object.fromEntries(Object.entries("""
-    + js_wrap_function_call(JS_GET_INTERACTIVES)
-    + """).filter(([xpath, evts]) => {
-    const element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
-    if (!element) return false;
-    const rect = element.getBoundingClientRect();
-    let iframe = element.ownerDocument.defaultView.frameElement;
-    while (iframe) {
-        const iframeRect = iframe.getBoundingClientRect();
-        rect.top += iframeRect.top;
-        rect.left += iframeRect.left;
-        rect.bottom += iframeRect.top;
-        rect.right += iframeRect.left;
-        iframe = iframe.ownerDocument.defaultView.frameElement;
-    }
-    const elemCenter = {
-        x: rect.left + rect.width / 2,
-        y: rect.top + rect.height / 2
-    };
-    if (elemCenter.x < 0) return false;
-    if (elemCenter.x > windowWidth) return false;
-    if (elemCenter.y < 0) return false;
-    if (elemCenter.y > windowHeight) return false;
-    if (arguments?.[0] !== true) return true; // whenever to check for elements above
-    let pointContainer = document.elementFromPoint(elemCenter.x, elemCenter.y);
-    do {
-        if (pointContainer === element) return true;
-        if (pointContainer == null) return true;
-    } while (pointContainer = pointContainer.parentNode);
-    return false;
-}));
-"""
-)
-
 JS_WAIT_DOM_IDLE = """
 return new Promise(resolve => {
     const timeout = arguments[0] || 10000;

lavague-core/lavague/core/retrievers.py

@@ -192,7 +192,8 @@ def get_html_with_xpath(
                 filter_by_possible_interactions,
                 xpath_prefix + frame_xpath,
             )
-            iframe_tag.replace_with(frame_soup_str)
+            frame_soup = BeautifulSoup(frame_soup_str, "html.parser")
+            iframe_tag.replace_with(frame_soup)
             self.driver.switch_parent_frame()
         return str(soup)
 

lavague-integrations/drivers/lavague-drivers-playwright/lavague/drivers/playwright/base.py

@@ -8,7 +8,6 @@
 from lavague.core.base_driver import (
     BaseDriver,
     JS_GET_INTERACTIVES,
-    JS_GET_INTERACTIVES_IN_VIEWPORT,
     JS_WAIT_DOM_IDLE,
     PossibleInteractionsByXpath,
     InteractionType,
@@ -312,7 +311,8 @@ def get_possible_interactions(
         self, in_viewport=True, foreground_only=True
     ) -> PossibleInteractionsByXpath:
         exe: Dict[str, List[str]] = self.execute_script(
-            JS_GET_INTERACTIVES_IN_VIEWPORT if in_viewport else JS_GET_INTERACTIVES,
+            JS_GET_INTERACTIVES,
+            in_viewport,
             foreground_only,
         )
         res = dict()

lavague-integrations/drivers/lavague-drivers-selenium/lavague/drivers/selenium/base.py

@@ -1,3 +1,4 @@
+from abc import ABC
 from typing import Any, Optional, Callable, Mapping, Dict, List
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.common.by import By
@@ -15,7 +16,6 @@
 from lavague.core.base_driver import (
     BaseDriver,
     JS_GET_INTERACTIVES,
-    JS_GET_INTERACTIVES_IN_VIEWPORT,
     JS_WAIT_DOM_IDLE,
     JS_GET_SCROLLABLE_PARENT,
     PossibleInteractionsByXpath,
@@ -49,6 +49,20 @@
 )
 
 
+class XPathResolved(ABC):
+    def __init__(self, xpath: str, driver: any, element: WebElement) -> None:
+        self.xpath = xpath
+        self._driver = driver
+        self.element = element
+        super().__init__()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self._driver.switch_default_frame()
+
+
 class SeleniumDriver(BaseDriver):
     driver: WebDriver
     last_hover_xpath: Optional[str] = None
@@ -208,10 +222,13 @@ def maximize_window(self) -> None:
 
     def check_visibility(self, xpath: str) -> bool:
         try:
-            element = self.resolve_xpath(xpath)
-            return (
+            # Done manually here to avoid issues
+            element = self.resolve_xpath(xpath).element
+            res = (
                 element is not None and element.is_displayed() and element.is_enabled()
             )
+            self.switch_default_frame()
+            return res
         except:
             return False
 
@@ -277,17 +294,18 @@ def switch_default_frame(self) -> None:
     def switch_parent_frame(self) -> None:
         self.driver.switch_to.parent_frame()
 
-    def resolve_xpath(self, xpath: Optional[str]) -> WebElement:
+    def resolve_xpath(self, xpath: Optional[str]) -> XPathResolved:
         if not xpath:
             raise NoSuchElementException("xpath is missing")
         before, sep, after = xpath.partition("iframe")
         if len(before) == 0:
             return None
         if len(sep) == 0:
-            return self.driver.find_element(By.XPATH, before)
+            res = self.driver.find_element(By.XPATH, before)
+            res = XPathResolved(xpath, self, res)
+            return res
         self.switch_frame(before + sep)
         element = self.resolve_xpath(after)
-        self.switch_default_frame()
         return element
 
     def exec_code(
@@ -348,18 +366,23 @@ def code_for_execute_script(self, js_code: str, *args) -> str:
         )
 
     def hover(self, xpath: str):
-        element = self.resolve_xpath(xpath)
-        self.last_hover_xpath = xpath
-        ActionChains(self.driver).move_to_element(element).perform()
+        with self.resolve_xpath(xpath) as element_resolved:
+            self.last_hover_xpath = xpath
+            ActionChains(self.driver).move_to_element(
+                element_resolved.element
+            ).perform()
 
     def scroll_page(self, direction: ScrollDirection = ScrollDirection.DOWN):
         self.driver.execute_script(direction.get_page_script())
 
     def get_scroll_anchor(self, xpath_anchor: Optional[str] = None) -> WebElement:
-        element = self.resolve_xpath(xpath_anchor or self.last_hover_xpath)
-        parent = self.driver.execute_script(JS_GET_SCROLLABLE_PARENT, element)
-        scroll_anchor = parent or element
-        return scroll_anchor
+        with self.resolve_xpath(
+            xpath_anchor or self.last_hover_xpath
+        ) as element_resolved:
+            element = element_resolved.element
+            parent = self.driver.execute_script(JS_GET_SCROLLABLE_PARENT, element)
+            scroll_anchor = parent or element
+            return scroll_anchor
 
     def get_scroll_container_size(self, scroll_anchor: WebElement):
         container = self.driver.execute_script(JS_GET_SCROLLABLE_PARENT, scroll_anchor)
@@ -421,39 +444,43 @@ def scroll(
             self.scroll_page(direction)
 
     def click(self, xpath: str):
-        element = self.resolve_xpath(xpath)
-        self.last_hover_xpath = xpath
-        try:
-            element.click()
-        except ElementClickInterceptedException:
+        with self.resolve_xpath(xpath) as element_resolved:
+            element = element_resolved.element
+            self.last_hover_xpath = xpath
             try:
-                # Move to the element and click at its position
-                ActionChains(self.driver).move_to_element(element).click().perform()
-            except WebDriverException as click_error:
+                element.click()
+            except ElementClickInterceptedException:
+                try:
+                    # Move to the element and click at its position
+                    ActionChains(self.driver).move_to_element(element).click().perform()
+                except WebDriverException as click_error:
+                    raise Exception(
+                        f"Failed to click at element coordinates of {xpath} : {str(click_error)}"
+                    )
+            except Exception as e:
+                import traceback
+
+                traceback.print_exc()
                 raise Exception(
-                    f"Failed to click at element coordinates of {xpath} : {str(click_error)}"
+                    f"An unexpected error occurred when trying to click on {xpath}: {str(e)}"
                 )
-        except Exception as e:
-            raise Exception(
-                f"An unexpected error occurred when trying to click on {xpath}: {str(e)}"
-            )
-        self.driver.switch_to.default_content()
 
     def set_value(self, xpath: str, value: str, enter: bool = False):
-        try:
-            elem = self.resolve_xpath(xpath)
-            self.last_hover_xpath = xpath
-            if elem.tag_name == "select":
-                # use the dropdown_select to set the value of a select
-                return self.dropdown_select(xpath, value)
-            if elem.tag_name == "input" and elem.get_attribute("type") == "file":
-                # set the value of a file input
-                return self.upload_file(xpath, value)
-
-            elem.clear()
-        except:
-            # might not be a clearable element, but global click + send keys can still success
-            pass
+        with self.resolve_xpath(xpath) as element_resolved:
+            elem = element_resolved.element
+            try:
+                self.last_hover_xpath = xpath
+                if elem.tag_name == "select":
+                    # use the dropdown_select to set the value of a select
+                    return self.dropdown_select(xpath, value)
+                if elem.tag_name == "input" and elem.get_attribute("type") == "file":
+                    # set the value of a file input
+                    return self.upload_file(xpath, value)
+
+                elem.clear()
+            except:
+                # might not be a clearable element, but global click + send keys can still success
+                pass
 
         self.click(xpath)
 
@@ -468,29 +495,29 @@ def set_value(self, xpath: str, value: str, enter: bool = False):
         )
         if enter:
             ActionChains(self.driver).send_keys(Keys.ENTER).perform()
-        self.driver.switch_to.default_content()
 
     def dropdown_select(self, xpath: str, value: str):
-        element = self.resolve_xpath(xpath)
-        self.last_hover_xpath = xpath
+        with self.resolve_xpath(xpath) as element_resolved:
+            element = element_resolved.element
+            self.last_hover_xpath = xpath
 
-        if element.tag_name != "select":
-            print(
-                f"Cannot use dropdown_select on {element.tag_name}, falling back to simple click on {xpath}"
-            )
-            return self.click(xpath)
+            if element.tag_name != "select":
+                print(
+                    f"Cannot use dropdown_select on {element.tag_name}, falling back to simple click on {xpath}"
+                )
+                return self.click(xpath)
 
-        select = Select(element)
-        try:
-            select.select_by_value(value)
-        except NoSuchElementException:
-            select.select_by_visible_text(value)
-        self.driver.switch_to.default_content()
+            select = Select(element)
+            try:
+                select.select_by_value(value)
+            except NoSuchElementException:
+                select.select_by_visible_text(value)
 
     def upload_file(self, xpath: str, file_path: str):
-        element = self.resolve_xpath(xpath)
-        self.last_hover_xpath = xpath
-        element.send_keys(file_path)
+        with self.resolve_xpath(xpath) as element_resolved:
+            element = element_resolved.element
+            self.last_hover_xpath = xpath
+            element.send_keys(file_path)
 
     def perform_wait(self, duration: float):
         import time
@@ -608,10 +635,14 @@ def exec_script_for_nodes(self, nodes: List["SeleniumNode"], script: str):
             )
 
         if len(special_nodes) > 0:
-            self.driver.execute_script(
-                script,
-                [n.element for n in special_nodes if n.element],
-            )
+            # iframe and shadow DOM must use the resolve_xpath method
+            for n in special_nodes:
+                if n.element:
+                    self.driver.execute_script(
+                        script,
+                        [n.element],
+                    )
+                    self.switch_default_frame()
 
     def remove_nodes_highlight(self, xpaths: List[str]):
         self.exec_script_for_nodes(
@@ -636,7 +667,8 @@ def get_possible_interactions(
         self, in_viewport=True, foreground_only=True
     ) -> PossibleInteractionsByXpath:
         exe: Dict[str, List[str]] = self.driver.execute_script(
-            JS_GET_INTERACTIVES_IN_VIEWPORT if in_viewport else JS_GET_INTERACTIVES,
+            JS_GET_INTERACTIVES,
+            in_viewport,
             foreground_only,
         )
         res = dict()
@@ -656,7 +688,7 @@ def element(self):
         if hasattr(self, "_element"):
             return self._element
         try:
-            self._element = self._driver.resolve_xpath(self.xpath)
+            self._element = self._driver.resolve_xpath(self.xpath).element
         except StaleElementReferenceException:
             self._element = None
         return self._element

lavague-tests/lavague/tests/cli.py

@@ -62,7 +62,7 @@ def cli(
 
 
 def _load_sites(directory, site):
-    sites_to_test: List[Path] = []
+    sites_to_test: List[TestConfig] = []
     try:
         for item in os.listdir(directory):
             if (len(site) == 0 or item in site) and os.path.isfile(