From 24317c15fbe1ea640ff70a938468fbc85223dccd Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:50:04 -0500 Subject: [PATCH 1/4] Update CDP Mode --- examples/cdp_mode/ReadMe.md | 2 + seleniumbase/core/browser_launcher.py | 21 +++++- seleniumbase/core/sb_cdp.py | 100 ++++++++++++++------------ seleniumbase/fixtures/base_case.py | 19 ++--- 4 files changed, 88 insertions(+), 54 deletions(-) diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 97b7edecd08..31d4bdea399 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -470,6 +470,8 @@ sb.cdp.is_exact_text_visible(text, selector="body") sb.cdp.wait_for_text(text, selector="body", timeout=None) sb.cdp.wait_for_text_not_visible(text, selector="body", timeout=None) sb.cdp.wait_for_element_visible(selector, timeout=None) +sb.cdp.wait_for_element_not_visible(selector, timeout=None) +sb.cdp.wait_for_element_absent(selector, timeout=None) sb.cdp.assert_element(selector, timeout=None) sb.cdp.assert_element_visible(selector, timeout=None) sb.cdp.assert_element_present(selector, timeout=None) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 4fac3296817..79fb720f71f 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -723,6 +723,8 @@ def uc_open_with_cdp_mode(driver, url=None): cdp.wait_for_text = CDPM.wait_for_text cdp.wait_for_text_not_visible = CDPM.wait_for_text_not_visible cdp.wait_for_element_visible = CDPM.wait_for_element_visible + cdp.wait_for_element_not_visible = CDPM.wait_for_element_not_visible + cdp.wait_for_element_absent = CDPM.wait_for_element_absent cdp.assert_element = CDPM.assert_element cdp.assert_element_visible = CDPM.assert_element_visible cdp.assert_element_present = CDPM.assert_element_present @@ -1628,9 +1630,19 @@ def _uc_gui_handle_captcha_(driver, frame="iframe", ctype=None): ): driver.uc_open_with_disconnect(driver.current_url, 3.8) with suppress(Exception): + if "--debug" in sys.argv: + if sb_config._saved_cf_tab_count == 1: + print(' pyautogui.press("\\t")') + else: + print( + ' pyautogui.press("\\t") * %s' + % sb_config._saved_cf_tab_count + ) for i in range(sb_config._saved_cf_tab_count): pyautogui.press("\t") time.sleep(0.027) + if "--debug" in sys.argv: + print(' pyautogui.press(" ")') pyautogui.press(" ") else: driver.disconnect() @@ -2310,7 +2322,14 @@ def _set_chrome_options( and not enable_3d_apis ): chrome_options.add_argument("--disable-gpu") - if not IS_LINUX and is_using_uc(undetectable, browser_name): + if ( + (not IS_LINUX and is_using_uc(undetectable, browser_name)) + or ( + IS_MAC + and binary_location + and "chrome-headless-shell" in binary_location + ) + ): chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--disable-application-cache") if IS_LINUX: diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index 2b0003df107..d8388022174 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -497,7 +497,7 @@ def __press_keys(self, element, text): element.send_keys("\r\n") time.sleep(0.044) self.__slow_mode_pause_if_set() - return self.loop.run_until_complete(self.page.wait()) + return self.loop.run_until_complete(self.page.sleep(0.025)) def __query_selector(self, element, selector): selector = self.__convert_to_css_if_xpath(selector) @@ -864,7 +864,7 @@ def send_keys(self, selector, text, timeout=None): text = text[:-1] + "\r\n" element.send_keys(text) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def press_keys(self, selector, text, timeout=None): """Similar to send_keys(), but presses keys at human speed.""" @@ -884,7 +884,7 @@ def press_keys(self, selector, text, timeout=None): element.send_keys("\r\n") time.sleep(0.044) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def type(self, selector, text, timeout=None): """Similar to send_keys(), but clears the text field first.""" @@ -899,7 +899,7 @@ def type(self, selector, text, timeout=None): text = text[:-1] + "\r\n" element.send_keys(text) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def set_value(self, selector, text, timeout=None): """Similar to send_keys(), but clears the text field first.""" @@ -937,7 +937,7 @@ def set_value(self, selector, text, timeout=None): self.__add_light_pause() self.send_keys(selector, "\n") self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def evaluate(self, expression): """Run a JavaScript expression and return the result.""" @@ -1377,7 +1377,7 @@ def gui_press_key(self, key): pyautogui.press(key) time.sleep(0.044) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def gui_press_keys(self, keys): self.__install_pyautogui_if_missing() @@ -1392,7 +1392,7 @@ def gui_press_keys(self, keys): pyautogui.press(key) time.sleep(0.044) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def gui_write(self, text): self.__install_pyautogui_if_missing() @@ -1405,7 +1405,7 @@ def gui_write(self, text): self.__make_sure_pyautogui_lock_is_writable() pyautogui.write(text) self.__slow_mode_pause_if_set() - self.loop.run_until_complete(self.page.wait()) + self.loop.run_until_complete(self.page.sleep(0.025)) def __gui_click_x_y(self, x, y, timeframe=0.25, uc_lock=False): self.__install_pyautogui_if_missing() @@ -1820,37 +1820,8 @@ def wait_for_element_visible(self, selector, timeout=None): time.sleep(0.1) raise Exception("Element {%s} was not visible!" % selector) - def assert_element(self, selector, timeout=None): - """Same as assert_element_visible()""" - self.assert_element_visible(selector, timeout=timeout) - return True - - def assert_element_visible(self, selector, timeout=None): - """Same as assert_element()""" - if not timeout: - timeout = settings.SMALL_TIMEOUT - try: - self.select(selector, timeout=timeout) - except Exception: - raise Exception("Element {%s} was not found!" % selector) - for i in range(30): - if self.is_element_visible(selector): - return True - time.sleep(0.1) - raise Exception("Element {%s} was not visible!" % selector) - - def assert_element_present(self, selector, timeout=None): - """Assert element is present in the DOM. (Visibility NOT required)""" - if not timeout: - timeout = settings.SMALL_TIMEOUT - try: - self.select(selector, timeout=timeout) - except Exception: - raise Exception("Element {%s} was not found!" % selector) - return True - - def assert_element_absent(self, selector, timeout=None): - """Assert element is not present in the DOM.""" + def wait_for_element_not_visible(self, selector, timeout=None): + """Wait for element to not be visible on page. (May still be in DOM)""" if not timeout: timeout = settings.SMALL_TIMEOUT start_ms = time.time() * 1000.0 @@ -1858,6 +1829,8 @@ def assert_element_absent(self, selector, timeout=None): for i in range(int(timeout * 10)): if not self.is_element_present(selector): return True + elif not self.is_element_visible(selector): + return True now_ms = time.time() * 1000.0 if now_ms >= stop_ms: break @@ -1866,12 +1839,12 @@ def assert_element_absent(self, selector, timeout=None): if timeout == 1: plural = "" raise Exception( - "Element {%s} was still present after %s second%s!" + "Element {%s} was still visible after %s second%s!" % (selector, timeout, plural) ) - def assert_element_not_visible(self, selector, timeout=None): - """Assert element is not visible on page. (May still be in DOM)""" + def wait_for_element_absent(self, selector, timeout=None): + """Wait for element to not be present in the DOM.""" if not timeout: timeout = settings.SMALL_TIMEOUT start_ms = time.time() * 1000.0 @@ -1879,8 +1852,6 @@ def assert_element_not_visible(self, selector, timeout=None): for i in range(int(timeout * 10)): if not self.is_element_present(selector): return True - elif not self.is_element_visible(selector): - return True now_ms = time.time() * 1000.0 if now_ms >= stop_ms: break @@ -1889,10 +1860,49 @@ def assert_element_not_visible(self, selector, timeout=None): if timeout == 1: plural = "" raise Exception( - "Element {%s} was still visible after %s second%s!" + "Element {%s} was still present after %s second%s!" % (selector, timeout, plural) ) + def assert_element(self, selector, timeout=None): + """Same as assert_element_visible()""" + self.assert_element_visible(selector, timeout=timeout) + return True + + def assert_element_visible(self, selector, timeout=None): + """Same as assert_element()""" + if not timeout: + timeout = settings.SMALL_TIMEOUT + try: + self.select(selector, timeout=timeout) + except Exception: + raise Exception("Element {%s} was not found!" % selector) + for i in range(30): + if self.is_element_visible(selector): + return True + time.sleep(0.1) + raise Exception("Element {%s} was not visible!" % selector) + + def assert_element_present(self, selector, timeout=None): + """Assert element is present in the DOM. (Visibility NOT required)""" + if not timeout: + timeout = settings.SMALL_TIMEOUT + try: + self.select(selector, timeout=timeout) + except Exception: + raise Exception("Element {%s} was not found!" % selector) + return True + + def assert_element_absent(self, selector, timeout=None): + """Assert element is not present in the DOM.""" + self.wait_for_element_absent(selector, timeout=timeout) + return True + + def assert_element_not_visible(self, selector, timeout=None): + """Assert element is not visible on page. (May still be in DOM)""" + self.wait_for_element_not_visible(selector, timeout=timeout) + return True + def assert_element_attribute(self, selector, attribute, value=None): attributes = self.get_element_attributes(selector) if attribute not in attributes: diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 8c6c57d0ca6..5a1dd8c54c1 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -9124,7 +9124,7 @@ def wait_for_element_not_present( original_selector = selector selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_absent(selector) + self.cdp.wait_for_element_absent(selector, timeout=timeout) return True return page_actions.wait_for_element_absent( self.driver, @@ -9585,7 +9585,7 @@ def assert_element_present( self.assert_elements_present(selector, by=by, timeout=timeout) return True if self.__is_cdp_swap_needed(): - self.cdp.assert_element_present(selector) + self.cdp.assert_element_present(selector, timeout=timeout) return True if self.__is_shadow_selector(selector): self.__assert_shadow_element_present(selector) @@ -9662,7 +9662,7 @@ def assert_element(self, selector, by="css selector", timeout=None): if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) if self.__is_cdp_swap_needed(): - self.cdp.assert_element(selector) + self.cdp.assert_element(selector, timeout=timeout) return True if isinstance(selector, list): self.assert_elements(selector, by=by, timeout=timeout) @@ -9955,7 +9955,7 @@ def assert_text( messenger_post, selector, by ) elif self.__is_cdp_swap_needed(): - self.cdp.assert_text(text, selector) + self.cdp.assert_text(text, selector, timeout=timeout) return True elif not self.is_connected(): self.connect() @@ -10005,7 +10005,7 @@ def assert_exact_text( original_selector = selector selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): - self.cdp.assert_exact_text(text, selector) + self.cdp.assert_exact_text(text, selector, timeout=timeout) return True if self.__is_shadow_selector(selector): self.__assert_exact_shadow_text_visible(text, selector, timeout) @@ -10245,6 +10245,9 @@ def wait_for_element_absent( timeout = self.__get_new_timeout(timeout) original_selector = selector selector, by = self.__recalculate_selector(selector, by) + if self.__is_cdp_swap_needed(): + self.cdp.wait_for_element_absent(selector, timeout=timeout) + return True return page_actions.wait_for_element_absent( self.driver, selector, @@ -10267,7 +10270,7 @@ def assert_element_absent(self, selector, by="css selector", timeout=None): if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_absent(selector) + self.cdp.assert_element_absent(selector, timeout=timeout) return True self.wait_for_element_absent(selector, by=by, timeout=timeout) return True @@ -10288,7 +10291,7 @@ def wait_for_element_not_visible( original_selector = selector selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_not_visible(selector) + self.cdp.wait_for_element_not_visible(selector, timeout=timeout) return True return page_actions.wait_for_element_not_visible( self.driver, @@ -10310,7 +10313,7 @@ def assert_element_not_visible( if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) if self.__is_cdp_swap_needed(): - self.cdp.assert_element_not_visible(selector) + self.cdp.assert_element_not_visible(selector, timeout=timeout) return True self.wait_for_element_not_visible(selector, by=by, timeout=timeout) if self.recorder_mode and self.__current_url_is_recordable(): From 0949787a392daf033beb4ecb3ce2ef86ae82deb0 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:52:35 -0500 Subject: [PATCH 2/4] Update CDP Mode examples --- examples/cdp_mode/ReadMe.md | 4 ++-- examples/cdp_mode/raw_cdp_nike.py | 2 +- examples/cdp_mode/raw_footlocker.py | 4 ++-- examples/cdp_mode/raw_nike.py | 4 ++-- examples/cdp_mode/raw_res_nike.py | 4 ++-- examples/cdp_mode/raw_science.py | 15 +++++++++++++++ examples/presenter/uc_presentation_4.py | 4 ++-- 7 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 examples/cdp_mode/raw_science.py diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 31d4bdea399..2186fa903e4 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -328,11 +328,11 @@ with SB(uc=True, test=True, ad_block=True) as sb: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: +with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_cdp_nike.py b/examples/cdp_mode/raw_cdp_nike.py index 48d8356337e..88c72539601 100644 --- a/examples/cdp_mode/raw_cdp_nike.py +++ b/examples/cdp_mode/raw_cdp_nike.py @@ -8,7 +8,7 @@ page = loop.run_until_complete(driver.get(url)) sb = sb_cdp.CDPMethods(loop, page, driver) -search = "Nike Fly Shoes" +search = "Road Racing Shoes" sb.click('div[data-testid="user-tools-container"]') sb.sleep(1) sb.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_footlocker.py b/examples/cdp_mode/raw_footlocker.py index 717a1e483d7..311d91485ad 100644 --- a/examples/cdp_mode/raw_footlocker.py +++ b/examples/cdp_mode/raw_footlocker.py @@ -5,9 +5,9 @@ sb.activate_cdp_mode(url) sb.sleep(2.5) sb.cdp.click_if_visible('button[id*="Agree"]') - sb.sleep(2.5) + sb.sleep(1.5) sb.cdp.mouse_click('input[aria-label="Search"]') - sb.sleep(2.5) + sb.sleep(1.5) search = "Nike Shoes" sb.cdp.press_keys('input[aria-label="Search"]', search) sb.sleep(2.5) diff --git a/examples/cdp_mode/raw_nike.py b/examples/cdp_mode/raw_nike.py index 3a811b0dd22..009f723dc49 100644 --- a/examples/cdp_mode/raw_nike.py +++ b/examples/cdp_mode/raw_nike.py @@ -1,10 +1,10 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: +with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_res_nike.py b/examples/cdp_mode/raw_res_nike.py index 416a001a8d0..459b143e77e 100644 --- a/examples/cdp_mode/raw_res_nike.py +++ b/examples/cdp_mode/raw_res_nike.py @@ -25,13 +25,13 @@ async def receive_handler(event: mycdp.network.ResponseReceived): print(event.response) -with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: +with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.cdp.add_handler(mycdp.network.RequestWillBeSent, send_handler) sb.cdp.add_handler(mycdp.network.ResponseReceived, receive_handler) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) diff --git a/examples/cdp_mode/raw_science.py b/examples/cdp_mode/raw_science.py new file mode 100644 index 00000000000..792dca7c278 --- /dev/null +++ b/examples/cdp_mode/raw_science.py @@ -0,0 +1,15 @@ +from seleniumbase import SB + +with SB(uc=True, incognito=True, test=True) as sb: + url = "https://earth.esa.int/eogateway/search" + sb.activate_cdp_mode(url) + sb.sleep(1) + sb.cdp.click_if_visible('button:contains("Accept cookies")') + for i in range(20): + sb.cdp.scroll_to_bottom() + sb.cdp.click_if_visible('button:contains("READ MORE")') + sb.sleep(1) + elements = sb.cdp.find_elements("h4 a span") + for element in elements: + print(element.text) + print("*** Total entries: %s" % len(elements)) diff --git a/examples/presenter/uc_presentation_4.py b/examples/presenter/uc_presentation_4.py index dbba53dad8b..42ff12b38e0 100644 --- a/examples/presenter/uc_presentation_4.py +++ b/examples/presenter/uc_presentation_4.py @@ -838,11 +838,11 @@ def test_presentation_4(self): ) self.begin_presentation(filename="uc_presentation.html") - with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: + with SB(uc=True, test=True, locale_code="en", pls="none") as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.gui_click_element('div[data-testid="user-tools-container"]') + sb.cdp.mouse_click('div[data-testid="user-tools-container"]') sb.sleep(1.5) search = "Nike Air Force 1" sb.cdp.press_keys('input[type="search"]', search) From 9f47f5ff425ab3e034d47daf84b6fd7308ea2aa5 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:53:08 -0500 Subject: [PATCH 3/4] Upgrade Selenium --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8763830f299..1a0f763cf1c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ trio-websocket==0.12.1 wsproto==1.2.0 websocket-client==1.8.0 selenium==4.27.1;python_version<"3.9" -selenium==4.28.1;python_version>="3.9" +selenium==4.29.0;python_version>="3.9" cssselect==1.2.0 sortedcontainers==2.4.0 execnet==2.1.1 diff --git a/setup.py b/setup.py index e4890264478..dbcf8eee29e 100755 --- a/setup.py +++ b/setup.py @@ -192,7 +192,7 @@ 'wsproto==1.2.0', 'websocket-client==1.8.0', 'selenium==4.27.1;python_version<"3.9"', - 'selenium==4.28.1;python_version>="3.9"', + 'selenium==4.29.0;python_version>="3.9"', 'cssselect==1.2.0', "sortedcontainers==2.4.0", 'execnet==2.1.1', From 4974aeff8a5380c317325e001add4babd51e0d3d Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 20 Feb 2025 14:53:49 -0500 Subject: [PATCH 4/4] Version 4.35.0 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index ae694d9e912..ff2fa154b90 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.34.17" +__version__ = "4.35.0"