diff --git a/.gitignore b/.gitignore index df944cb5c73..7aec1e4d7ce 100644 --- a/.gitignore +++ b/.gitignore @@ -4,18 +4,18 @@ # Packages *.egg *.egg-info -dist -build .eggs eggs -parts -bin -var -sdist develop-eggs -.installed.cfg +bin +build +dist lib lib64 +parts +sdist +var +.installed.cfg __pycache__ # Python3 pyvenv diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ef4d2f5fd5..0833240789b 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@

CHANGELOG

-### See: [SeleniumBase/releases](https://github.com/seleniumbase/SeleniumBase/releases) +## See: [SeleniumBase/releases](https://github.com/seleniumbase/SeleniumBase/releases) 🗂️ 📋 + +### (For CDP updates, see the [CDP Mode docs](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md)) diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index 4dc76a6020a..5480986d4e7 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -71,21 +71,21 @@ To find out if WebDriver is connected or disconnected, call: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.pokemon.com/us" sb.activate_cdp_mode(url) - sb.sleep(2.5) - sb.cdp.click_if_visible("button#onetrust-reject-all-handler") - sb.sleep(1.2) - sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]') + sb.sleep(3.2) + sb.cdp.click("button#onetrust-accept-btn-handler") sb.sleep(1.2) + sb.cdp.click("a span.icon_pokeball") + sb.sleep(2.5) sb.cdp.click('b:contains("Show Advanced Search")') - sb.sleep(1.2) + sb.sleep(2.5) sb.cdp.click('span[data-type="type"][data-value="electric"]') sb.sleep(0.5) sb.scroll_into_view("a#advSearch") sb.sleep(0.5) - sb.cdp.click("a#advSearch") + sb.cdp.mouse_click("a#advSearch") sb.sleep(1.2) sb.cdp.click('img[src*="img/pokedex/detail/025.png"]') sb.cdp.assert_text("Pikachu", 'div[class*="title"]') @@ -102,11 +102,13 @@ with SB(uc=True, test=True, locale_code="en") as sb: sb.cdp.highlight_overlay("div.pokemon-ability-info") sb.sleep(2) sb.cdp.click('a[href="https://www.pokemon.com/us/play-pokemon/"]') + sb.sleep(0.6) sb.cdp.click('h3:contains("Find an Event")') location = "Concord, MA, USA" sb.cdp.type('input[data-testid="location-search"]', location) - sb.sleep(1) + sb.sleep(1.5) sb.cdp.click("div.autocomplete-dropdown-container div.suggestion-item") + sb.sleep(0.6) sb.cdp.click('img[alt="search-icon"]') sb.sleep(2) events = sb.cdp.select_all('div[data-testid="event-name"]') @@ -129,10 +131,10 @@ with SB(uc=True, test=True, locale_code="en") as sb: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.hyatt.com/" sb.activate_cdp_mode(url) - sb.sleep(2) + sb.sleep(2.5) sb.cdp.click_if_visible('button[aria-label="Close"]') sb.sleep(1) sb.cdp.click('span:contains("Explore")') @@ -176,7 +178,7 @@ with SB(uc=True, test=True, locale_code="en") as sb: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.bestwestern.com/en_US.html" sb.activate_cdp_mode(url) sb.sleep(2.5) @@ -220,7 +222,7 @@ with SB(uc=True, test=True, locale_code="en") as sb: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.walmart.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) @@ -266,7 +268,7 @@ with SB(uc=True, test=True, locale_code="en") as sb: ```python from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) @@ -395,12 +397,17 @@ sb.cdp.uncheck_if_checked(selector) sb.cdp.unselect_if_selected(selector) sb.cdp.is_element_present(selector) sb.cdp.is_element_visible(selector) -sb.cdp.assert_element_present(selector) -sb.cdp.assert_element_absent(selector) +sb.cdp.wait_for_element_visible(selector) sb.cdp.assert_element(selector) sb.cdp.assert_element_visible(selector) +sb.cdp.assert_element_present(selector) +sb.cdp.assert_element_absent(selector) sb.cdp.assert_element_not_visible(selector) +sb.cdp.assert_element_attribute(selector, attribute, value=None) sb.cdp.assert_title(title) +sb.cdp.assert_title_contains(substring) +sb.cdp.assert_url(url) +sb.cdp.assert_url_contains(substring) sb.cdp.assert_text(text, selector="html") sb.cdp.assert_exact_text(text, selector="html") sb.cdp.scroll_into_view(selector) diff --git a/examples/cdp_mode/raw_async.py b/examples/cdp_mode/raw_async.py index 74b85a078ab..778740685d6 100644 --- a/examples/cdp_mode/raw_async.py +++ b/examples/cdp_mode/raw_async.py @@ -37,7 +37,7 @@ async def main(): time.sleep(1) element = loop.run_until_complete(page.select("span.icon_pokeball")) loop.run_until_complete(element.click_async()) - time.sleep(1.5) + time.sleep(2) print(loop.run_until_complete(page.evaluate("document.title"))) time.sleep(1) @@ -46,17 +46,22 @@ async def main(): sb = sb_cdp.CDPMethods(loop, page, driver) sb.set_locale("en") sb.open("https://www.priceline.com/") - sb.sleep(3) + sb.sleep(2.5) sb.internalize_links() # Don't open links in a new tab sb.click("#link_header_nav_experiences") - sb.sleep(2.5) + sb.sleep(3.5) sb.remove_elements("msm-cookie-banner") sb.sleep(1.5) location = "Amsterdam" - sb.press_keys('input[data-test-id*="search"]', location) + where_to = 'div[data-automation*="experiences"] input' + button = 'button[data-automation*="experiences-search"]' + sb.gui_click_element(where_to) + sb.press_keys(where_to, location) sb.sleep(1) - sb.click('input[data-test-id*="search"]') - sb.sleep(2) - sb.click('span[data-test-id*="autocomplete"]') - sb.sleep(5) + sb.gui_click_element(button) + sb.sleep(3) print(sb.get_title()) + print("************") + cards = sb.select_all('h2[data-automation*="product-list-card"]') + for card in cards: + print("* %s" % card.text) diff --git a/examples/cdp_mode/raw_bestwestern.py b/examples/cdp_mode/raw_bestwestern.py index da2dfddb51a..fe7ee033e4d 100644 --- a/examples/cdp_mode/raw_bestwestern.py +++ b/examples/cdp_mode/raw_bestwestern.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.bestwestern.com/en_US.html" sb.activate_cdp_mode(url) sb.sleep(2.5) diff --git a/examples/cdp_mode/raw_cdp.py b/examples/cdp_mode/raw_cdp.py index 990f10fa43e..c733859c934 100644 --- a/examples/cdp_mode/raw_cdp.py +++ b/examples/cdp_mode/raw_cdp.py @@ -1,6 +1,5 @@ """Example of using CDP Mode without WebDriver""" import asyncio -from contextlib import suppress from seleniumbase import decorators from seleniumbase.core import sb_cdp from seleniumbase.undetected import cdp_driver @@ -16,29 +15,28 @@ def main(): sb = sb_cdp.CDPMethods(loop, page, driver) sb.set_locale("en") # This test expects English locale sb.open(url1) - sb.sleep(3) + sb.sleep(2.5) sb.internalize_links() # Don't open links in a new tab sb.click("#link_header_nav_experiences") - sb.sleep(2.5) + sb.sleep(3.5) sb.remove_elements("msm-cookie-banner") sb.sleep(1.5) location = "Amsterdam" - sb.press_keys('input[data-test-id*="search"]', location) + where_to = 'div[data-automation*="experiences"] input' + button = 'button[data-automation*="experiences-search"]' + sb.gui_click_element(where_to) + sb.press_keys(where_to, location) sb.sleep(1) - sb.click('input[data-test-id*="search"]') - sb.sleep(2) - sb.click('span[data-test-id*="autocomplete"]') - sb.sleep(5) + sb.gui_click_element(button) + sb.sleep(3) print(sb.get_title()) - header = sb.get_text('h2[data-testid*="RelatedVenues"]') - print("*** %s: ***" % header) - cards = sb.select_all("div.venue-card__body") + print("************") + for i in range(8): + sb.scroll_down(50) + sb.sleep(0.2) + cards = sb.select_all('h2[data-automation*="product-list-card"]') for card in cards: - with suppress(Exception): - venue = card.text.split("\n")[0].strip() - rating = card.text.split("\n")[1].strip() - reviews = card.text.split("\n")[2].strip()[1:-1] - print("* %s: %s from %s reviews." % (venue, rating, reviews)) + print("* %s" % card.text) if __name__ == "__main__": diff --git a/examples/cdp_mode/raw_cdp_with_sb.py b/examples/cdp_mode/raw_cdp_with_sb.py index 0a3ee41e9b8..14ce948d04f 100644 --- a/examples/cdp_mode/raw_cdp_with_sb.py +++ b/examples/cdp_mode/raw_cdp_with_sb.py @@ -1,5 +1,4 @@ """Example of using CDP Mode with WebDriver""" -from contextlib import suppress from seleniumbase import SB @@ -9,23 +8,22 @@ sb.sleep(2.5) sb.internalize_links() # Don't open links in a new tab sb.click("#link_header_nav_experiences") - sb.sleep(2.5) + sb.sleep(3.5) sb.remove_elements("msm-cookie-banner") sb.sleep(1.5) location = "Amsterdam" - sb.press_keys('input[data-test-id*="search"]', location) + where_to = 'div[data-automation*="experiences"] input' + button = 'button[data-automation*="experiences-search"]' + sb.cdp.gui_click_element(where_to) + sb.press_keys(where_to, location) sb.sleep(1) - sb.click('input[data-test-id*="search"]') - sb.sleep(2) - sb.click('span[data-test-id*="autocomplete"]') - sb.sleep(5) + sb.cdp.gui_click_element(button) + sb.sleep(3) print(sb.get_title()) - header = sb.get_text('h2[data-testid*="RelatedVenues"]') - print("*** %s: ***" % header) - cards = sb.select_all("div.venue-card__body") + print("************") + for i in range(8): + sb.cdp.scroll_down(50) + sb.sleep(0.2) + cards = sb.select_all('h2[data-automation*="product-list-card"]') for card in cards: - with suppress(Exception): - venue = card.text.split("\n")[0].strip() - rating = card.text.split("\n")[1].strip() - reviews = card.text.split("\n")[2].strip()[1:-1] - print("* %s: %s from %s reviews." % (venue, rating, reviews)) + print("* %s" % card.text) diff --git a/examples/cdp_mode/raw_easyjet.py b/examples/cdp_mode/raw_easyjet.py index d8128fd6402..518cfe8270d 100644 --- a/examples/cdp_mode/raw_easyjet.py +++ b/examples/cdp_mode/raw_easyjet.py @@ -1,15 +1,17 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.easyjet.com/en/" sb.activate_cdp_mode(url) sb.sleep(2.5) - sb.cdp.click_if_visible('button#ensRejectAll') + sb.cdp.click_if_visible("button#ensCloseBanner") sb.sleep(1.2) sb.cdp.click('input[name="from"]') sb.sleep(1.2) sb.cdp.type('input[name="from"]', "London") - sb.sleep(1.2) + sb.sleep(0.6) + sb.cdp.click_if_visible("button#ensCloseBanner") + sb.sleep(0.6) sb.cdp.click('span[data-testid="airport-name"]') sb.sleep(1.2) sb.cdp.type('input[name="to"]', "Venice") @@ -23,16 +25,20 @@ sb.cdp.click('[data-testid="month"]:last-of-type [aria-disabled="false"]') sb.sleep(1.2) sb.cdp.click('button[data-testid="submit"]') - sb.sleep(3.5) + sb.sleep(2.5) sb.connect() - sb.sleep(0.5) - if "/buy/flights" not in sb.get_current_url(): - sb.driver.close() - sb.switch_to_newest_window() - days = sb.find_elements("div.flight-grid-day") + sb.sleep(1.2) + for window in sb.driver.window_handles: + sb.switch_to_window(window) + if "/buy/flights" in sb.get_current_url(): + break + sb.click_if_visible("button#ensCloseBanner") + days = sb.find_elements('div[class*="FlightGridLayout_column"]') for day in days: + if not day.text.strip(): + continue print("**** " + " ".join(day.text.split("\n")[0:2]) + " ****") - fares = day.find_elements("css selector", "button.selectable") + fares = day.find_elements("css selector", 'button[class*="flightDet"]') if not fares: print("No flights today!") for fare in fares: diff --git a/examples/cdp_mode/raw_footlocker.py b/examples/cdp_mode/raw_footlocker.py index adb0613d656..717a1e483d7 100644 --- a/examples/cdp_mode/raw_footlocker.py +++ b/examples/cdp_mode/raw_footlocker.py @@ -1,18 +1,18 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.footlocker.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) sb.cdp.click_if_visible('button[id*="Agree"]') - sb.sleep(1.5) + sb.sleep(2.5) sb.cdp.mouse_click('input[aria-label="Search"]') - sb.sleep(1.5) + sb.sleep(2.5) search = "Nike Shoes" sb.cdp.press_keys('input[aria-label="Search"]', search) - sb.sleep(2) + sb.sleep(2.5) sb.cdp.mouse_click('ul[id*="typeahead"] li div') - sb.sleep(3) + sb.sleep(3.5) elements = sb.cdp.select_all("a.ProductCard-link") if elements: print('**** Found results for "%s": ****' % search) diff --git a/examples/cdp_mode/raw_hyatt.py b/examples/cdp_mode/raw_hyatt.py index ef17928e316..f6867823305 100644 --- a/examples/cdp_mode/raw_hyatt.py +++ b/examples/cdp_mode/raw_hyatt.py @@ -1,9 +1,9 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.hyatt.com/" sb.activate_cdp_mode(url) - sb.sleep(2) + sb.sleep(2.5) sb.cdp.click_if_visible('button[aria-label="Close"]') sb.sleep(1) sb.cdp.click('span:contains("Explore")') diff --git a/examples/cdp_mode/raw_nike.py b/examples/cdp_mode/raw_nike.py index 2c1ff671d07..3a811b0dd22 100644 --- a/examples/cdp_mode/raw_nike.py +++ b/examples/cdp_mode/raw_nike.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.nike.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) diff --git a/examples/cdp_mode/raw_pokemon.py b/examples/cdp_mode/raw_pokemon.py index f6d32b21415..38d29e85972 100644 --- a/examples/cdp_mode/raw_pokemon.py +++ b/examples/cdp_mode/raw_pokemon.py @@ -1,20 +1,20 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.pokemon.com/us" sb.activate_cdp_mode(url) - sb.sleep(2.5) - sb.cdp.click_if_visible("button#onetrust-reject-all-handler") - sb.sleep(1.2) - sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]') + sb.sleep(3.2) + sb.cdp.click("button#onetrust-accept-btn-handler") sb.sleep(1.2) + sb.cdp.click("a span.icon_pokeball") + sb.sleep(2.5) sb.cdp.click('b:contains("Show Advanced Search")') - sb.sleep(1.2) + sb.sleep(2.5) sb.cdp.click('span[data-type="type"][data-value="electric"]') sb.sleep(0.5) sb.scroll_into_view("a#advSearch") sb.sleep(0.5) - sb.cdp.click("a#advSearch") + sb.cdp.mouse_click("a#advSearch") sb.sleep(1.2) sb.cdp.click('img[src*="img/pokedex/detail/025.png"]') sb.cdp.assert_text("Pikachu", 'div[class*="title"]') @@ -31,11 +31,13 @@ sb.cdp.highlight_overlay("div.pokemon-ability-info") sb.sleep(2) sb.cdp.click('a[href="https://www.pokemon.com/us/play-pokemon/"]') + sb.sleep(0.6) sb.cdp.click('h3:contains("Find an Event")') location = "Concord, MA, USA" sb.cdp.type('input[data-testid="location-search"]', location) - sb.sleep(1) + sb.sleep(1.5) sb.cdp.click("div.autocomplete-dropdown-container div.suggestion-item") + sb.sleep(0.6) sb.cdp.click('img[alt="search-icon"]') sb.sleep(2) events = sb.cdp.select_all('div[data-testid="event-name"]') diff --git a/examples/cdp_mode/raw_priceline.py b/examples/cdp_mode/raw_priceline.py index 079bdc17eba..6b604cb45fd 100644 --- a/examples/cdp_mode/raw_priceline.py +++ b/examples/cdp_mode/raw_priceline.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: window_handle = sb.driver.current_window_handle url = "https://www.priceline.com" sb.activate_cdp_mode(url) @@ -24,6 +24,7 @@ sb.sleep(0.2) sb.switch_to_newest_window() sb.sleep(0.6) + sb.sleep(0.8) for y in range(1, 9): sb.scroll_to_y(y * 400) sb.sleep(1.25) diff --git a/examples/cdp_mode/raw_req_async.py b/examples/cdp_mode/raw_req_async.py index 56e331b7ef1..7101ce6c7a8 100644 --- a/examples/cdp_mode/raw_req_async.py +++ b/examples/cdp_mode/raw_req_async.py @@ -1,9 +1,17 @@ """Using CDP.fetch.RequestPaused to filter content in real-time.""" import asyncio +import colorama import mycdp +import sys from seleniumbase import decorators from seleniumbase.undetected import cdp_driver +c1 = colorama.Fore.RED + colorama.Back.LIGHTYELLOW_EX +c2 = colorama.Fore.BLUE + colorama.Back.LIGHTCYAN_EX +cr = colorama.Style.RESET_ALL +if "linux" in sys.platform: + c1 = c2 = cr = "" + class RequestPausedTest(): async def request_paused_handler(self, event, tab): @@ -15,7 +23,7 @@ async def request_paused_handler(self, event, tab): ) else: # Block the data (images) TIMED_OUT = mycdp.network.ErrorReason.TIMED_OUT - s = f"BLOCKING | {r.method} | {r.url}" + s = f"{c1}BLOCKING{cr} | {c2}{r.method}{cr} | {r.url}" print(f" >>> ------------\n{s}") tab.feed_cdp( mycdp.fetch.fail_request(event.request_id, TIMED_OUT) diff --git a/examples/cdp_mode/raw_req_sb.py b/examples/cdp_mode/raw_req_sb.py index 241b5668683..a547ab7a68d 100644 --- a/examples/cdp_mode/raw_req_sb.py +++ b/examples/cdp_mode/raw_req_sb.py @@ -1,7 +1,15 @@ """Using CDP.fetch.RequestPaused to filter content in real-time.""" +import colorama import mycdp +import sys from seleniumbase import SB +c1 = colorama.Fore.RED + colorama.Back.LIGHTYELLOW_EX +c2 = colorama.Fore.BLUE + colorama.Back.LIGHTCYAN_EX +cr = colorama.Style.RESET_ALL +if "linux" in sys.platform: + c1 = c2 = cr = "" + async def request_paused_handler(event, tab): r = event.request @@ -10,7 +18,7 @@ async def request_paused_handler(event, tab): tab.feed_cdp(mycdp.fetch.continue_request(request_id=event.request_id)) else: # Block the data (images) TIMED_OUT = mycdp.network.ErrorReason.TIMED_OUT - s = f"BLOCKING | {r.method} | {r.url}" + s = f"{c1}BLOCKING{cr} | {c2}{r.method}{cr} | {r.url}" print(f" >>> ------------\n{s}") tab.feed_cdp(mycdp.fetch.fail_request(event.request_id, TIMED_OUT)) diff --git a/examples/cdp_mode/raw_southwest.py b/examples/cdp_mode/raw_southwest.py index 0c3f7c9d57a..226b2870684 100644 --- a/examples/cdp_mode/raw_southwest.py +++ b/examples/cdp_mode/raw_southwest.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.southwest.com/air/booking/" sb.activate_cdp_mode(url) sb.sleep(2.8) diff --git a/examples/cdp_mode/raw_walmart.py b/examples/cdp_mode/raw_walmart.py index 83292ad1969..9a7a8c05989 100644 --- a/examples/cdp_mode/raw_walmart.py +++ b/examples/cdp_mode/raw_walmart.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb: url = "https://www.walmart.com/" sb.activate_cdp_mode(url) sb.sleep(2.5) diff --git a/examples/cdp_mode/raw_wsform.py b/examples/cdp_mode/raw_wsform.py index 8abed99808b..faee7c35f9d 100644 --- a/examples/cdp_mode/raw_wsform.py +++ b/examples/cdp_mode/raw_wsform.py @@ -1,6 +1,6 @@ from seleniumbase import SB -with SB(uc=True, test=True, locale_code="en") as sb: +with SB(uc=True, test=True, locale_code="en", incognito=True) as sb: url = "https://wsform.com/demo/" sb.activate_cdp_mode(url) sb.scroll_into_view("div.grid") diff --git a/examples/cdp_mode/raw_xhr_sb.py b/examples/cdp_mode/raw_xhr_sb.py index 71c0d6b2fc5..9c03f39495f 100644 --- a/examples/cdp_mode/raw_xhr_sb.py +++ b/examples/cdp_mode/raw_xhr_sb.py @@ -63,12 +63,11 @@ async def crawl(): listenXHR(tab) # Change url to something that makes ajax requests - tab = await driver.get("https://resttesttest.com/") - time.sleep(1) - # Click AJAX button on https://resttesttest.com/ - element = await tab.select("button#submitajax") - await element.click_async() + tab = await driver.get("https://learn.microsoft.com/en-us/") time.sleep(2) + for i in range(75): + await tab.scroll_down(3) + time.sleep(0.02) xhr_responses = await receiveXHR(tab, xhr_requests) for response in xhr_responses: @@ -87,6 +86,6 @@ async def crawl(): if __name__ == "__main__": - print("================= Starting =================") + print("================== Starting ==================") loop = asyncio.new_event_loop() loop.run_until_complete(crawl()) diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md index 091f2bf71ef..62c8a41f3b6 100644 --- a/help_docs/method_summary.md +++ b/help_docs/method_summary.md @@ -362,6 +362,8 @@ self.get_cookie(name) self.get_cookies() +self.get_cookie_string() + self.add_cookie(cookie_dict, expiry=False) self.add_cookies(cookies, expiry=False) diff --git a/help_docs/uc_mode.md b/help_docs/uc_mode.md index 9bd84fbc82d..60cad7c8d23 100644 --- a/help_docs/uc_mode.md +++ b/help_docs/uc_mode.md @@ -4,6 +4,8 @@ 👤 SeleniumBase UC Mode (Undetected-Chromedriver Mode) allows bots to appear human, which lets them evade detection from anti-bot services that try to block them or trigger CAPTCHAs on various websites. +> #### (For the successor to default UC Mode, see **[CDP Mode 🐙](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/ReadMe.md)**) + --- diff --git a/requirements.txt b/requirements.txt index b9fbdde70e0..cd629526eb7 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -pip>=24.2 +pip>=24.3.1 packaging>=24.2 setuptools~=70.2;python_version<"3.10" setuptools>=75.5.0;python_version>="3.10" @@ -64,7 +64,7 @@ rich==13.9.4 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.6.1;python_version<"3.9" -coverage>=7.6.5;python_version>="3.9" +coverage>=7.6.7;python_version>="3.9" pytest-cov>=5.0.0;python_version<"3.9" pytest-cov>=6.0.0;python_version>="3.9" flake8==5.0.4;python_version<"3.9" diff --git a/sbase/steps.py b/sbase/steps.py index 8e650493a69..871ce6aaca0 100644 --- a/sbase/steps.py +++ b/sbase/steps.py @@ -1189,3 +1189,10 @@ def set_attributes(context, selector, attribute, value): if attribute.endswith("'") or attribute.endswith('"'): attribute = attribute[:-1] sb.set_attributes(selector, attribute, value) + + +@step("Activate CDP Mode") +@step("User activates CDP Mode") +def activate_cdp_mode(context): + sb = context.sb + sb.activate_cdp_mode() diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index e28e25bbe63..e2bf5cb5ee1 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.32.11" +__version__ = "4.32.12" diff --git a/seleniumbase/common/decorators.py b/seleniumbase/common/decorators.py index 0a1850716cc..21803cd8254 100644 --- a/seleniumbase/common/decorators.py +++ b/seleniumbase/common/decorators.py @@ -1,11 +1,18 @@ +import colorama import logging import math +import sys import time import warnings from contextlib import contextmanager from functools import wraps from seleniumbase.common.exceptions import TimeoutException +c1 = colorama.Fore.BLUE + colorama.Back.LIGHTYELLOW_EX +cr = colorama.Style.RESET_ALL +if "linux" in sys.platform: + c1 = cr = "" + @contextmanager def print_runtime(description=None, limit=None): @@ -44,19 +51,20 @@ def my_method(): end_time = time.time() run_time = end_time - start_time name = description + info = c1 + "" + cr # Print times with a statistically significant number of decimal places if run_time < 0.0001: - print(" - {%s} ran for %.7f seconds." % (name, run_time)) + print("%s - {%s} ran for %.7f seconds." % (info, name, run_time)) elif run_time < 0.001: - print(" - {%s} ran for %.6f seconds." % (name, run_time)) + print("%s - {%s} ran for %.6f seconds." % (info, name, run_time)) elif run_time < 0.01: - print(" - {%s} ran for %.5f seconds." % (name, run_time)) + print("%s - {%s} ran for %.5f seconds." % (info, name, run_time)) elif run_time < 0.1: - print(" - {%s} ran for %.4f seconds." % (name, run_time)) + print("%s - {%s} ran for %.4f seconds." % (info, name, run_time)) elif run_time < 1: - print(" - {%s} ran for %.3f seconds." % (name, run_time)) + print("%s - {%s} ran for %.3f seconds." % (info, name, run_time)) else: - print(" - {%s} ran for %.2f seconds." % (name, run_time)) + print("%s - {%s} ran for %.2f seconds." % (info, name, run_time)) if limit and limit > 0 and run_time > limit: message = ( "\n {%s} duration of %.2fs exceeded the time limit of %.2fs!" diff --git a/seleniumbase/console_scripts/sb_mkrec.py b/seleniumbase/console_scripts/sb_mkrec.py index 99a448cad13..01ca9cfe431 100644 --- a/seleniumbase/console_scripts/sb_mkrec.py +++ b/seleniumbase/console_scripts/sb_mkrec.py @@ -144,7 +144,9 @@ def main(): elif option.lower() in ("--gui", "--headed"): if "linux" in sys.platform: force_gui = True - elif option.lower() in ("--uc", "--undetected", "--undetectable"): + elif option.lower() in ( + "--uc", "--cdp", "--undetected", "--undetectable" + ): use_uc = True elif option.lower() in ("--rec-behave", "--behave", "--gherkin"): rec_behave = True diff --git a/seleniumbase/console_scripts/sb_recorder.py b/seleniumbase/console_scripts/sb_recorder.py index 132f07be9da..a4507daaf98 100644 --- a/seleniumbase/console_scripts/sb_recorder.py +++ b/seleniumbase/console_scripts/sb_recorder.py @@ -9,6 +9,7 @@ Options: --uc / --undetected (Use undetectable mode.) + --cdp (Same as "--uc" and "--undetectable".) --behave (Also output Behave/Gherkin files.) Output: @@ -151,6 +152,7 @@ def do_recording(file_name, url, overwrite_enabled, use_chrome, window): command += " --edge" if ( "--uc" in command_args + or "--cdp" in command_args or "--undetected" in command_args or "--undetectable" in command_args ): @@ -193,6 +195,7 @@ def do_playback(file_name, use_chrome, window, demo_mode=False): command_args = sys.argv[2:] if ( "--uc" in command_args + or "--cdp" in command_args or "--undetected" in command_args or "--undetectable" in command_args ): diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 3c21ecf6871..bf962e98576 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -683,12 +683,17 @@ def uc_open_with_cdp_mode(driver, url=None): cdp.is_checked = CDPM.is_checked cdp.is_element_present = CDPM.is_element_present cdp.is_element_visible = CDPM.is_element_visible + cdp.wait_for_element_visible = CDPM.wait_for_element_visible + cdp.assert_element = CDPM.assert_element + cdp.assert_element_visible = CDPM.assert_element_visible cdp.assert_element_present = CDPM.assert_element_present cdp.assert_element_absent = CDPM.assert_element_absent - cdp.assert_element = CDPM.assert_element - cdp.assert_element_visible = CDPM.assert_element cdp.assert_element_not_visible = CDPM.assert_element_not_visible + cdp.assert_element_attribute = CDPM.assert_element_attribute cdp.assert_title = CDPM.assert_title + cdp.assert_title_contains = CDPM.assert_title_contains + cdp.assert_url = CDPM.assert_url + cdp.assert_url_contains = CDPM.assert_url_contains cdp.assert_text = CDPM.assert_text cdp.assert_exact_text = CDPM.assert_exact_text cdp.scroll_into_view = CDPM.scroll_into_view diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index e2ef183311c..da22407a648 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -20,8 +20,16 @@ def __init__(self, loop, page, driver): self.driver = driver def __slow_mode_pause_if_set(self): - if hasattr(sb_config, "slow_mode") and sb_config.slow_mode: - time.sleep(0.16) + if ( + (hasattr(sb_config, "demo_mode") and sb_config.demo_mode) + or "--demo" in sys.argv + ): + time.sleep(0.48) + elif ( + (hasattr(sb_config, "slow_mode") and sb_config.slow_mode) + or "--slow" in sys.argv + ): + time.sleep(0.24) def __add_light_pause(self): time.sleep(0.007) @@ -543,7 +551,7 @@ def click_visible_elements(self, selector, limit=0): if (width != 0 or height != 0): element.click() click_count += 1 - time.sleep(0.0375) + time.sleep(0.042) self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.wait()) except Exception: @@ -660,10 +668,10 @@ def press_keys(self, selector, text, timeout=settings.SMALL_TIMEOUT): text = text[:-1] for key in text: element.send_keys(key) - time.sleep(0.0375) + time.sleep(0.042) if submit: element.send_keys("\r\n") - time.sleep(0.0375) + time.sleep(0.042) self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.wait()) @@ -733,7 +741,7 @@ def maximize(self): return elif self.get_window()[1].window_state.value == "minimized": self.loop.run_until_complete(self.page.maximize()) - time.sleep(0.0375) + time.sleep(0.042) return self.loop.run_until_complete(self.page.maximize()) def minimize(self): @@ -743,7 +751,7 @@ def minimize(self): def medimize(self): if self.get_window()[1].window_state.value == "minimized": self.loop.run_until_complete(self.page.medimize()) - time.sleep(0.0375) + time.sleep(0.042) return self.loop.run_until_complete(self.page.medimize()) def set_window_rect(self, x, y, width, height): @@ -752,7 +760,7 @@ def set_window_rect(self, x, y, width, height): self.page.set_window_size( left=x, top=y, width=width, height=height) ) - time.sleep(0.0375) + time.sleep(0.042) return self.loop.run_until_complete( self.page.set_window_size( left=x, top=y, width=width, height=height) @@ -1117,7 +1125,7 @@ def gui_press_key(self, key): ) with gui_lock: pyautogui.press(key) - time.sleep(0.0375) + time.sleep(0.042) self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.wait()) @@ -1131,7 +1139,7 @@ def gui_press_keys(self, keys): with gui_lock: for key in keys: pyautogui.press(key) - time.sleep(0.0375) + time.sleep(0.042) self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.wait()) @@ -1472,25 +1480,53 @@ def is_element_visible(self, selector): return True return False + def wait_for_element_visible( + self, selector, timeout=settings.SMALL_TIMEOUT + ): + try: + self.select(selector, timeout=timeout) + except Exception: + raise Exception("Element {%s} was not found!" % selector) + for i in range(30): + if self.is_element_visible(selector): + return self.select(selector) + time.sleep(0.1) + raise Exception("Element {%s} was not visible!" % selector) + def assert_element(self, selector, timeout=settings.SMALL_TIMEOUT): + """Same as assert_element_visible()""" try: self.select(selector, timeout=timeout) except Exception: - raise Exception("Element {%s} not found!" % selector) + raise Exception("Element {%s} was not found!" % selector) for i in range(30): if self.is_element_visible(selector): return True time.sleep(0.1) - raise Exception("Element {%s} not visible!" % selector) + raise Exception("Element {%s} was not visible!" % selector) + + def assert_element_visible(self, selector, timeout=settings.SMALL_TIMEOUT): + """Same as assert_element()""" + try: + self.select(selector, timeout=timeout) + except Exception: + raise Exception("Element {%s} was not found!" % selector) + for i in range(30): + if self.is_element_visible(selector): + return True + time.sleep(0.1) + raise Exception("Element {%s} was not visible!" % selector) def assert_element_present(self, selector, timeout=settings.SMALL_TIMEOUT): + """Assert element is present in the DOM. (Visibility NOT required)""" try: self.select(selector, timeout=timeout) except Exception: - raise Exception("Element {%s} not found!" % selector) + raise Exception("Element {%s} was not found!" % selector) return True def assert_element_absent(self, selector, timeout=settings.SMALL_TIMEOUT): + """Assert element is not present in the DOM.""" start_ms = time.time() * 1000.0 stop_ms = start_ms + (timeout * 1000.0) for i in range(int(timeout * 10)): @@ -1511,6 +1547,7 @@ def assert_element_absent(self, selector, timeout=settings.SMALL_TIMEOUT): def assert_element_not_visible( self, selector, timeout=settings.SMALL_TIMEOUT ): + """Assert element is not visible on page. (May still be in DOM)""" start_ms = time.time() * 1000.0 stop_ms = start_ms + (timeout * 1000.0) for i in range(int(timeout * 10)): @@ -1530,6 +1567,21 @@ def assert_element_not_visible( % (selector, timeout, plural) ) + def assert_element_attribute(self, selector, attribute, value=None): + attributes = self.get_element_attributes(selector) + if attribute not in attributes: + raise Exception( + "Attribute {%s} was not found in element {%s}!" + % (attribute, selector) + ) + if value and attributes[attribute] != value: + raise Exception( + "Expected value {%s} of attribute {%s} " + "was not found in element {%s}! " + "(Actual value was {%s})" + % (value, attribute, selector, attributes[attribute]) + ) + def assert_title(self, title): expected = title.strip() actual = self.get_title().strip() @@ -1541,11 +1593,55 @@ def assert_title(self, title): raise Exception(error % (expected, actual)) except Exception: time.sleep(2) - expected = title.strip() actual = self.get_title().strip() if expected != actual: raise Exception(error % (expected, actual)) + def assert_title_contains(self, substring): + expected = substring.strip() + actual = self.get_title().strip() + error = ( + "Expected title substring [%s] does not appear " + "in the actual page title [%s]!" + ) + try: + if expected not in actual: + raise Exception(error % (expected, actual)) + except Exception: + time.sleep(2) + actual = self.get_title().strip() + if expected not in actual: + raise Exception(error % (expected, actual)) + + def assert_url(self, url): + expected = url.strip() + actual = self.get_current_url().strip() + error = "Expected URL [%s] does not match the actual URL [%s]!" + try: + if expected != actual: + raise Exception(error % (expected, actual)) + except Exception: + time.sleep(2) + actual = self.get_current_url().strip() + if expected != actual: + raise Exception(error % (expected, actual)) + + def assert_url_contains(self, substring): + expected = substring.strip() + actual = self.get_current_url().strip() + error = ( + "Expected URL substring [%s] does not appear " + "in the full URL [%s]!" + ) + try: + if expected not in actual: + raise Exception(error % (expected, actual)) + except Exception: + time.sleep(2) + actual = self.get_current_url().strip() + if expected not in actual: + raise Exception(error % (expected, actual)) + def assert_text( self, text, selector="html", timeout=settings.SMALL_TIMEOUT ): diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index b9ab185fac4..b4ef0da46fe 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -1310,9 +1310,13 @@ def get_title(self): return self.get_page_title() def get_user_agent(self): + if self.__is_cdp_swap_needed(): + return self.cdp.get_user_agent() return self.execute_script("return navigator.userAgent;") def get_locale_code(self): + if self.__is_cdp_swap_needed(): + return self.cdp.get_locale_code() return self.execute_script( "return navigator.language || navigator.languages[0];" ) @@ -4547,6 +4551,11 @@ def get_cookie(self, name): def get_cookies(self): return self.driver.get_cookies() + def get_cookie_string(self): + if self.__is_cdp_swap_needed(): + return self.cdp.get_cookie_string() + return self.execute_script("return document.cookie;") + def add_cookie(self, cookie_dict, expiry=False): """Usage examples: self.add_cookie({'name': 'foo', 'value': 'bar'}) @@ -5429,7 +5438,10 @@ def __process_recorded_actions(self): new_file = True sb_config._recorded_actions[filename] = [] data.append("from seleniumbase import BaseCase") - data.append("BaseCase.main(__name__, __file__)") + if "--uc" in sys.argv: + data.append('BaseCase.main(__name__, __file__, "--uc")') + else: + data.append("BaseCase.main(__name__, __file__)") data.append("") data.append("") data.append("class %s(BaseCase):" % classname) @@ -5439,7 +5451,13 @@ def __process_recorded_actions(self): data.append("class %s(BaseCase):" % classname) data.append(" def %s(self):" % methodname) if len(sb_actions) > 0: + if "--uc" in sys.argv: + data.append(" self.activate_cdp_mode()") for action in sb_actions: + if "--uc" in sys.argv: + action = action.replace( + "self.type(", "self.press_keys(" + ) data.append(" " + action) else: data.append(" pass") @@ -5617,6 +5635,9 @@ def __process_recorded_behave_actions(self, srt_actions, colorama): data.append(" Scenario: %s" % scenario_test) if len(behave_actions) > 0: count = 0 + if "--uc" in sys.argv: + data.append(" Given Activate CDP Mode") + count += 1 for action in behave_actions: if count == 0: data.append(" Given " + action) @@ -7645,7 +7666,10 @@ def wait_for_attribute( if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT: timeout = self.__get_new_timeout(timeout) selector, by = self.__recalculate_selector(selector, by) - if self.__is_shadow_selector(selector): + if self.__is_cdp_swap_needed(): + self.cdp.assert_element_attribute(selector, attribute, value) + return + elif self.__is_shadow_selector(selector): return self.__wait_for_shadow_attribute_present( selector, attribute, value=value, timeout=timeout ) @@ -7670,6 +7694,9 @@ def assert_attribute( if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) selector, by = self.__recalculate_selector(selector, by) + if self.__is_cdp_swap_needed(): + self.cdp.assert_element_attribute(selector, attribute, value) + return self.wait_for_attribute( selector, attribute, value=value, by=by, timeout=timeout ) @@ -7768,6 +7795,9 @@ def assert_title_contains(self, substring): but then the title switches over to the actual page title. In Recorder Mode, this assertion is skipped because the Recorder changes the page title to the selector of the hovered element.""" + if self.__is_cdp_swap_needed(): + self.cdp.assert_title_contains(substring) + return self.wait_for_ready_state_complete() expected = substring.strip() actual = self.get_page_title().strip() @@ -7811,6 +7841,9 @@ def assert_title_contains(self, substring): def assert_url(self, url): """Asserts that the web page URL matches the expected URL.""" + if self.__is_cdp_swap_needed(): + self.cdp.assert_url(url) + return self.wait_for_ready_state_complete() expected = url.strip() actual = self.get_current_url().strip() @@ -7846,6 +7879,9 @@ def assert_url(self, url): def assert_url_contains(self, substring): """Asserts that the URL substring appears in the full URL.""" + if self.__is_cdp_swap_needed(): + self.cdp.assert_url_contains(substring) + return self.wait_for_ready_state_complete() expected = substring.strip() actual = self.get_current_url().strip() @@ -8044,6 +8080,8 @@ def is_alert_present(self): def is_online(self): """Return True if connected to the Internet.""" + if self.__is_cdp_swap_needed(): + return self.cdp.evaluate("navigator.onLine;") return self.execute_script("return navigator.onLine;") def is_connected(self): @@ -8952,7 +8990,9 @@ def wait_for_element_clickable( timeout = self.__get_new_timeout(timeout) original_selector = selector selector, by = self.__recalculate_selector(selector, by) - if self.__is_shadow_selector(selector): + if self.__is_cdp_swap_needed(): + return self.cdp.select(selector) + elif self.__is_shadow_selector(selector): # If a shadow selector, use visible instead of clickable return self.__wait_for_shadow_element_visible(selector, timeout) return page_actions.wait_for_element_clickable( @@ -9353,7 +9393,7 @@ def wait_for_element_present( selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): return self.cdp.select(selector) - if self.__is_shadow_selector(selector): + elif self.__is_shadow_selector(selector): return self.__wait_for_shadow_element_present(selector, timeout) return page_actions.wait_for_element_present( self.driver, @@ -9416,6 +9456,8 @@ def wait_for_query_selector( if self.timeout_multiplier and timeout == settings.LARGE_TIMEOUT: timeout = self.__get_new_timeout(timeout) css_selector = self.convert_to_css_selector(selector, by=by) + if self.__is_cdp_swap_needed(): + return self.cdp.select(css_selector) return js_utils.wait_for_css_query_selector( self.driver, css_selector, timeout ) @@ -9637,7 +9679,7 @@ def wait_for_text_visible( selector, by = self.__recalculate_selector(selector, by) if self.__is_cdp_swap_needed(): return self.cdp.find_element(selector) - if self.__is_shadow_selector(selector): + elif self.__is_shadow_selector(selector): return self.__wait_for_shadow_text_visible(text, selector, timeout) return page_actions.wait_for_text_visible( self.driver, text, selector, by, timeout @@ -10160,6 +10202,9 @@ def assert_element_not_visible( timeout = settings.SMALL_TIMEOUT if self.timeout_multiplier and timeout == settings.SMALL_TIMEOUT: timeout = self.__get_new_timeout(timeout) + if self.__is_cdp_swap_needed(): + self.cdp.assert_element_not_visible(selector) + return True self.wait_for_element_not_visible(selector, by=by, timeout=timeout) if self.recorder_mode and self.__current_url_is_recordable(): if self.get_session_storage_item("pause_recorder") == "no": @@ -14496,7 +14541,7 @@ def __wait_for_shadow_attribute_present( message = ( "Expected value {%s} for attribute {%s} of element " "{%s} was not present after %s second%s! " - "(The actual value was {%s})" + "(Actual value was {%s})" % ( value, attribute, diff --git a/seleniumbase/undetected/__init__.py b/seleniumbase/undetected/__init__.py index 24895b78ffd..696c888b956 100644 --- a/seleniumbase/undetected/__init__.py +++ b/seleniumbase/undetected/__init__.py @@ -455,7 +455,7 @@ def disconnect(self): if self.service.is_connectable(): self.stop_client() self.service.stop() - self._is_connected = False + self._is_connected = False def connect(self): """Starts the chromedriver service that runs in the background diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index 5eaa63de93f..7899436f2eb 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -211,7 +211,11 @@ async def start( expert=expert, **kwargs, ) - return await Browser.create(config) + try: + return await Browser.create(config) + except Exception: + time.sleep(0.15) + return await Browser.create(config) async def start_async(*args, **kwargs) -> Browser: diff --git a/setup.py b/setup.py index fed28f5952b..ca5948598d0 100755 --- a/setup.py +++ b/setup.py @@ -147,7 +147,7 @@ ], python_requires=">=3.8", install_requires=[ - 'pip>=24.2', + 'pip>=24.3.1', 'packaging>=24.2', 'setuptools~=70.2;python_version<"3.10"', # Newer ones had issues 'setuptools>=75.5.0;python_version>="3.10"', @@ -222,7 +222,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.6.1;python_version<"3.9"', - 'coverage>=7.6.5;python_version>="3.9"', + 'coverage>=7.6.7;python_version>="3.9"', 'pytest-cov>=5.0.0;python_version<"3.9"', 'pytest-cov>=6.0.0;python_version>="3.9"', ],