Proxy Rotation

stores/amazon_monitoring.py

@@ -28,6 +28,8 @@
 from fake_useragent import UserAgent
 from amazoncaptcha import AmazonCaptcha
 
+from itertools import cycle
+
 from urllib.parse import urlparse
 
 import re
@@ -66,22 +68,6 @@
     policy = asyncio.WindowsSelectorEventLoopPolicy()
     asyncio.set_event_loop_policy(policy)
 
-# PDP_URL = "https://smile.amazon.com/gp/product/"
-# AMAZON_DOMAIN = "www.amazon.com.au"
-# AMAZON_DOMAIN = "www.amazon.com.br"
-# AMAZON_DOMAIN = "www.amazon.ca"
-# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.cn"
-# AMAZON_DOMAIN = "www.amazon.fr"
-# AMAZON_DOMAIN = "www.amazon.de"
-# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.in"
-# AMAZON_DOMAIN = "www.amazon.it"
-# AMAZON_DOMAIN = "www.amazon.co.jp"
-# AMAZON_DOMAIN = "www.amazon.com.mx"
-# AMAZON_DOMAIN = "www.amazon.nl"
-# AMAZON_DOMAIN = "www.amazon.es"
-# AMAZON_DOMAIN = "www.amazon.co.uk"
-# AMAZON_DOMAIN = "www.amazon.com"
-# AMAZON_DOMAIN = "www.amazon.se"
 
 AMAZON_URLS = {
     "BASE_URL": "https://{domain}/",
@@ -95,6 +81,7 @@
 
 PDP_PATH = "/dp/"
 REALTIME_INVENTORY_PATH = "gp/aod/ajax?asin="
+BAD_PROXIES_PATH = "html_saves/bad_proxies.json"
 
 CONFIG_FILE_PATH = "config/amazon_requests_config.json"
 PROXY_FILE_PATH = "config/proxies.json"
@@ -111,6 +98,33 @@
 amazon_config = {}
 
 
+class ItemsHandler:
+    @classmethod
+    def create_items_pool(cls, item_list):
+        cls.items = cycle(item_list)
+
+    @classmethod
+    def assign_next_item(cls):
+        return next(cls.items)
+
+class BadProxyCollector:
+    @classmethod
+    def __init__(cls):
+        if os.path.exists(BAD_PROXIES_PATH):
+            with open(BAD_PROXIES_PATH) as f:
+                cls.collection = json.load(f)
+        else:
+            cls.collection = {}
+
+    @classmethod
+    def collect(cls, status, connector):
+        if status == 503:
+            proxy = str(connector.proxy_url)
+            cls.collection.update({proxy : "null"})
+            with open(BAD_PROXIES_PATH, "w") as f:
+                json.dump(cls.collection, f, indent=4)
+
+
 class AmazonMonitoringHandler(BaseStoreHandler):
     http_client = False
     http_20_client = False
@@ -123,8 +137,7 @@ def __init__(
         delay: float,
         amazon_config,
         tasks=1,
-        checkshipping=False,
-    ) -> None:
+        checkshipping=False,) -> None:
         log.debug("Initializing AmazonMonitoringHandler")
         super().__init__()
 
@@ -133,62 +146,46 @@ def __init__(
         self.notification_handler = notification_handler
         self.check_shipping = checkshipping
         self.item_list: typing.List[FGItem] = item_list
-        self.stock_checks = 0
         self.start_time = int(time.time())
         self.amazon_config = amazon_config
         ua = UserAgent()
 
         self.proxies = get_proxies(path=PROXY_FILE_PATH)
+        ItemsHandler.create_items_pool(self.item_list)
 
         # Initialize the Session we'll use for stock checking
         log.debug("Initializing Monitoring Sessions")
         self.sessions_list: Optional[List[AmazonMonitor]] = []
-        for idx in range(len(item_list)):
-            connector = None
-            if self.proxies and idx < len(self.proxies):
-                connector = ProxyConnector.from_url(self.proxies[idx]["https"])
+        for idx in range(len(self.proxies)):
+            connector = ProxyConnector.from_url(self.proxies[idx])
             self.sessions_list.append(
                 AmazonMonitor(
                     headers=HEADERS,
-                    item=item_list[idx],
                     amazon_config=self.amazon_config,
                     connector=connector,
                     delay=delay,
                 )
             )
             self.sessions_list[idx].headers.update({"user-agent": ua.random})
 
-
-# class Offers(NamedTuple):
-#     asin: str
-#     offerlistingid: str
-#     merchantid: str
-#     price: float
-#     timestamp: float
-#     __slots__ = ()
-#
-#     def __str__(self):
-#         return f"ASIN: {self.asin}; offerListingId: {self.offerlistingid}; merchantId: {self.merchantid}; price: {self.price}"
-
-
 class AmazonMonitor(aiohttp.ClientSession):
     def __init__(
         self,
-        item: FGItem,
         amazon_config: Dict,
         delay: float,
         *args,
         **kwargs,
     ):
         super(self.__class__, self).__init__(*args, **kwargs)
-        self.item = item
+        self.item = self.next_item()
+        self.check_count = 1
         self.amazon_config = amazon_config
         self.domain = urlparse(self.item.furl.url).netloc
 
         self.delay = delay
-        if item.purchase_delay > 0:
+        if self.item.purchase_delay > 0:
             self.delay = 20
-        self.block_purchase_until = time.time() + item.purchase_delay
+        self.block_purchase_until = time.time() + self.item.purchase_delay
         log.debug("Initializing Monitoring Task")
 
     def assign_config(self, azn_config):
@@ -197,14 +194,13 @@ def assign_config(self, azn_config):
     def assign_delay(self, delay: float = 5):
         self.delay = delay
 
-    def assign_item(self, item: FGItem):
-        self.item = item
+    def next_item(self):
+        return ItemsHandler.assign_next_item()
 
     def fail_recreate(self):
         # Something wrong, start a new task then kill this one
         log.debug("Max consecutive request fails reached. Restarting session")
         session = AmazonMonitor(
-            item=self.item,
             amazon_config=self.amazon_config,
             delay=self.delay,
             connector=self.connector,
@@ -218,14 +214,17 @@ async def stock_check(self, queue: asyncio.Queue, future: asyncio.Future):
         # Do first response outside of while loop, so we can continue on captcha checks
         # and return to start of while loop with that response. Requires the next response
         # to be grabbed at end of while loop
-        log.debug(f"Monitoring Task Started for {self.item.id}")
+
+        # log.debug(f"Monitoring Task Started for {self.item.id}")
+        collector = BadProxyCollector()
 
         fail_counter = 0  # Count sequential get fails
         delay = self.delay
         end_time = time.time() + delay
         status, response_text = await self.aio_get(url=self.item.furl.url)
 
-        save_html_response("stock-check", status, response_text)
+        # save_html_response("stock-check", status, response_text)
+        collector.collect(status, self.connector)
 
         # do this after each request
         fail_counter = check_fail(status=status, fail_counter=fail_counter)
@@ -234,10 +233,10 @@ async def stock_check(self, queue: asyncio.Queue, future: asyncio.Future):
             future.set_result(session)
             return
 
-        check_count = 1
+
         # Loop will only exit if a qualified seller is returned.
         while True:
-            log.debug(f"{self.item.id} Stock Check Count: {check_count}")
+            log.debug(f"{self.item.id} : {self.connector.proxy_url} : Stock Check Count = {self.check_count}")
             tree = check_response(response_text)
             if tree is not None:
                 if captcha_element := has_captcha(tree):
@@ -287,15 +286,17 @@ async def stock_check(self, queue: asyncio.Queue, future: asyncio.Future):
             await wait_timer(end_time)
             end_time = time.time() + delay
             status, response_text = await self.aio_get(url=self.item.furl.url)
-            save_html_response("stock-check", status, response_text)
+            # save_html_response("stock-check", status, response_text)
+            collector.collect(status, self.connector)
             # do this after each request
             fail_counter = check_fail(status=status, fail_counter=fail_counter)
             if fail_counter == -1:
                 session = self.fail_recreate()
                 future.set_result(session)
                 return
 
-            check_count += 1
+            self.check_count += 1
+            self.next_item()
 
     async def aio_get(self, url):
         text = None
@@ -526,6 +527,7 @@ def get_proxies(path=PROXY_FILE_PATH):
 
     return proxies
 
+
     # def verify(self):
     #     log.debug("Verifying item list...")
     #     items_to_purge = []
@@ -644,3 +646,34 @@ def get_proxies(path=PROXY_FILE_PATH):
     #     pickle.dump(item_cache, open(item_cache_file, "wb"))
     #
     #     return True
+
+
+# class Offers(NamedTuple):
+#     asin: str
+#     offerlistingid: str
+#     merchantid: str
+#     price: float
+#     timestamp: float
+#     __slots__ = ()
+#
+#     def __str__(self):
+#         return f"ASIN: {self.asin}; offerListingId: {self.offerlistingid}; merchantId: {self.merchantid}; price: {self.price}"
+
+
+# PDP_URL = "https://smile.amazon.com/gp/product/"
+# AMAZON_DOMAIN = "www.amazon.com.au"
+# AMAZON_DOMAIN = "www.amazon.com.br"
+# AMAZON_DOMAIN = "www.amazon.ca"
+# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.cn"
+# AMAZON_DOMAIN = "www.amazon.fr"
+# AMAZON_DOMAIN = "www.amazon.de"
+# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.in"
+# AMAZON_DOMAIN = "www.amazon.it"
+# AMAZON_DOMAIN = "www.amazon.co.jp"
+# AMAZON_DOMAIN = "www.amazon.com.mx"
+# AMAZON_DOMAIN = "www.amazon.nl"
+# AMAZON_DOMAIN = "www.amazon.es"
+# AMAZON_DOMAIN = "www.amazon.co.uk"
+# AMAZON_DOMAIN = "www.amazon.com"
+# AMAZON_DOMAIN = "www.amazon.se"
+