add proxy manager in scraper

krankir · Jul 14, 2023 · 6258f00 · 6258f00
1 parent 60b2981
commit 6258f00
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 1 deletion.
diff --git a/mvideo_scrap_bot/requirements.txt b/mvideo_scrap_bot/requirements.txt
@@ -51,3 +51,4 @@ uvicorn==0.22.0
 vine==5.0.0
 wcwidth==0.2.6
 yarl==1.9.2
+proxy-manager-g4==1.1.0
diff --git a/requirements.txt b/requirements.txt
@@ -51,3 +51,4 @@ uvicorn==0.22.0
 vine==5.0.0
 wcwidth==0.2.6
 yarl==1.9.2
+proxy-manager-g4==1.1.0
diff --git a/scrap_1_hour/requirements.txt b/scrap_1_hour/requirements.txt
@@ -51,3 +51,4 @@ uvicorn==0.22.0
 vine==5.0.0
 wcwidth==0.2.6
 yarl==1.9.2
+proxy-manager-g4==1.1.0
diff --git a/scrap_data/models.py b/scrap_data/models.py
@@ -3,5 +3,5 @@
 
 class Item(BaseModel):
     modelName: str
-    description: str
+    description: str = 'Описание отсутствует'
     rating: dict[str, float]
diff --git a/scrap_data/scrap_main.py b/scrap_data/scrap_main.py
@@ -5,6 +5,14 @@
     cookies, headers, cookies_price, headers_price,
 )
 from scrap_data.models import Item
+from proxy_manager_g4 import ProxyManager
+from proxy_manager_g4.consts import PROTOCOL_HTTPS
+
+
+proxy_manager = ProxyManager(protocol=PROTOCOL_HTTPS, anonymity=True)
+
+pr = proxy_manager.get_random()
+proxy = {pr.ip: pr.port}
 
 
 class ScrapDataProduct:
@@ -29,6 +37,7 @@ def scrap(self):
                                 cookies=cookies,
                                 headers=headers,
                                 timeout=10,
+                                proxies=proxy
                                 )
         products_infos = Item.parse_obj(response.json()['body'])
         data_dict = {
@@ -55,6 +64,7 @@ def scrap_price(self):
             cookies=cookies_price,
             headers=headers_price,
             timeout=10,
+            proxies=proxy
         )
         price = (response_pr.json()).get('body').get('materialPrices')[0].get(
             'price').get('salePrice')