Merge pull request #142 from Crinibus/dev

Add timeout to requests.get, simplify try except in method "Scraper.get_info" and format with Black
Crinibus · May 19, 2021 · 76b65c1 · 76b65c1
2 parents 475a589 + c76f7e7
commit 76b65c1
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 9 deletions.
diff --git a/scraper/scrape.py b/scraper/scrape.py
@@ -26,7 +26,7 @@ def scrape_info(self) -> None:
     def request_url(url: str) -> BeautifulSoup:
         try:
             response = requests.get(
-                url, headers=REQUEST_HEADER, cookies=REQUEST_COOKIES
+                url, headers=REQUEST_HEADER, cookies=REQUEST_COOKIES, timeout=10
             )
             return BeautifulSoup(response.text, "html.parser")
         except requests.exceptions.RequestException:  # temporary try expect for all requests errors
@@ -36,16 +36,11 @@ def get_info(self, soup: BeautifulSoup) -> None:
         try:
             website_function = get_website_function(self.website_name)
             self.info = website_function(soup)
-        except AttributeError:
+        except (AttributeError, ValueError):
             self.logger.warning(
                 f"Could not get all the data needed from url: {self.url}"
             )
             self.info = Info(None, None, None, valid=False)
-        except ValueError:
-            self.logger.warning(
-                f"Could not get a price, maybe the product is no longer for sale - url: {self.url}"
-            )
-            self.info = Info(None, None, None, valid=False)
 
     def save_info(self) -> None:
         if self.info.valid:
@@ -77,7 +72,9 @@ def update_data(self) -> dict:
             if latest_datapoint["date"] == date:
                 latest_datapoint["price"] = self.info.price
             else:
-                product_info["datapoints"].append({"date": date, "price": self.info.price})
+                product_info["datapoints"].append(
+                    {"date": date, "price": self.info.price}
+                )
         else:
             product_info["datapoints"].append({"date": date, "price": self.info.price})
 

diff --git a/scraper/visualize.py b/scraper/visualize.py
@@ -99,7 +99,9 @@ def format_data() -> Generator[dict, None, None]:
 
             for website_name, website_info in product_info.items():
                 dates = [datapoint["date"] for datapoint in website_info["datapoints"]]
-                prices = [datapoint["price"] for datapoint in website_info["datapoints"]]
+                prices = [
+                    datapoint["price"] for datapoint in website_info["datapoints"]
+                ]
                 product_data["websites"].append(
                     {
                         "website_name": website_name,