Remove usage of urlembed service

OpenToAllCTF · Apr 3, 2019 · 68c6062 · 68c6062
1 parent a7352be
commit 68c6062
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 15 deletions.
diff --git a/handlers/linksave_handler.py b/handlers/linksave_handler.py
@@ -36,15 +36,19 @@ def execute(cls, slack_wrapper, args, timestamp, channel_id, user_id, user_is_ad
             slack_wrapper.post_message(channel_id, "Save Link failed: Unable to extract URL", timestamp)
             return
 
-        url_data = unfurl(url.group())
+        try:
+            url_data = unfurl(url.group())
+        except requests.exceptions.Timeout as e:
+            slack_wrapper.post_message(channel_id, "Save Link failed: Request timed out", timestamp)
+            log.error(e)
+            return
 
         data = {
             "options[staticman-token]": LINKSAVE_CONFIG["staticman-token"],
             "fields[title]": url_data["title"],
             "fields[link]": url.group(),
             "fields[excerpt]": url_data["desc"],
             "fields[category]": args[0],
-            "fields[content]": url_data["content"],
             "fields[header][overlay_image]": url_data["img"],
             "fields[user]": profile_details["display_name"] or profile_details["real_name"]
         }

diff --git a/util/savelinkhelper.py b/util/savelinkhelper.py
@@ -10,7 +10,7 @@
 
 def get_title(soup: BeautifulSoup):
     title = soup.find("meta", property=re.compile("title", re.I)) or \
-        soup.find("meta", attrs={"name": re.compile("title", re.I)})
+            soup.find("meta", attrs={"name": re.compile("title", re.I)})
     if title:
         title = title["content"]
     else:
@@ -22,32 +22,30 @@ def get_title(soup: BeautifulSoup):
 
 def get_desc(soup: BeautifulSoup):
     desc = soup.find("meta", property=re.compile("desc", re.I)) or \
-        soup.find("meta", attrs={"name": re.compile("desc", re.I)})
+           soup.find("meta", attrs={"name": re.compile("desc", re.I)})
     if desc:
         return desc["content"].strip()
 
     return ""
 
 
-def get_content(url: str):
-    resp = requests.get("https://urlembed.com/json/url/{}".format(url))
-    if not resp.ok:
-        return "", ""
-    resp = resp.json()
-    content = BeautifulSoup(resp["content"], "html.parser").prettify()
-    return content, resp["url"]  # resp["url"] is image's URL
+def get_img(soup: BeautifulSoup):
+    img = soup.find("meta", property=re.compile("image", re.I)) or \
+          soup.find("meta", attrs={"name": re.compile("image", re.I)})
+    if img:
+        return img["content"].strip()
+
+    return ""
 
 
 def unfurl(url: str):
-    resp = requests.get(url).text
+    resp = requests.get(url, timeout=15).text
     soup = BeautifulSoup(resp, "html.parser")
-    content, img = get_content(url)
 
     details = {
         "title": get_title(soup),
         "desc": get_desc(soup),
-        "content": content,
-        "img": img
+        "img": get_img(soup)
     }
 
     return details