"urllib" to "requests" library change

openedx2zim/annex.py

@@ -3,7 +3,7 @@
 import uuid
 import json
 import pathlib
-import collections
+from collections import OrderedDict, defaultdict
 
 from bs4 import BeautifulSoup
 
@@ -17,7 +17,7 @@ class MoocForum:
  def __init__(self, scraper):
  self.scraper = scraper
  self.threads = []
- self.categories = collections.OrderedDict()
+ self.categories = OrderedDict()
  self.staff_user = []
  self.output_path = self.scraper.build_dir.joinpath("forum")
  self.output_path.mkdir(parents=True, exist_ok=True)
@@ -219,7 +219,7 @@ def annex_forum(self):
  self.update_thread_children(thread)
 
  def render_forum(self):
- thread_by_category = collections.defaultdict(list)
+ thread_by_category = defaultdict(list)
  for thread in self.threads:
  thread_by_category[thread["commentable_id"]].append(thread)
  jinja(

openedx2zim/html_processor.py

@@ -1,6 +1,6 @@
 import pathlib
 import re
-import urllib
+import requests
 
 import xxhash
 import lxml.html
@@ -26,9 +26,10 @@ def download_and_get_filename(
  """downloads a file from src and return the name of the downloaded file
 
  with_ext: ensure that downloaded file has the given extension
- filter_ext: download only if the file to download has an extension in this list"""
+ filter_ext: download only if the file to download has an extension in this list
+ """
 
- server_path = pathlib.Path(urllib.parse.urlparse(src).path)
+ server_path = pathlib.Path(requests.utils.urlparse(src).path)
  ext = with_ext if with_ext else server_path.suffix
 
  if server_path.suffix:
@@ -57,7 +58,8 @@ def download_dependencies_from_css(
 
  - css_org_url: URL to the CSS file on the internet
  - css_path: path of CSS on the filesystem (Path)
- - output_path_from_css: string representing path of the output directory relative to css_path"""
+ - output_path_from_css: string representing path of the output directory relative to css_path
+ """
 
  def encapsulate(url):
  return f"url({url})"
@@ -71,7 +73,7 @@ def remove_quotes(url):
 
  # ensure the original CSS url has netloc
  css_org_url = prepare_url(css_org_url, netloc, path_on_server)
- css_org_url = urllib.parse.urlparse(css_org_url)
+ css_org_url = requests.utils.urlparse(css_org_url)
 
  with open(css_path, "r") as fp:
  content = fp.read()
@@ -93,7 +95,7 @@ def remove_quotes(url):
  continue
 
  # add netloc if not present
- parsed_url = urllib.parse.urlparse(css_url)
+ parsed_url = requests.utils.urlparse(css_url)
  if parsed_url.netloc == "":
  if parsed_url.path.startswith("/"):
  css_url = (
@@ -120,7 +122,7 @@ def remove_quotes(url):
  path_on_server=path_on_server,
  with_ext=".css",
  )
- parsed_css_url = urllib.parse.urlparse(css_url)
+ parsed_css_url = requests.utils.urlparse(css_url)
  self.download_dependencies_from_css(
  css_org_url=css_url,
  css_path=output_path.joinpath(filename),
@@ -147,7 +149,7 @@ def remove_quotes(url):
  def download_images_from_html(
  self, html_body, output_path, path_from_html, netloc, path_on_server
  ):
- """ download images from <img> tag and fix path """
+ """download images from <img> tag and fix path"""
 
  imgs = html_body.xpath("//img")
  for img in imgs:
@@ -171,7 +173,7 @@ def download_images_from_html(
  return bool(imgs)
 
  def get_root_from_asset(self, path_from_html, root_from_html):
- """ get path to root from the downloaded/generated asset """
+ """get path to root from the downloaded/generated asset"""
 
  # return original root if path_from_html is empty
  if path_from_html == "":
@@ -200,7 +202,7 @@ def download_documents_from_html(
  netloc,
  path_on_server,
  ):
- """ download documents from <a> tag and fix path """
+ """download documents from <a> tag and fix path"""
 
  anchors = html_body.xpath("//a")
  for anchor in anchors:
@@ -242,7 +244,7 @@ def get_path_and_netloc_to_send(self, netloc, path_on_server, downloaded_asset_u
  """get the path and netloc to send recursively after downloading asset from downloaded_asset_url
  path_on_server is the current path on server and netloc is the current netloc"""
 
- parsed_src = urllib.parse.urlparse(downloaded_asset_url)
+ parsed_src = requests.utils.urlparse(downloaded_asset_url)
  path_recursive = path_on_server
  if parsed_src.path:
  asset_path_on_server = pathlib.Path(parsed_src.path)
@@ -262,7 +264,7 @@ def get_path_and_netloc_to_send(self, netloc, path_on_server, downloaded_asset_u
  def download_css_from_html(
  self, html_body, output_path, path_from_html, netloc, path_on_server
  ):
- """ download css files from <link> tag and fix path """
+ """download css files from <link> tag and fix path"""
 
  css_files = html_body.xpath("//link")
  for css in css_files:
@@ -298,7 +300,7 @@ def download_css_from_html(
  def download_js_from_html(
  self, html_body, output_path, path_from_html, netloc, path_on_server
  ):
- """ download javascript from <script> tag and fix path """
+ """download javascript from <script> tag and fix path"""
 
  js_files = html_body.xpath("//script")
  for js in js_files:
@@ -320,7 +322,7 @@ def download_js_from_html(
  def download_sources_from_html(
  self, html_body, output_path, path_from_html, netloc, path_on_server
  ):
- """ downloads content from <source> tags """
+ """downloads content from <source> tags"""
 
  sources = html_body.xpath("//source")
  for source in sources:
@@ -348,7 +350,7 @@ def download_iframes_from_html(
  netloc,
  path_on_server,
  ):
- """ download youtube videos and pdf files from iframes in html content """
+ """download youtube videos and pdf files from iframes in html content"""
 
  iframes = html_body.xpath("//iframe")
  for iframe in iframes:
@@ -395,7 +397,9 @@ def download_iframes_from_html(
  # handle iframe recursively
  iframe_url = prepare_url(src, netloc)
  try:
- src_content = self.scraper.instance_connection.get_page(iframe_url)
+ src_content = self.scraper.instance_connection.get_page(
+ iframe_url
+ )
  except Exception:
  continue
  path_recursive, netloc_recursive = self.get_path_and_netloc_to_send(
@@ -425,7 +429,7 @@ def download_iframes_from_html(
  return bool(iframes)
 
  def handle_jump_to_paths(self, target_path):
- """ return a fixed path in zim for a inter-xblock path containing jump_to """
+ """return a fixed path in zim for a inter-xblock path containing jump_to"""
 
  def check_descendants_and_return_path(xblock_extractor):
  if xblock_extractor.xblock_json["type"] in ["vertical", "course"]:
@@ -436,15 +440,17 @@ def check_descendants_and_return_path(xblock_extractor):
 
  for xblock_extractor in self.scraper.xblock_extractor_objects:
  if (xblock_extractor.xblock_json["block_id"] == target_path.parts[-1]) or (
- urllib.parse.urlparse(xblock_extractor.xblock_json["lms_web_url"]).path
+ requests.utils.urlparse(
+ xblock_extractor.xblock_json["lms_web_url"]
+ ).path
  == str(target_path)
  ):
  # we have a path match, we now check xblock type to redirect properly
  # Only vertical and course xblocks have HTMLs
  return check_descendants_and_return_path(xblock_extractor)
 
  def rewrite_internal_links(self, html_body, root_from_html, netloc):
- """ rewrites internal links and ensures no root-relative links are left behind """
+ """rewrites internal links and ensures no root-relative links are left behind"""
 
  def update_root_relative_path(anchor, fixed_path, root_from_html, netloc):
  """updates a root-relative path to the fixed path in zim
@@ -456,12 +462,12 @@ def update_root_relative_path(anchor, fixed_path, root_from_html, netloc):
  anchor.attrib["href"] = netloc + anchor.attrib["href"]
 
  anchors = html_body.xpath("//a")
- path_prefix = f"{self.scraper.instance_config['course_prefix']}{urllib.parse.unquote_plus(self.scraper.course_id)}"
+ path_prefix = f"{self.scraper.instance_config['course_prefix']}{requests.utils.unquote(self.scraper.course_id)}"
  has_changed = False
  for anchor in anchors:
  if "href" not in anchor.attrib:
  continue
- src = urllib.parse.urlparse(anchor.attrib["href"])
+ src = requests.utils.urlparse(anchor.attrib["href"])
 
  # ignore external links
  if src.netloc and src.netloc != self.scraper.instance_url:
@@ -507,7 +513,7 @@ def dl_dependencies_and_fix_links(
  netloc=None,
  path_on_server="",
  ):
- """ downloads all static dependencies from an HTML content, and fixes links """
+ """downloads all static dependencies from an HTML content, and fixes links"""
 
  if not netloc:
  netloc = self.scraper.instance_url
@@ -551,7 +557,7 @@ def dl_dependencies_and_fix_links(
  return content
 
  def defer_scripts(self, content, output_path, path_from_html):
- """ defer all scripts in content. For inline scripts, they're placed in a *.js file and deferred """
+ """defer all scripts in content. For inline scripts, they're placed in a *.js file and deferred"""
 
  soup = BeautifulSoup(content, "lxml")
  script_tags = soup.find_all("script")
@@ -590,7 +596,8 @@ def extract_head_css_js(self, soup, output_path, path_from_html, root_from_html)
 
  output_path: a Path object to store the downloaded CSS/JS to
  path_from_html: a string representing the path to output_path from the resultant HTML
- root_from_html: a string representing the path to the root from the resultant HTML"""
+ root_from_html: a string representing the path to the root from the resultant HTML
+ """
 
  html_headers = soup.find("head")
  head_css_js = (
@@ -622,7 +629,8 @@ def extract_body_end_scripts(
 
  output_path: a Path object to store the downloaded CSS/JS to
  path_from_html: a string representing the path to output_path from the resultant HTML
- root_from_html: a string representing the path to the root from the resultant HTML"""
+ root_from_html: a string representing the path to the root from the resultant HTML
+ """
 
  html_body = soup.find("body")
  body_scripts = html_body.find_all("script", recursive=False)

openedx2zim/instance_connection.py

@@ -3,7 +3,7 @@
 import http
 import json
 import sys
-import urllib
+import requests
 
 from .constants import getLogger, LANGUAGE_COOKIES, OPENEDX_LANG_MAP
 
@@ -25,17 +25,17 @@ def __init__(self, email, password, instance_config, locale, build_dir, debug):
  self.debug = debug
 
  def get_response(self, url, post_data, headers, max_attempts=5):
- req = urllib.request.Request(url, post_data, headers)
+ req = requests.post(url, data=post_data, headers=headers)
  for attempt in range(max_attempts):
  try:
- return urllib.request.urlopen(req).read().decode("utf-8")
- except urllib.error.HTTPError as exc:
+ requests.urlopen(req).content.decode("utf-8")
+ except requests.exceptions.HTTPError as exc:
  logger.debug(f"HTTP Error (won't retry this kind of error) while opening {url}: {exc}")
  if self.debug:
  responseData = exc.read().decode("utf8", 'ignore')
  print(responseData, file=sys.stderr)
  raise exc
- except urllib.error.URLError as exc:
+ except requests.exceptions.RequestException as exc:
  if attempt < max_attempts - 1:
  logger.debug(f"Error opening {url}: {exc}\nRetrying ...")
  continue
@@ -56,11 +56,9 @@ def update_csrf_token_in_headers(self):
  self.headers.update({"X-CSRFToken": csrf_token})
 
  def generate_connection_headers(self):
- opener = urllib.request.build_opener(
- urllib.request.HTTPCookieProcessor(self.cookie_jar)
- )
+ opener = requests.Session().cookies.update(self.cookie_jar)
  opener.addheaders = [("User-Agent", "Mozilla/5.0")]
- urllib.request.install_opener(opener)
+ requests.Session().cookies = opener.cookies
  opener.open(self.instance_config["instance_url"] + "/login")
  self.headers = {
  "User-Agent": "Mozilla/5.0",
@@ -74,9 +72,7 @@ def generate_connection_headers(self):
 
  def establish_connection(self):
  self.generate_connection_headers()
- post_data = urllib.parse.urlencode(
- {"email": self.email, "password": self.password, "remember": False}
- ).encode("utf-8")
+ post_data = {"email": self.email, "password": self.password, "remember": False}
  # API login can also be used : /user_api/v1/account/login_session/
  self.instance_connection = self.get_api_json(
  self.instance_config["login_page"], post_data, max_attempts=1
@@ -122,5 +118,5 @@ def get_page(self, url):
 
  def get_redirection(self, url):
  self.update_csrf_token_in_headers()
- req = urllib.request.Request(url, None, self.headers)
- return urllib.request.urlopen(req).geturl()
+ response = requests.get(url, headers=self.headers, allow_redirects=False)
+ return response.headers.get('Location', response.url)