Fix prusa3d#4407 initial depenency bootstrapping fails

- replace urlretrieve with requests
- cleanup download_and_unzip function during bootstrap
- adjust python requirements for pipeline and dev
- fix docker scripts
- create .dependencies folder during bootstrapping in case it does not
  exist

requirements.txt

@@ -14,4 +14,5 @@ pytest~=7.3.2
 python-bidi==0.4.2
 pyyaml~=6.0
 qoi~=0.5.0
+requests==2.32.3
 setuptools~=70.1.0

utils/bootstrap.py

@@ -21,7 +21,8 @@
 import stat
 from argparse import ArgumentParser
 from pathlib import Path
-from urllib.request import urlretrieve
+from urllib.parse import urlparse
+import requests
 
 assert sys.version_info >= (3, 8), 'Python 3.8+ is required.'
 is_windows = platform.system() == 'Windows'
@@ -135,25 +136,53 @@ def find_single_subdir(path: Path):
         raise RuntimeError
 
 
+def download_url(url: str, filename: Path):
+    """Download file from url and write it to given filename"""
+    with requests.get(url, stream=True) as response:
+        response.raise_for_status()
+        with open(filename, 'wb') as file:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    file.write(chunk)
+
+
 def download_and_unzip(url: str, directory: Path):
     """Download a compressed file and extract it at `directory`."""
     extract_dir = directory.with_suffix('.temp')
     shutil.rmtree(directory, ignore_errors=True)
     shutil.rmtree(extract_dir, ignore_errors=True)
 
-    print('Downloading ' + directory.name)
-    f, _ = urlretrieve(url, filename=None)
-    print('Extracting ' + directory.name)
-    if '.tar.bz2' in url or '.tar.gz' in url or '.tar.xz' in url:
-        obj = tarfile.open(f)
+    print('Downloading ' + directory.name, end=" ")
+
+    # temporary local filepath
+    parsed_url = urlparse(url)
+    file = Path(parsed_url.path).name
+    filename = Path(dependencies_dir) / file
+
+    download_url(url=url, filename=filename)
+
+    print('done')
+    print('Extracting ' + file, end=" ")
+
+    # Check if tar or zip
+    if any(url.endswith(ext) for ext in ['.tar.bz2', '.tar.gz', '.tar.xz']):
+        with tarfile.open(filename) as obj:
+            obj.extractall(path=extract_dir)
     else:
-        obj = zipfile.ZipFile(f, 'r')
-    obj.extractall(path=str(extract_dir))
+        with zipfile.ZipFile(filename, 'r') as obj:
+            obj.extractall(path=extract_dir)
 
     subdir = find_single_subdir(extract_dir)
-    shutil.move(str(subdir), str(directory))
+    shutil.move(subdir, directory)
+
+    print('done')
+
+    # remove temp unzip folder
     shutil.rmtree(extract_dir, ignore_errors=True)
 
+    # remove downloaded zip
+    os.remove(filename)
+
 
 def run(*cmd):
     process = subprocess.run([str(a) for a in cmd],
@@ -204,7 +233,7 @@ def install_dependency(dependency):
         os.mkdir(installation_directory)
         for file in files:
             basename = file.split('/')[-1]
-            urlretrieve(file, installation_directory / basename)
+            download_url(url=file, filename=installation_directory / basename)
     else:
         raise ('dependency is missing payload')
 
@@ -284,6 +313,10 @@ def install_pip_packages():
 
 
 def bootstrap():
+    # create dependency directory if not exists
+    if not os.path.exists(dependencies_dir):
+        os.makedirs(dependencies_dir)
+
     for dependency in dependencies:
         if recommended_version_is_available(dependency):
             continue

utils/holly/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:24.04
 RUN apt-get clean && \
     apt-get update -qq -y && \
-    apt-get install libjpeg-dev git python3 python3-pip python3-venv -y
+    apt-get install libjpeg-dev git python3 python3-pip python3-venv python3-requests -y
 WORKDIR /work
 ADD utils/bootstrap.py utils/bootstrap.py
 ADD utils/debug/10_custom_config_template.cfg utils/debug/10_custom_config_template.cfg

utils/holly/build-requirements.txt

@@ -10,3 +10,4 @@ pyyaml~=6.0
 qoi~=0.5.0
 numpy==1.26.4
 setuptools~=70.1.0
+requests==2.32.3