feature(downloader): add option to control over stream mode (#541)

This PR introduces a new option `QDT_STREAMED_DOWNLOADS` to control over
stream mode for file downloader. By default, files are streamed but it
can lead to some errors. This setting helps to avoid buggy behaviors.

docs/usage/settings.md

@@ -24,6 +24,7 @@ Some others parameters can be set using environment variables.
 | `QDT_LOCAL_WORK_DIR` | Local folder where QDT download remote resources (profiles, plugins, etc.) | `~/.cache/qgis-deployment-toolbelt/default/` |
 | `QDT_LOGS_DIR` | Folder where QDT writes the log files, which are automatically rotated. | `~/.cache/qgis-deployment-toolbelt/logs/` |
 | `QDT_QGIS_EXE_PATH` | Path to the QGIS executable to use. Used in shortcuts. | `/usr/bin/qgis` on Linux and MacOS, `%PROGRAMFILES%/QGIS 3.28/bin/qgis-ltr-bin.exe` on Windows. |
+| `QDT_STREAMED_DOWNLOADS` | If set to `false`, the content of remote files is fully downloaded before being written locally. | `true` |
 | `QDT_SSL_USE_SYSTEM_STORES` | By default, a bundle of SSL certificates is used, through [certifi](https://pypi.org/project/certifi/). If this environment variable is set to True, QDT tries to uses the system certificates store. Based on [truststore](https://truststore.readthedocs.io/). See also [How to use custom SSL certificates](../guides/howto_use_custom_ssl_certs.md). | `False` |
 | `QDT_OSGEO4W_INSTALL_DIR` | Path to the OSGEO4W install directory. Used to search for installed QGIS and shortcuts creation. | `C:\\OSGeo4W`. |
 

qgis_deployment_toolbelt/commands/deployment.py

@@ -29,6 +29,7 @@
 from qgis_deployment_toolbelt.utils.check_path import check_path
 from qgis_deployment_toolbelt.utils.file_downloader import download_remote_file_to_local
 from qgis_deployment_toolbelt.utils.slugger import sluggy
+from qgis_deployment_toolbelt.utils.str2bool import str2bool
 
 # #############################################################################
 # ########## Globals ###############
@@ -73,6 +74,7 @@ def get_remote_scenario_from_url(remote_url: str) -> Path:
  get_qdt_working_directory().parent,
  local_filepath_for_remote_scenario,
  ),
+ use_stream=str2bool(getenv("QDT_STREAMED_DOWNLOADS", True)),
  )
 
 

qgis_deployment_toolbelt/commands/upgrade.py

@@ -273,6 +273,7 @@ def run(args: argparse.Namespace):
  remote_url_to_download=remote_url,
  local_file_path=dest_filepath,
  content_type=remote_content_type,
+ use_stream=str2bool(getenv("QDT_STREAMED_DOWNLOADS", True)),
  )
  except Exception as err:
  exit_cli_error(f"Download new version failed. Trace: {err}")

qgis_deployment_toolbelt/jobs/job_plugins_downloader.py

@@ -14,6 +14,7 @@
 # Standard library
 import logging
 from concurrent.futures import ThreadPoolExecutor
+from os import getenv
 from pathlib import Path
 from shutil import copy2
 
@@ -23,6 +24,7 @@
 from qgis_deployment_toolbelt.plugins.plugin import QgisPlugin
 from qgis_deployment_toolbelt.utils.check_path import check_path
 from qgis_deployment_toolbelt.utils.file_downloader import download_remote_file_to_local
+from qgis_deployment_toolbelt.utils.str2bool import str2bool
 
 # #############################################################################
 # ########## Globals ###############
@@ -233,6 +235,7 @@ def download_remote_plugins(
  local_file_path=plugin_download_path,
  remote_url_to_download=plugin.download_url,
  content_type="application/zip",
+ use_stream=str2bool(getenv("QDT_STREAMED_DOWNLOADS", True)),
  )
  logger.info(
  f"Plugin {plugin.name} from {plugin.download_url} "
@@ -267,6 +270,7 @@ def download_remote_plugins(
  local_file_path=plugin_download_path,
  remote_url_to_download=plugin.download_url,
  content_type="application/zip",
+ use_stream=str2bool(getenv("QDT_STREAMED_DOWNLOADS", True)),
  )
  downloaded_plugins.append(plugin)
  except Exception as err:

qgis_deployment_toolbelt/profiles/remote_http_handler.py

@@ -15,6 +15,7 @@
 # Standard library
 import logging
 from concurrent.futures import ThreadPoolExecutor
+from os import getenv
 from pathlib import Path
 from shutil import rmtree
 
@@ -29,6 +30,7 @@
 from qgis_deployment_toolbelt.utils.file_downloader import download_remote_file_to_local
 from qgis_deployment_toolbelt.utils.formatters import url_ensure_trailing_slash
 from qgis_deployment_toolbelt.utils.proxies import get_proxy_settings
+from qgis_deployment_toolbelt.utils.str2bool import str2bool
 from qgis_deployment_toolbelt.utils.tree_files_reader import tree_to_download_list
 
 # #############################################################################
@@ -154,6 +156,7 @@ def download_files_to_local(
  # func parameters
  local_file_path=target_folder.joinpath(file_to_download),
  remote_url_to_download=f"{base_url}{file_to_download}",
+ use_stream=str2bool(getenv("QDT_STREAMED_DOWNLOADS", True)),
  )
  downloaded_files.append(
  (

qgis_deployment_toolbelt/utils/file_downloader.py

@@ -12,7 +12,7 @@
 
 # 3rd party
 import truststore
-from requests import Session
+from requests import Response, Session
 from requests.exceptions import ConnectionError, HTTPError
 from requests.utils import requote_uri
 
@@ -44,7 +44,8 @@ def download_remote_file_to_local(
  user_agent: str = f"{__title_clean__}/{__version__}",
  content_type: str | None = None,
  chunk_size: int = 8192,
- timeout=(800, 800),
+ timeout: tuple[int, int] = (800, 800),
+ use_stream: bool = True,
 ) -> Path:
  """Check if the local index file exists. If not, download the search index from \
  remote URL. If it does exist, check if it has been modified.
@@ -57,7 +58,8 @@ def download_remote_file_to_local(
  content_type (str | None, optional): HTTP content-type. Defaults to None.
  chunk_size (int, optional): size of each chunk to read and write in bytes. \
  Defaults to 8192.
- timeout (tuple, optional): custom timeout (request, response). Defaults to (800, 800).
+ timeout (tuple[int, int], optional): custom timeout (request, response). Defaults to (800, 800).
+ use_stream (bool, optional): Option to enable/disable streaming download. Defaults to True.
 
  Returns:
  Path: path to the local file (should be the same as local_file_path)
@@ -84,20 +86,34 @@ def download_remote_file_to_local(
  url=requote_uri(remote_url_to_download), stream=True, timeout=timeout
  ) as req:
  req.raise_for_status()
+ if use_stream:
+ with local_file_path.open(mode="wb") as buffile:
+ for chunk in req.iter_content(chunk_size=chunk_size):
+ if chunk:
+ buffile.write(chunk)
+ else:
+ # Download download the entire content at once
+ local_file_path.write_bytes(req.content)
 
- with local_file_path.open(mode="wb") as buffile:
- for chunk in req.iter_content(chunk_size=chunk_size):
- if chunk:
- buffile.write(chunk)
  logger.info(
  f"Downloading {remote_url_to_download} to {local_file_path} "
  f"({convert_octets(local_file_path.stat().st_size)}) succeeded."
  )
  except HTTPError as error:
  logger.error(
  f"Downloading {remote_url_to_download} to {local_file_path} failed. "
- f"Cause: HTTPError. Trace: {error}"
+ f"Cause: HTTPError. Trace: {error}."
  )
+ if isinstance(req, Response):
+ http_error_details = {
+ "status": req.status_code,
+ "headers": req.headers,
+ "body": req.content,
+ }
+ logger.error(
+ f"Addtional details grabbed from HTTP response: {http_error_details}"
+ )
+
  raise error
  except ConnectionError as error:
  logger.error(

tests/test_utils_file_downloader.py

@@ -43,6 +43,20 @@ def test_download_file_exists(self):
  self.assertTrue(downloaded_file.exists())
  self.assertTrue(downloaded_file.is_file())
 
+ # disabling stream mode
+ with tempfile.TemporaryDirectory(
+ prefix="qdt_test_downloader_nostream_", ignore_cleanup_errors=True
+ ) as tmpdirname:
+ # file that already exist locally
+ downloaded_file = download_remote_file_to_local(
+ remote_url_to_download="https://raw.githubusercontent.com/Guts/qgis-deployment-cli/main/README.md",
+ local_file_path=Path(tmpdirname).joinpath("README_from_remote.md"),
+ use_stream=False,
+ )
+ self.assertIsInstance(downloaded_file, Path)
+ self.assertTrue(downloaded_file.exists())
+ self.assertTrue(downloaded_file.is_file())
+
  def test_download_file_raise_http_error(self):
  """Test download handling an HTTP error."""
  with tempfile.TemporaryDirectory(