From 7999f6b0f65950acb3344625a8e8542d03c0fd64 Mon Sep 17 00:00:00 2001 From: Ludeeus Date: Tue, 28 Dec 2021 10:34:53 +0000 Subject: [PATCH] Move download utils to HacsRepository --- custom_components/hacs/repositories/base.py | 130 +++++++++++++++++- custom_components/hacs/utils/download.py | 129 ----------------- .../download/test_gather_files_to_download.py | 30 ++-- .../download/test_should_try_releases.py | 13 +- tests/helpers/functions/test_logger.py | 1 + 5 files changed, 148 insertions(+), 155 deletions(-) delete mode 100644 custom_components/hacs/utils/download.py diff --git a/custom_components/hacs/repositories/base.py b/custom_components/hacs/repositories/base.py index 90325e7cb87..917d3d7c719 100644 --- a/custom_components/hacs/repositories/base.py +++ b/custom_components/hacs/repositories/base.py @@ -5,6 +5,7 @@ from datetime import datetime import json import os +import pathlib import shutil import tempfile from typing import TYPE_CHECKING, Any, List, Optional @@ -25,7 +26,8 @@ ) from ..utils.backup import Backup, BackupNetDaemon from ..utils.decode import decode_content -from ..utils.download import dowload_repository_content, gather_files_to_download +from ..utils.decorator import concurrent +from ..utils.filters import filter_content_return_one_of_type from ..utils.logger import get_hacs_logger from ..utils.path import is_safe from ..utils.queue_manager import QueueManager @@ -42,6 +44,15 @@ from ..base import HacsBase +class FileInformation: + """FileInformation.""" + + def __init__(self, url, path, name): + self.download_url = url + self.path = path + self.name = name + + @attr.s(auto_attribs=True) class RepositoryData: """RepositoryData class.""" @@ -465,6 +476,21 @@ def localpath(self) -> str | None: """Return localpath.""" return None + @property + def should_try_releases(self) -> bool: + """Return a boolean indicating whether to download releases or not.""" + if self.data.zip_release: + if self.data.filename.endswith(".zip"): + if self.ref != self.data.default_branch: + return True + if self.ref == self.data.default_branch: + return False + if self.data.category not in ["plugin", "theme"]: + return False + if not self.data.releases: + return False + return True + async def validate_repository(self) -> None: """Validate.""" @@ -605,7 +631,7 @@ def cleanup_temp_dir(): async def download_content(self) -> None: """Download the content of a directory.""" - contents = gather_files_to_download(self) + contents = self.gather_files_to_download() self.logger.debug(self.data.filename) if not contents: raise HacsException("No content to download") @@ -616,7 +642,7 @@ async def download_content(self) -> None: if self.data.content_in_root and self.data.filename: if content.name != self.data.filename: continue - download_queue.add(dowload_repository_content(self, content)) + download_queue.add(self.dowload_repository_content(content)) await download_queue.execute() async def async_get_hacs_json(self, ref: str = None) -> dict[str, Any] | None: @@ -985,3 +1011,101 @@ async def common_update_data(self, ignore_issues: bool = False, force: bool = Fa self.logger.error("%s %s", self, exception) if not ignore_issues: raise HacsException(exception) from None + + def gather_files_to_download(self) -> list[FileInformation]: + """Return a list of file objects to be downloaded.""" + files = [] + tree = self.tree + ref = f"{self.ref}".replace("tags/", "") + releaseobjects = self.releases.objects + category = self.data.category + remotelocation = self.content.path.remote + + if self.should_try_releases: + for release in releaseobjects or []: + if ref == release.tag_name: + for asset in release.assets or []: + files.append(asset) + if files: + return files + + if self.content.single: + for treefile in tree: + if treefile.filename == self.data.file_name: + files.append( + FileInformation( + treefile.download_url, treefile.full_path, treefile.filename + ) + ) + return files + + if category == "plugin": + for treefile in tree: + if treefile.path in ["", "dist"]: + if remotelocation == "dist" and not treefile.filename.startswith("dist"): + continue + if not remotelocation: + if not treefile.filename.endswith(".js"): + continue + if treefile.path != "": + continue + if not treefile.is_directory: + files.append( + FileInformation( + treefile.download_url, treefile.full_path, treefile.filename + ) + ) + if files: + return files + + if self.data.content_in_root: + if not self.data.filename: + if category == "theme": + tree = filter_content_return_one_of_type(self.tree, "", "yaml", "full_path") + + for path in tree: + if path.is_directory: + continue + if path.full_path.startswith(self.content.path.remote): + files.append(FileInformation(path.download_url, path.full_path, path.filename)) + return files + + @concurrent(10) + async def dowload_repository_content(self, content: FileInformation) -> None: + """Download content.""" + try: + self.logger.debug("Downloading %s", content.name) + + filecontent = await self.hacs.async_download_file(content.download_url) + + if filecontent is None: + self.validate.errors.append(f"[{content.name}] was not downloaded.") + return + + # Save the content of the file. + if self.content.single or content.path is None: + local_directory = self.content.path.local + + else: + _content_path = content.path + if not self.data.content_in_root: + _content_path = _content_path.replace(f"{self.content.path.remote}", "") + + local_directory = f"{self.content.path.local}/{_content_path}" + local_directory = local_directory.split("/") + del local_directory[-1] + local_directory = "/".join(local_directory) + + # Check local directory + pathlib.Path(local_directory).mkdir(parents=True, exist_ok=True) + + local_file_path = (f"{local_directory}/{content.name}").replace("//", "/") + + result = await self.hacs.async_save_file(local_file_path, filecontent) + if result: + self.logger.info("Download of %s completed", content.name) + return + self.validate.errors.append(f"[{content.name}] was not downloaded.") + + except BaseException as exception: # pylint: disable=broad-except + self.validate.errors.append(f"Download was not completed [{exception}]") diff --git a/custom_components/hacs/utils/download.py b/custom_components/hacs/utils/download.py deleted file mode 100644 index cce98e50d3c..00000000000 --- a/custom_components/hacs/utils/download.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Helpers to download repository content.""" -from __future__ import annotations - -import pathlib - -from ..utils import filters -from ..utils.decorator import concurrent - - -class FileInformation: - def __init__(self, url, path, name): - self.download_url = url - self.path = path - self.name = name - - -def should_try_releases(repository): - """Return a boolean indicating whether to download releases or not.""" - if repository.data.zip_release: - if repository.data.filename.endswith(".zip"): - if repository.ref != repository.data.default_branch: - return True - if repository.ref == repository.data.default_branch: - return False - if repository.data.category not in ["plugin", "theme"]: - return False - if not repository.data.releases: - return False - return True - - -def gather_files_to_download(repository): - """Return a list of file objects to be downloaded.""" - files = [] - tree = repository.tree - ref = f"{repository.ref}".replace("tags/", "") - releaseobjects = repository.releases.objects - category = repository.data.category - remotelocation = repository.content.path.remote - - if should_try_releases(repository): - for release in releaseobjects or []: - if ref == release.tag_name: - for asset in release.assets or []: - files.append(asset) - if files: - return files - - if repository.content.single: - for treefile in tree: - if treefile.filename == repository.data.file_name: - files.append( - FileInformation(treefile.download_url, treefile.full_path, treefile.filename) - ) - return files - - if category == "plugin": - for treefile in tree: - if treefile.path in ["", "dist"]: - if remotelocation == "dist" and not treefile.filename.startswith("dist"): - continue - if not remotelocation: - if not treefile.filename.endswith(".js"): - continue - if treefile.path != "": - continue - if not treefile.is_directory: - files.append( - FileInformation( - treefile.download_url, treefile.full_path, treefile.filename - ) - ) - if files: - return files - - if repository.data.content_in_root: - if not repository.data.filename: - if category == "theme": - tree = filters.filter_content_return_one_of_type( - repository.tree, "", "yaml", "full_path" - ) - - for path in tree: - if path.is_directory: - continue - if path.full_path.startswith(repository.content.path.remote): - files.append(FileInformation(path.download_url, path.full_path, path.filename)) - return files - - -@concurrent(10) -async def dowload_repository_content(repository, content): - """Download content.""" - try: - repository.logger.debug(f"Downloading {content.name}") - - filecontent = await repository.hacs.async_download_file(content.download_url) - - if filecontent is None: - repository.validate.errors.append(f"[{content.name}] was not downloaded.") - return - - # Save the content of the file. - if repository.content.single or content.path is None: - local_directory = repository.content.path.local - - else: - _content_path = content.path - if not repository.data.content_in_root: - _content_path = _content_path.replace(f"{repository.content.path.remote}", "") - - local_directory = f"{repository.content.path.local}/{_content_path}" - local_directory = local_directory.split("/") - del local_directory[-1] - local_directory = "/".join(local_directory) - - # Check local directory - pathlib.Path(local_directory).mkdir(parents=True, exist_ok=True) - - local_file_path = (f"{local_directory}/{content.name}").replace("//", "/") - - result = await repository.hacs.async_save_file(local_file_path, filecontent) - if result: - repository.logger.info(f"Download of {content.name} completed") - return - repository.validate.errors.append(f"[{content.name}] was not downloaded.") - - except BaseException as exception: # pylint: disable=broad-except - repository.validate.errors.append(f"Download was not completed [{exception}]") diff --git a/tests/helpers/download/test_gather_files_to_download.py b/tests/helpers/download/test_gather_files_to_download.py index 94d55c7b1c3..d9a16910161 100644 --- a/tests/helpers/download/test_gather_files_to_download.py +++ b/tests/helpers/download/test_gather_files_to_download.py @@ -3,8 +3,6 @@ from aiogithubapi.objects.repository.content import AIOGitHubAPIRepositoryTreeContent from aiogithubapi.objects.repository.release import AIOGitHubAPIRepositoryRelease -from custom_components.hacs.utils.download import gather_files_to_download - def test_gather_files_to_download(repository): repository.content.path.remote = "" @@ -13,7 +11,7 @@ def test_gather_files_to_download(repository): {"path": "test/path/file.file", "type": "blob"}, "test/test", "main" ) ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "test/path/file.file" in files @@ -28,7 +26,7 @@ def test_gather_plugin_files_from_root(repository_plugin): ), ] repository_plugin.update_filenames() - files = [x.path for x in gather_files_to_download(repository_plugin)] + files = [x.path for x in repository_plugin.gather_files_to_download()] assert "test.js" in files assert "dir" not in files assert "aaaa.js" in files @@ -54,7 +52,7 @@ def test_gather_plugin_files_from_dist(repository_plugin): {"path": "dist/subdir/file.file", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "test.js" not in files assert "dist/image.png" in files assert "dist/subdir/file.file" in files @@ -75,7 +73,7 @@ def test_gather_plugin_multiple_plugin_files_from_dist(repository_plugin): {"path": "dist/something_other.js", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "test.js" not in files assert "dist/test.js" in files assert "dist/something_other.js" in files @@ -87,7 +85,7 @@ def test_gather_plugin_files_from_release(repository_plugin): repository.data.releases = True release = AIOGitHubAPIRepositoryRelease({"tag_name": "3", "assets": [{"name": "test.js"}]}) repository.releases.objects = [release] - files = [x.name for x in gather_files_to_download(repository)] + files = [x.name for x in repository.gather_files_to_download()] assert "test.js" in files @@ -100,7 +98,7 @@ def test_gather_plugin_files_from_release_multiple(repository_plugin): {"tag_name": "3", "assets": [{"name": "test.js"}, {"name": "test.png"}]} ) ] - files = [x.name for x in gather_files_to_download(repository)] + files = [x.name for x in repository.gather_files_to_download()] assert "test.js" in files assert "test.png" in files @@ -113,7 +111,7 @@ def test_gather_zip_release(repository_plugin): repository.releases.objects = [ AIOGitHubAPIRepositoryRelease({"tag_name": "3", "assets": [{"name": "test.zip"}]}) ] - files = [x.name for x in gather_files_to_download(repository)] + files = [x.name for x in repository.gather_files_to_download()] assert "test.zip" in files @@ -132,7 +130,7 @@ def test_single_file_repo(repository): {"path": "readme.md", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "readme.md" not in files assert "test.yaml" not in files assert "test.file" in files @@ -152,7 +150,7 @@ def test_gather_content_in_root_theme(repository_theme): {"path": "test2.yaml", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "test2.yaml" not in files assert "test.yaml" in files @@ -171,7 +169,7 @@ def test_gather_netdaemon_files_base(repository_netdaemon): {"path": ".github/file.file", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert ".github/file.file" not in files assert "test.cs" not in files assert "apps/test/test.cs" in files @@ -189,7 +187,7 @@ def test_gather_appdaemon_files_base(repository_appdaemon): {"path": ".github/file.file", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert ".github/file.file" not in files assert "test.py" not in files assert "apps/test/test.py" in files @@ -216,7 +214,7 @@ def test_gather_appdaemon_files_with_subdir(repository_appdaemon): {"path": ".github/file.file", "type": "blob"}, "test/test", "main" ), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert ".github/file.file" not in files assert "test.py" not in files assert "apps/test/test.py" in files @@ -236,7 +234,7 @@ def test_gather_plugin_multiple_files_in_root(repository_plugin): AIOGitHubAPIRepositoryTreeContent({"path": "dep3.js", "type": "blob"}, "test/test", "main"), AIOGitHubAPIRepositoryTreeContent({"path": "info.md", "type": "blob"}, "test/test", "main"), ] - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "test.js" in files assert "dep1.js" in files assert "dep2.js" in files @@ -253,6 +251,6 @@ def test_gather_plugin_different_card_name(repository_plugin): AIOGitHubAPIRepositoryTreeContent({"path": "info.md", "type": "blob"}, "test/test", "main"), ] repository_plugin.update_filenames() - files = [x.path for x in gather_files_to_download(repository)] + files = [x.path for x in repository.gather_files_to_download()] assert "card.js" in files assert "info.md" not in files diff --git a/tests/helpers/download/test_should_try_releases.py b/tests/helpers/download/test_should_try_releases.py index d1b402c803b..b7b935278a4 100644 --- a/tests/helpers/download/test_should_try_releases.py +++ b/tests/helpers/download/test_should_try_releases.py @@ -1,42 +1,41 @@ """Helpers: Download: should_try_releases.""" # pylint: disable=missing-docstring -from custom_components.hacs.utils.download import should_try_releases def test_base(repository): repository.ref = "dummy" repository.data.category = "plugin" repository.data.releases = True - assert should_try_releases(repository) + assert repository.should_try_releases def test_ref_is_default(repository): repository.ref = "main" repository.data.category = "plugin" repository.data.releases = True - assert not should_try_releases(repository) + assert not repository.should_try_releases def test_category_is_wrong(repository): repository.ref = "dummy" repository.data.category = "integration" repository.data.releases = True - assert not should_try_releases(repository) + assert not repository.should_try_releases def test_no_releases(repository): repository.ref = "dummy" repository.data.category = "plugin" repository.data.releases = False - assert not should_try_releases(repository) + assert not repository.should_try_releases def test_zip_release(repository): repository.data.releases = False repository.data.zip_release = True repository.data.filename = "test.zip" - assert should_try_releases(repository) + assert repository.should_try_releases # Select a branch repository.ref = "main" - assert not should_try_releases(repository) + assert not repository.should_try_releases diff --git a/tests/helpers/functions/test_logger.py b/tests/helpers/functions/test_logger.py index 3af45cf5800..a512130d68b 100644 --- a/tests/helpers/functions/test_logger.py +++ b/tests/helpers/functions/test_logger.py @@ -1,3 +1,4 @@ +# pylint: disable=missing-module-docstring, missing-function-docstring from logging import Logger from custom_components.hacs.utils.logger import get_hacs_logger