From 2706cd54426001cd6b9e4fa54b21b35947283f54 Mon Sep 17 00:00:00 2001 From: ralbertazzi Date: Wed, 19 Apr 2023 21:42:13 +0200 Subject: [PATCH] feat: use ArtifactCache in get_package_from_url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com> --- src/poetry/console/commands/debug/resolve.py | 2 +- src/poetry/console/commands/init.py | 6 +++- src/poetry/console/commands/show.py | 2 +- src/poetry/factory.py | 10 +++--- src/poetry/installation/installer.py | 3 +- src/poetry/packages/direct_origin.py | 33 +++++++++++++------- src/poetry/poetry.py | 2 +- src/poetry/puzzle/provider.py | 9 +++--- src/poetry/repositories/repository_pool.py | 12 +++++++ src/poetry/utils/dependency_specification.py | 18 ++++++++--- tests/conftest.py | 6 ++++ tests/installation/test_chef.py | 7 +---- tests/installation/test_executor.py | 5 --- tests/utils/test_dependency_specification.py | 10 ++++-- 14 files changed, 82 insertions(+), 43 deletions(-) diff --git a/src/poetry/console/commands/debug/resolve.py b/src/poetry/console/commands/debug/resolve.py index 05cf4b15736..ec168f6adf8 100644 --- a/src/poetry/console/commands/debug/resolve.py +++ b/src/poetry/console/commands/debug/resolve.py @@ -113,7 +113,7 @@ def handle(self) -> int: if self.option("install"): env = EnvManager(self.poetry).get() - pool = RepositoryPool() + pool = RepositoryPool(config=self.poetry.config) locked_repository = Repository("poetry-locked") for op in ops: locked_repository.add_package(op.package) diff --git a/src/poetry/console/commands/init.py b/src/poetry/console/commands/init.py index a87ac9a0fd1..54c0af92bd7 100644 --- a/src/poetry/console/commands/init.py +++ b/src/poetry/console/commands/init.py @@ -434,11 +434,15 @@ def _parse_requirements(self, requirements: list[str]) -> list[dict[str, Any]]: try: cwd = self.poetry.file.parent + artifact_cache = self.poetry.pool.artifact_cache except (PyProjectException, RuntimeError): cwd = Path.cwd() + artifact_cache = self._get_pool().artifact_cache parser = RequirementsParser( - self.env if isinstance(self, EnvCommand) else None, cwd + artifact_cache=artifact_cache, + env=self.env if isinstance(self, EnvCommand) else None, + cwd=cwd, ) return [parser.parse(requirement) for requirement in requirements] diff --git a/src/poetry/console/commands/show.py b/src/poetry/console/commands/show.py index 1b8b03e718b..aa8dafc543f 100644 --- a/src/poetry/console/commands/show.py +++ b/src/poetry/console/commands/show.py @@ -213,7 +213,7 @@ def _display_packages_information( from poetry.utils.helpers import get_package_version_display_string locked_packages = locked_repository.packages - pool = RepositoryPool(ignore_repository_names=True) + pool = RepositoryPool(ignore_repository_names=True, config=self.poetry.config) pool.add_repository(locked_repository) solver = Solver( root, diff --git a/src/poetry/factory.py b/src/poetry/factory.py index a9a488d6bf7..413814b3ddc 100644 --- a/src/poetry/factory.py +++ b/src/poetry/factory.py @@ -118,7 +118,7 @@ def get_package(cls, name: str, version: str) -> ProjectPackage: @classmethod def create_pool( cls, - auth_config: Config, + config: Config, sources: Iterable[dict[str, Any]] = (), io: IO | None = None, disable_cache: bool = False, @@ -132,11 +132,11 @@ def create_pool( if disable_cache: logger.debug("Disabling source caches") - pool = RepositoryPool() + pool = RepositoryPool(config=config) for source in sources: repository = cls.create_package_source( - source, auth_config, disable_cache=disable_cache + source, config, disable_cache=disable_cache ) priority = Priority[source.get("priority", Priority.PRIMARY.name).upper()] if "default" in source or "secondary" in source: @@ -184,7 +184,7 @@ def create_pool( @classmethod def create_package_source( - cls, source: dict[str, str], auth_config: Config, disable_cache: bool = False + cls, source: dict[str, str], config: Config, disable_cache: bool = False ) -> LegacyRepository: from poetry.repositories.legacy_repository import LegacyRepository from poetry.repositories.single_page_repository import SinglePageRepository @@ -206,7 +206,7 @@ def create_package_source( return repository_class( name, url, - config=auth_config, + config=config, disable_cache=disable_cache, ) diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index c049f0da500..3b35e3253b3 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -52,6 +52,7 @@ def __init__( self._package = package self._locker = locker self._pool = pool + self._config = config self._dry_run = False self._requires_synchronization = False @@ -314,7 +315,7 @@ def _do_install(self) -> int: ) # We resolve again by only using the lock file - pool = RepositoryPool(ignore_repository_names=True) + pool = RepositoryPool(ignore_repository_names=True, config=self._config) # Making a new repo containing the packages # newly resolved and the ones from the current lock file diff --git a/src/poetry/packages/direct_origin.py b/src/poetry/packages/direct_origin.py index ce785ea5e12..cdc16d55c49 100644 --- a/src/poetry/packages/direct_origin.py +++ b/src/poetry/packages/direct_origin.py @@ -2,12 +2,13 @@ import functools import os -import tempfile import urllib.parse from pathlib import Path from typing import TYPE_CHECKING +from poetry.core.packages.utils.link import Link + from poetry.inspection.info import PackageInfo from poetry.inspection.info import PackageInfoError from poetry.utils.helpers import download_file @@ -18,6 +19,8 @@ if TYPE_CHECKING: from poetry.core.packages.package import Package + from poetry.utils.cache import ArtifactCache + @functools.lru_cache(maxsize=None) def _get_package_from_git( @@ -53,6 +56,9 @@ def _get_package_from_git( class DirectOrigin: + def __init__(self, artifact_cache: ArtifactCache) -> None: + self._artifact_cache = artifact_cache + @classmethod def get_package_from_file(cls, file_path: Path) -> Package: try: @@ -70,17 +76,22 @@ def get_package_from_file(cls, file_path: Path) -> Package: def get_package_from_directory(cls, directory: Path) -> Package: return PackageInfo.from_directory(path=directory).to_package(root_dir=directory) - @classmethod - def get_package_from_url(cls, url: str) -> Package: + def get_package_from_url(self, url: str) -> Package: file_name = os.path.basename(urllib.parse.urlparse(url).path) - with tempfile.TemporaryDirectory() as temp_dir: - dest = Path(temp_dir) / file_name - download_file(url, dest) - package = cls.get_package_from_file(dest) - - package.files = [ - {"file": file_name, "hash": "sha256:" + get_file_hash(dest)} - ] + link = Link(url) + artifact = self._artifact_cache.get_cached_archive_for_link(link, strict=True) + + if not artifact: + artifact = ( + self._artifact_cache.get_cache_directory_for_link(link) / file_name + ) + artifact.parent.mkdir(parents=True, exist_ok=True) + download_file(url, artifact) + + package = self.get_package_from_file(artifact) + package.files = [ + {"file": file_name, "hash": "sha256:" + get_file_hash(artifact)} + ] package._source_type = "url" package._source_url = url diff --git a/src/poetry/poetry.py b/src/poetry/poetry.py index 3cb227eaaaa..57c05b3e540 100644 --- a/src/poetry/poetry.py +++ b/src/poetry/poetry.py @@ -49,7 +49,7 @@ def __init__( self._locker = locker self._config = config - self._pool = RepositoryPool() + self._pool = RepositoryPool(config=config) self._plugin_manager: PluginManager | None = None self._disable_cache = disable_cache diff --git a/src/poetry/puzzle/provider.py b/src/poetry/puzzle/provider.py index 14e9d13afb1..e996849d43a 100644 --- a/src/poetry/puzzle/provider.py +++ b/src/poetry/puzzle/provider.py @@ -107,6 +107,7 @@ def __init__( ) -> None: self._package = package self._pool = pool + self._direct_origin = DirectOrigin(self._pool.artifact_cache) self._io = io self._env: Env | None = None self._python_constraint = package.python_constraint @@ -311,7 +312,7 @@ def _search_for_vcs(self, dependency: VCSDependency) -> Package: Basically, we clone the repository in a temporary directory and get the information we need by checking out the specified reference. """ - package = DirectOrigin.get_package_from_vcs( + package = self._direct_origin.get_package_from_vcs( dependency.vcs, dependency.source, branch=dependency.branch, @@ -330,7 +331,7 @@ def _search_for_vcs(self, dependency: VCSDependency) -> Package: def _search_for_file(self, dependency: FileDependency) -> Package: dependency.validate(raise_error=True) - package = DirectOrigin.get_package_from_file(dependency.full_path) + package = self._direct_origin.get_package_from_file(dependency.full_path) self.validate_package_for_dependency(dependency=dependency, package=package) @@ -348,7 +349,7 @@ def _search_for_file(self, dependency: FileDependency) -> Package: def _search_for_directory(self, dependency: DirectoryDependency) -> Package: dependency.validate(raise_error=True) - package = DirectOrigin.get_package_from_directory(dependency.full_path) + package = self._direct_origin.get_package_from_directory(dependency.full_path) self.validate_package_for_dependency(dependency=dependency, package=package) @@ -360,7 +361,7 @@ def _search_for_directory(self, dependency: DirectoryDependency) -> Package: return package def _search_for_url(self, dependency: URLDependency) -> Package: - package = DirectOrigin.get_package_from_url(dependency.url) + package = self._direct_origin.get_package_from_url(dependency.url) self.validate_package_for_dependency(dependency=dependency, package=package) diff --git a/src/poetry/repositories/repository_pool.py b/src/poetry/repositories/repository_pool.py index 304f7e9ed33..1fb59ae3ec7 100644 --- a/src/poetry/repositories/repository_pool.py +++ b/src/poetry/repositories/repository_pool.py @@ -8,8 +8,10 @@ from enum import IntEnum from typing import TYPE_CHECKING +from poetry.config.config import Config from poetry.repositories.abstract_repository import AbstractRepository from poetry.repositories.exceptions import PackageNotFound +from poetry.utils.cache import ArtifactCache if TYPE_CHECKING: @@ -40,6 +42,8 @@ def __init__( self, repositories: list[Repository] | None = None, ignore_repository_names: bool = False, + *, + config: Config | None = None, ) -> None: super().__init__("poetry-repository-pool") self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict() @@ -50,6 +54,10 @@ def __init__( for repository in repositories: self.add_repository(repository) + self._artifact_cache = ArtifactCache( + cache_dir=(config or Config.create()).artifacts_cache_directory + ) + @property def repositories(self) -> list[Repository]: """ @@ -77,6 +85,10 @@ def _sorted_repositories(self) -> list[PrioritizedRepository]: self._repositories.values(), key=lambda prio_repo: prio_repo.priority ) + @property + def artifact_cache(self) -> ArtifactCache: + return self._artifact_cache + def has_default(self) -> bool: return self._contains_priority(Priority.DEFAULT) diff --git a/src/poetry/utils/dependency_specification.py b/src/poetry/utils/dependency_specification.py index 9bfb8dcd2d6..b26cc895b47 100644 --- a/src/poetry/utils/dependency_specification.py +++ b/src/poetry/utils/dependency_specification.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from poetry.core.packages.vcs_dependency import VCSDependency + from poetry.utils.cache import ArtifactCache from poetry.utils.env import Env @@ -57,7 +58,14 @@ def dependency_to_specification( class RequirementsParser: - def __init__(self, env: Env | None = None, cwd: Path | None = None) -> None: + def __init__( + self, + *, + artifact_cache: ArtifactCache, + env: Env | None = None, + cwd: Path | None = None, + ) -> None: + self._direct_origin = DirectOrigin(artifact_cache) self._env = env self._cwd = cwd or Path.cwd() @@ -120,7 +128,7 @@ def _parse_git_url(self, requirement: str) -> DependencySpec | None: pair["subdirectory"] = parsed.subdirectory source_root = self._env.path.joinpath("src") if self._env else None - package = DirectOrigin.get_package_from_vcs( + package = self._direct_origin.get_package_from_vcs( "git", url=url.url, rev=pair.get("rev"), @@ -139,7 +147,7 @@ def _parse_url(self, requirement: str) -> DependencySpec | None: return self._parse_git_url(requirement) if url_parsed.scheme in ["http", "https"]: - package = DirectOrigin.get_package_from_url(requirement) + package = self._direct_origin.get_package_from_url(requirement) assert package.source_url is not None return {"name": package.name, "url": package.source_url} @@ -158,9 +166,9 @@ def _parse_path(self, requirement: str) -> DependencySpec | None: path = self._cwd.joinpath(requirement) if path.is_file(): - package = DirectOrigin.get_package_from_file(path.resolve()) + package = self._direct_origin.get_package_from_file(path.resolve()) else: - package = DirectOrigin.get_package_from_directory(path.resolve()) + package = self._direct_origin.get_package_from_directory(path.resolve()) return { "name": package.name, diff --git a/tests/conftest.py b/tests/conftest.py index 76c11a685f2..eeef39d130a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,7 @@ from poetry.layouts import layout from poetry.repositories import Repository from poetry.repositories import RepositoryPool +from poetry.utils.cache import ArtifactCache from poetry.utils.env import EnvManager from poetry.utils.env import SystemEnv from poetry.utils.env import VirtualEnv @@ -216,6 +217,11 @@ def config( return c +@pytest.fixture +def artifact_cache(config: Config) -> ArtifactCache: + return ArtifactCache(cache_dir=config.artifacts_cache_directory) + + @pytest.fixture() def config_dir(tmp_path: Path) -> Path: path = tmp_path / "config" diff --git a/tests/installation/test_chef.py b/tests/installation/test_chef.py index ca00ba01e73..d134b90f9c0 100644 --- a/tests/installation/test_chef.py +++ b/tests/installation/test_chef.py @@ -14,7 +14,6 @@ from poetry.factory import Factory from poetry.installation.chef import Chef from poetry.repositories import RepositoryPool -from poetry.utils.cache import ArtifactCache from poetry.utils.env import EnvManager from tests.repositories.test_pypi_repository import MockRepository @@ -22,6 +21,7 @@ if TYPE_CHECKING: from pytest_mock import MockerFixture + from poetry.utils.cache import ArtifactCache from tests.conftest import Config from tests.types import FixtureDirGetter @@ -40,11 +40,6 @@ def setup(mocker: MockerFixture, pool: RepositoryPool) -> None: mocker.patch.object(Factory, "create_pool", return_value=pool) -@pytest.fixture -def artifact_cache(config: Config) -> ArtifactCache: - return ArtifactCache(cache_dir=config.artifacts_cache_directory) - - def test_prepare_sdist( config: Config, config_cache_dir: Path, diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 18d5cd1b60c..4e539dac1c1 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -133,11 +133,6 @@ def pool() -> RepositoryPool: return pool -@pytest.fixture -def artifact_cache(config: Config) -> ArtifactCache: - return ArtifactCache(cache_dir=config.artifacts_cache_directory) - - @pytest.fixture def mock_file_downloads( http: type[httpretty.httpretty], fixture_dir: FixtureDirGetter diff --git a/tests/utils/test_dependency_specification.py b/tests/utils/test_dependency_specification.py index 89cd2c600a1..a782a55924e 100644 --- a/tests/utils/test_dependency_specification.py +++ b/tests/utils/test_dependency_specification.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from pytest_mock import MockerFixture + from poetry.utils.cache import ArtifactCache from poetry.utils.dependency_specification import DependencySpec @@ -104,7 +105,10 @@ ], ) def test_parse_dependency_specification( - requirement: str, specification: DependencySpec, mocker: MockerFixture + requirement: str, + specification: DependencySpec, + mocker: MockerFixture, + artifact_cache: ArtifactCache, ) -> None: original = Path.exists @@ -116,5 +120,7 @@ def _mock(self: Path) -> bool: mocker.patch("pathlib.Path.exists", _mock) assert not DeepDiff( - RequirementsParser().parse(requirement), specification, ignore_order=True + RequirementsParser(artifact_cache=artifact_cache).parse(requirement), + specification, + ignore_order=True, )