From 63423b2c49085475826128e86f87c203e83d57e6 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 31 Aug 2023 15:39:47 +0800 Subject: [PATCH] feat: Consider packages installed if the venv includes them from the system-site (#2219) --- news/2216.feature.md | 1 + src/pdm/environments/python.py | 10 +++++++++ src/pdm/installers/synchronizers.py | 20 +++++++++-------- src/pdm/models/venv.py | 33 ++++++++++++++++++++++++++++- src/pdm/models/working_set.py | 22 ++++++++++++++----- src/pdm/pytest.py | 3 +++ 6 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 news/2216.feature.md diff --git a/news/2216.feature.md b/news/2216.feature.md new file mode 100644 index 0000000000..817d15db55 --- /dev/null +++ b/news/2216.feature.md @@ -0,0 +1 @@ +Consider packages as installed if the venv includes them from the system-site-packages. diff --git a/src/pdm/environments/python.py b/src/pdm/environments/python.py index 7297b7b001..32d02c5d58 100644 --- a/src/pdm/environments/python.py +++ b/src/pdm/environments/python.py @@ -5,6 +5,7 @@ from pdm.environments.base import BaseEnvironment from pdm.models.in_process import get_sys_config_paths +from pdm.models.working_set import WorkingSet if TYPE_CHECKING: from pdm.project import Project @@ -40,3 +41,12 @@ def process_env(self) -> dict[str, str]: if venv is not None and self.prefix is None: env.update(venv.env_vars()) return env + + def get_working_set(self) -> WorkingSet: + scheme = self.get_paths() + paths = [scheme["platlib"], scheme["purelib"]] + venv = self.interpreter.get_venv() + shared_paths = [] + if venv is not None and venv.include_system_site_packages: + shared_paths.extend(venv.base_paths) + return WorkingSet(paths, shared_paths=list(dict.fromkeys(shared_paths))) diff --git a/src/pdm/installers/synchronizers.py b/src/pdm/installers/synchronizers.py index 4ca1697041..6966b28edb 100644 --- a/src/pdm/installers/synchronizers.py +++ b/src/pdm/installers/synchronizers.py @@ -215,6 +215,7 @@ def compare_with_working_set(self) -> tuple[list[str], list[str], list[str]]: candidates = self.candidates.copy() to_update: set[str] = set() to_remove: set[str] = set() + to_add: set[str] = set() locked_repository = self.environment.project.locked_repository all_candidate_keys = list(locked_repository.all_candidates) @@ -224,23 +225,24 @@ def compare_with_working_set(self) -> tuple[list[str], list[str], list[str]]: if key in candidates: can = candidates.pop(key) if self._should_update(dist, can): - to_update.add(key) + if working_set.is_owned(key): + to_update.add(key) + else: + to_add.add(key) elif ( - self.only_keep or self.clean and key not in all_candidate_keys - ) and key not in self.SEQUENTIAL_PACKAGES: + (self.only_keep or self.clean and key not in all_candidate_keys) + and key not in self.SEQUENTIAL_PACKAGES + and working_set.is_owned(key) + ): # Remove package only if it is not required by any group # Packages for packaging will never be removed to_remove.add(key) - to_add = { + to_add.update( strip_extras(name)[0] for name, _ in candidates.items() if name != self.self_key and strip_extras(name)[0] not in working_set - } - return ( - sorted(to_add), - sorted(to_update), - sorted(to_remove), ) + return (sorted(to_add), sorted(to_update), sorted(to_remove)) def synchronize(self) -> None: """Synchronize the working set with pinned candidates.""" diff --git a/src/pdm/models/venv.py b/src/pdm/models/venv.py index 749e8c670e..623157e122 100644 --- a/src/pdm/models/venv.py +++ b/src/pdm/models/venv.py @@ -4,7 +4,9 @@ import sys from pathlib import Path -from pdm.utils import get_venv_like_prefix +from pdm.compat import cached_property +from pdm.models.in_process import get_sys_config_paths +from pdm.utils import find_python_in_path, get_venv_like_prefix IS_WIN = sys.platform == "win32" BIN_DIR = "Scripts" if IS_WIN else "bin" @@ -46,3 +48,32 @@ def from_interpreter(cls, interpreter: Path) -> VirtualEnv | None: def env_vars(self) -> dict[str, str]: key = "CONDA_PREFIX" if self.is_conda else "VIRTUAL_ENV" return {key: str(self.root)} + + @cached_property + def venv_config(self) -> dict[str, str]: + venv_cfg = self.root / "pyvenv.cfg" + if not venv_cfg.exists(): + return {} + parsed: dict[str, str] = {} + with venv_cfg.open() as fp: + for line in fp: + if "=" in line: + k, v = line.split("=", 1) + k = k.strip().lower() + v = v.strip() + if k == "include-system-site-packages": + v = v.lower() + parsed[k] = v + return parsed + + @property + def include_system_site_packages(self) -> bool: + return self.venv_config.get("include-system-site-packages") == "true" + + @cached_property + def base_paths(self) -> list[str]: + home = Path(self.venv_config["home"]) + base_executable = find_python_in_path(home) or find_python_in_path(home.parent) + assert base_executable is not None + paths = get_sys_config_paths(str(base_executable)) + return [paths["purelib"], paths["platlib"]] diff --git a/src/pdm/models/working_set.py b/src/pdm/models/working_set.py index bcce0001e7..4d535613f8 100644 --- a/src/pdm/models/working_set.py +++ b/src/pdm/models/working_set.py @@ -2,6 +2,7 @@ import itertools import sys +from collections import ChainMap from pathlib import Path from typing import Iterable, Iterator, Mapping @@ -60,23 +61,34 @@ def distributions(path: list[str]) -> Iterable[im.Distribution]: class WorkingSet(Mapping[str, im.Distribution]): """A dictionary of currently installed distributions""" - def __init__(self, paths: list[str] | None = None): + def __init__(self, paths: list[str] | None = None, shared_paths: list[str] | None = None) -> None: if paths is None: paths = sys.path + if shared_paths is None: + shared_paths = [] self._dist_map = { normalize_name(dist.metadata["Name"]): dist for dist in distributions(path=list(dict.fromkeys(paths))) if dist.metadata["Name"] } + self._shared_map = { + normalize_name(dist.metadata["Name"]): dist + for dist in distributions(path=list(dict.fromkeys(shared_paths))) + if dist.metadata["Name"] + } + self._iter_map = ChainMap(self._dist_map, self._shared_map) def __getitem__(self, key: str) -> im.Distribution: - return self._dist_map[key] + return self._iter_map[key] + + def is_owned(self, key: str) -> bool: + return key in self._dist_map def __len__(self) -> int: - return len(self._dist_map) + return len(self._iter_map) def __iter__(self) -> Iterator[str]: - return iter(self._dist_map) + return iter(self._iter_map) def __repr__(self) -> str: - return repr(self._dist_map) + return repr(self._iter_map) diff --git a/src/pdm/pytest.py b/src/pdm/pytest.py index f18f517013..dd5585ace8 100644 --- a/src/pdm/pytest.py +++ b/src/pdm/pytest.py @@ -265,6 +265,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: def add_distribution(self, dist: Distribution) -> None: self._data[dist.name] = dist + def is_owned(self, key: str) -> bool: + return key in self._data + def __getitem__(self, key: str) -> Distribution: return self._data[key]