feat: Consider packages installed if the venv includes them from the …

…system-site (#2219)

news/2216.feature.md

@@ -0,0 +1 @@
+Consider packages as installed if the venv includes them from the system-site-packages.

src/pdm/environments/python.py

@@ -5,6 +5,7 @@
 
 from pdm.environments.base import BaseEnvironment
 from pdm.models.in_process import get_sys_config_paths
+from pdm.models.working_set import WorkingSet
 
 if TYPE_CHECKING:
  from pdm.project import Project
@@ -40,3 +41,12 @@ def process_env(self) -> dict[str, str]:
  if venv is not None and self.prefix is None:
  env.update(venv.env_vars())
  return env
+
+ def get_working_set(self) -> WorkingSet:
+ scheme = self.get_paths()
+ paths = [scheme["platlib"], scheme["purelib"]]
+ venv = self.interpreter.get_venv()
+ shared_paths = []
+ if venv is not None and venv.include_system_site_packages:
+ shared_paths.extend(venv.base_paths)
+ return WorkingSet(paths, shared_paths=list(dict.fromkeys(shared_paths)))

src/pdm/installers/synchronizers.py

@@ -215,6 +215,7 @@ def compare_with_working_set(self) -> tuple[list[str], list[str], list[str]]:
  candidates = self.candidates.copy()
  to_update: set[str] = set()
  to_remove: set[str] = set()
+ to_add: set[str] = set()
  locked_repository = self.environment.project.locked_repository
  all_candidate_keys = list(locked_repository.all_candidates)
 
@@ -224,23 +225,24 @@ def compare_with_working_set(self) -> tuple[list[str], list[str], list[str]]:
  if key in candidates:
  can = candidates.pop(key)
  if self._should_update(dist, can):
- to_update.add(key)
+ if working_set.is_owned(key):
+ to_update.add(key)
+ else:
+ to_add.add(key)
  elif (
- self.only_keep or self.clean and key not in all_candidate_keys
- ) and key not in self.SEQUENTIAL_PACKAGES:
+ (self.only_keep or self.clean and key not in all_candidate_keys)
+ and key not in self.SEQUENTIAL_PACKAGES
+ and working_set.is_owned(key)
+ ):
  # Remove package only if it is not required by any group
  # Packages for packaging will never be removed
  to_remove.add(key)
- to_add = {
+ to_add.update(
  strip_extras(name)[0]
  for name, _ in candidates.items()
  if name != self.self_key and strip_extras(name)[0] not in working_set
- }
- return (
- sorted(to_add),
- sorted(to_update),
- sorted(to_remove),
  )
+ return (sorted(to_add), sorted(to_update), sorted(to_remove))
 
  def synchronize(self) -> None:
  """Synchronize the working set with pinned candidates."""

src/pdm/models/venv.py

@@ -4,7 +4,9 @@
 import sys
 from pathlib import Path
 
-from pdm.utils import get_venv_like_prefix
+from pdm.compat import cached_property
+from pdm.models.in_process import get_sys_config_paths
+from pdm.utils import find_python_in_path, get_venv_like_prefix
 
 IS_WIN = sys.platform == "win32"
 BIN_DIR = "Scripts" if IS_WIN else "bin"
@@ -46,3 +48,32 @@ def from_interpreter(cls, interpreter: Path) -> VirtualEnv | None:
  def env_vars(self) -> dict[str, str]:
  key = "CONDA_PREFIX" if self.is_conda else "VIRTUAL_ENV"
  return {key: str(self.root)}
+
+ @cached_property
+ def venv_config(self) -> dict[str, str]:
+ venv_cfg = self.root / "pyvenv.cfg"
+ if not venv_cfg.exists():
+ return {}
+ parsed: dict[str, str] = {}
+ with venv_cfg.open() as fp:
+ for line in fp:
+ if "=" in line:
+ k, v = line.split("=", 1)
+ k = k.strip().lower()
+ v = v.strip()
+ if k == "include-system-site-packages":
+ v = v.lower()
+ parsed[k] = v
+ return parsed
+
+ @property
+ def include_system_site_packages(self) -> bool:
+ return self.venv_config.get("include-system-site-packages") == "true"
+
+ @cached_property
+ def base_paths(self) -> list[str]:
+ home = Path(self.venv_config["home"])
+ base_executable = find_python_in_path(home) or find_python_in_path(home.parent)
+ assert base_executable is not None
+ paths = get_sys_config_paths(str(base_executable))
+ return [paths["purelib"], paths["platlib"]]

src/pdm/models/working_set.py

@@ -2,6 +2,7 @@
 
 import itertools
 import sys
+from collections import ChainMap
 from pathlib import Path
 from typing import Iterable, Iterator, Mapping
 
@@ -60,23 +61,34 @@ def distributions(path: list[str]) -> Iterable[im.Distribution]:
 class WorkingSet(Mapping[str, im.Distribution]):
  """A dictionary of currently installed distributions"""
 
- def __init__(self, paths: list[str] | None = None):
+ def __init__(self, paths: list[str] | None = None, shared_paths: list[str] | None = None) -> None:
  if paths is None:
  paths = sys.path
+ if shared_paths is None:
+ shared_paths = []
  self._dist_map = {
  normalize_name(dist.metadata["Name"]): dist
  for dist in distributions(path=list(dict.fromkeys(paths)))
  if dist.metadata["Name"]
  }
+ self._shared_map = {
+ normalize_name(dist.metadata["Name"]): dist
+ for dist in distributions(path=list(dict.fromkeys(shared_paths)))
+ if dist.metadata["Name"]
+ }
+ self._iter_map = ChainMap(self._dist_map, self._shared_map)
 
  def __getitem__(self, key: str) -> im.Distribution:
- return self._dist_map[key]
+ return self._iter_map[key]
+
+ def is_owned(self, key: str) -> bool:
+ return key in self._dist_map
 
  def __len__(self) -> int:
- return len(self._dist_map)
+ return len(self._iter_map)
 
  def __iter__(self) -> Iterator[str]:
- return iter(self._dist_map)
+ return iter(self._iter_map)
 
  def __repr__(self) -> str:
- return repr(self._dist_map)
+ return repr(self._iter_map)

src/pdm/pytest.py

@@ -265,6 +265,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
  def add_distribution(self, dist: Distribution) -> None:
  self._data[dist.name] = dist
 
+ def is_owned(self, key: str) -> bool:
+ return key in self._data
+
  def __getitem__(self, key: str) -> Distribution:
  return self._data[key]