Improve detection of installed packages in an environment

poetry/repositories/installed_repository.py

@@ -1,6 +1,10 @@
+import itertools
+
 from typing import Set
+from typing import Union
 
 from poetry.core.packages import Package
+from poetry.core.utils.helpers import module_name
 from poetry.utils._compat import Path
 from poetry.utils._compat import metadata
 from poetry.utils.env import Env
@@ -11,11 +15,17 @@
 _VENDORS = Path(__file__).parent.parent.joinpath("_vendor")
 
 
+try:
+ FileNotFoundError
+except NameError:
+ FileNotFoundError = OSError
+
+
 class InstalledRepository(Repository):
  @classmethod
- def get_package_paths(cls, sitedir, name): # type: (Path, str) -> Set[Path]
+ def get_package_paths(cls, env, name): # type: (Env, str) -> Set[Path]
  """
- Process a .pth file within the site-packages directory, and return any valid
+ Process a .pth file within the site-packages directories, and return any valid
  paths. We skip executable .pth files as there is no reliable means to do this
  without side-effects to current run-time. Mo check is made that the item refers
  to a directory rather than a file, however, in order to maintain backwards
@@ -24,25 +34,72 @@ def get_package_paths(cls, sitedir, name): # type: (Path, str) -> Set[Path]
 
  Reference: https://docs.python.org/3.8/library/site.html
 
- :param sitedir: The site-packages directory to search for .pth file.
+ :param env: The environment to search for the .pth file in.
  :param name: The name of the package to search .pth file for.
  :return: A `Set` of valid `Path` objects.
  """
  paths = set()
 
- pth_file = sitedir.joinpath("{}.pth".format(name))
- if pth_file.exists():
+ # we identify the candidate pth files to check, this is done so to handle cases
+ # where the pth file for foo-bar might have been installed as either foo-bar.pth or
+ # foo_bar.pth (expected) in either pure or platform lib directories.
+ candidates = itertools.product(
+ {env.purelib, env.platlib}, {name, module_name(name)},
+ )
+
+ for lib, module in candidates:
+ pth_file = lib.joinpath(module).with_suffix(".pth")
+ if not pth_file.exists():
+ continue
+
  with pth_file.open() as f:
  for line in f:
  line = line.strip()
  if line and not line.startswith(("#", "import ", "import\t")):
  path = Path(line)
  if not path.is_absolute():
- path = sitedir.joinpath(path)
+ try:
+ path = lib.joinpath(path).resolve()
+ except FileNotFoundError:
+ # this is required to handle pathlib oddity on win32 python==3.5
+ path = lib.joinpath(path)
  paths.add(path)
-
  return paths
 
+ @classmethod
+ def set_package_vcs_properties_from_path(
+ cls, src, package
+ ): # type: (Path, Package) -> None
+ from poetry.core.vcs.git import Git
+
+ git = Git()
+ revision = git.rev_parse("HEAD", src).strip()
+ url = git.remote_url(src)
+
+ package.source_type = "git"
+ package.source_url = url
+ package.source_reference = revision
+
+ @classmethod
+ def set_package_vcs_properties(cls, package, env): # type: (Package, Env) -> None
+ src = env.path / "src" / package.name
+ cls.set_package_vcs_properties_from_path(src, package)
+
+ @classmethod
+ def is_vcs_package(cls, package, env): # type: (Union[Path, Package], Env) -> bool
+ # A VCS dependency should have been installed
+ # in the src directory.
+ src = env.path / "src"
+ if isinstance(package, Package):
+ return src.joinpath(package.name).is_dir()
+
+ try:
+ package.relative_to(env.path / "src")
+ except ValueError:
+ return False
+ else:
+ return True
+
  @classmethod
  def load(cls, env): # type: (Env) -> InstalledRepository
  """
@@ -75,41 +132,26 @@ def load(cls, env): # type: (Env) -> InstalledRepository
 
  repo.add_package(package)
 
- is_standard_package = True
- try:
- path.relative_to(env.site_packages)
- except ValueError:
- is_standard_package = False
+ is_standard_package = env.is_path_relative_to_lib(path)
 
  if is_standard_package:
  if path.name.endswith(".dist-info"):
- paths = cls.get_package_paths(
- sitedir=env.site_packages, name=package.pretty_name
- )
+ paths = cls.get_package_paths(env=env, name=package.pretty_name)
  if paths:
- # TODO: handle multiple source directories?
- package.source_type = "directory"
- package.source_url = paths.pop().as_posix()
-
+ for src in paths:
+ if cls.is_vcs_package(src, env):
+ cls.set_package_vcs_properties(package, env)
+ break
+ else:
+ # TODO: handle multiple source directories?
+ package.source_type = "directory"
+ package.source_url = paths.pop().as_posix()
  continue
 
- src_path = env.path / "src"
-
- # A VCS dependency should have been installed
- # in the src directory. If not, it's a path dependency
- try:
- path.relative_to(src_path)
-
- from poetry.core.vcs.git import Git
-
- git = Git()
- revision = git.rev_parse("HEAD", src_path / package.name).strip()
- url = git.remote_url(src_path / package.name)
-
- package.source_type = "git"
- package.source_url = url
- package.source_reference = revision
- except ValueError:
+ if cls.is_vcs_package(path, env):
+ cls.set_package_vcs_properties(package, env)
+ else:
+ # If not, it's a path dependency
  package.source_type = "directory"
  package.source_url = str(path.parent)
 

poetry/utils/env.py

@@ -754,6 +754,8 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None
  self._site_packages = None
  self._paths = None
  self._supported_tags = None
+ self._purelib = None
+ self._platlib = None
 
  @property
  def path(self): # type: () -> Path
@@ -810,10 +812,37 @@ def pip_version(self):
  @property
  def site_packages(self): # type: () -> Path
  if self._site_packages is None:
- self._site_packages = Path(self.paths["purelib"])
+ self._site_packages = Path(self.purelib)
 
  return self._site_packages
 
+ @property
+ def purelib(self): # type: () -> Path
+ if self._purelib is None:
+ self._purelib = Path(self.paths["purelib"])
+
+ return self._purelib
+
+ @property
+ def platlib(self): # type: () -> Path
+ if self._platlib is None:
+ if "platlib" in self.paths:
+ self._platlib = Path(self.paths["platlib"])
+ else:
+ self._platlib = self.purelib
+
+ return self._platlib
+
+ def is_path_relative_to_lib(self, path): # type: (Path) -> bool
+ for lib_path in [self.purelib, self.platlib]:
+ try:
+ path.relative_to(lib_path)
+ return True
+ except ValueError:
+ pass
+
+ return False
+
  @property
  def sys_path(self): # type: () -> List[str]
  raise NotImplementedError()

tests/fixtures/complete.toml

@@ -22,7 +22,7 @@ toml = "^0.9"
 # Dependencies with extras
 requests = { version = "^2.13", extras = [ "security" ] }
 # Python specific dependencies with prereleases allowed
-pathlib2 = { version = "^2.2", python = "~2.7", allows-prereleases = true }
+pathlib2 = { version = "^2.2", python = "~2.7", allow-prereleases = true }
 # Git dependencies
 cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" }
 

tests/installation/test_installer.py

@@ -890,7 +890,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
  repo.add_package(package_a)
  repo.add_package(package_b)
 
- package.add_dependency("A", {"version": "*", "allows-prereleases": True})
+ package.add_dependency("A", {"version": "*", "allow-prereleases": True})
  package.add_dependency("B", "^1.1")
 
  installer.update(True)

tests/installation/test_installer_old.py

@@ -861,7 +861,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
  repo.add_package(package_a)
  repo.add_package(package_b)
 
- package.add_dependency("A", {"version": "*", "allows-prereleases": True})
+ package.add_dependency("A", {"version": "*", "allow-prereleases": True})
  package.add_dependency("B", "^1.1")
 
  installer.update(True)

tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: bender
+Version: 2.0.5
+Summary: Python datetimes made easy
+License: MIT
+Keywords: cli,commands
+Author: Leela
+Author-email: leela@planetexpress.com
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/x-rst

tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender.pth

@@ -0,0 +1 @@
+../../../src/bender

tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.1
+Name: lib64
+Version: 2.3.4
+Summary: lib64 description.
+License: MIT
+Keywords: cli,commands
+Author: Foo Bar
+Author-email: foo@bar.com
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/x-rst
+
+lib64
+####

tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: bender
+Version: 2.0.5
+Summary: Python datetimes made easy
+License: MIT
+Keywords: cli,commands
+Author: Leela
+Author-email: leela@planetexpress.com
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/x-rst