Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve detection of installed packages #1786

Merged
merged 1 commit into from
Jan 5, 2020
Merged

Improve detection of installed packages #1786

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion poetry/installation/pip_installer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,12 @@ def install(self, package, update=False):

self.run(*args)

def update(self, _, target):
def update(self, package, target):
if package.source_type != target.source_type:
# If the source type has changed, we remove the current
# package to avoid perpetual updates in some cases
self.remove(package)

self.install(target, update=True)

def remove(self, package):
Expand Down
85 changes: 46 additions & 39 deletions poetry/repositories/installed_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,45 +15,52 @@ def load(cls, env): # type: (Env) -> InstalledRepository
For now, it uses the pip "freeze" command.
"""
repo = cls()
seen = set()

for distribution in sorted(
metadata.distributions(path=env.sys_path), key=lambda d: str(d._path),
):
name = distribution.metadata["name"]
version = distribution.metadata["version"]
package = Package(name, version, version)
package.description = distribution.metadata.get("summary", "")

repo.add_package(package)

path = Path(str(distribution._path))
is_standard_package = True
try:
path.relative_to(env.site_packages)
except ValueError:
is_standard_package = False

if is_standard_package:
continue

src_path = env.path / "src"

# A VCS dependency should have been installed
# in the src directory. If not, it's a path dependency
try:
path.relative_to(src_path)

from poetry.vcs.git import Git

git = Git()
revision = git.rev_parse("HEAD", src_path / package.name).strip()
url = git.remote_url(src_path / package.name)

package.source_type = "git"
package.source_url = url
package.source_reference = revision
except ValueError:
package.source_type = "directory"
package.source_url = str(path.parent)
for entry in env.sys_path:
for distribution in sorted(
metadata.distributions(path=[entry]), key=lambda d: str(d._path),
):
name = distribution.metadata["name"]
version = distribution.metadata["version"]
package = Package(name, version, version)
package.description = distribution.metadata.get("summary", "")

if package.name in seen:
continue

seen.add(package.name)

repo.add_package(package)

path = Path(str(distribution._path))
is_standard_package = True
try:
path.relative_to(env.site_packages)
except ValueError:
is_standard_package = False

if is_standard_package:
continue

src_path = env.path / "src"

# A VCS dependency should have been installed
# in the src directory. If not, it's a path dependency
try:
path.relative_to(src_path)

from poetry.vcs.git import Git

git = Git()
revision = git.rev_parse("HEAD", src_path / package.name).strip()
url = git.remote_url(src_path / package.name)

package.source_type = "git"
package.source_url = url
package.source_reference = revision
except ValueError:
package.source_type = "directory"
package.source_url = str(path.parent)

return repo
43 changes: 29 additions & 14 deletions poetry/utils/env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,7 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None

self._marker_env = None
self._pip_version = None
self._site_packages = None

@property
def path(self): # type: () -> Path
Expand Down Expand Up @@ -758,20 +759,25 @@ def pip_version(self):

@property
def site_packages(self): # type: () -> Path
# It seems that PyPy3 virtual environments
# have their site-packages directory at the root
if self._path.joinpath("site-packages").exists():
return self._path.joinpath("site-packages")

if self._is_windows:
return self._path / "Lib" / "site-packages"

return (
self._path
/ "lib"
/ "python{}.{}".format(*self.version_info[:2])
/ "site-packages"
)
if self._site_packages is None:
site_packages = []
dist_packages = []
for entry in self.sys_path:
entry = Path(entry)
if entry.name == "site-packages":
site_packages.append(entry)
elif entry.name == "dist-packages":
dist_packages.append(entry)

if not site_packages and not dist_packages:
raise RuntimeError("Unable to find the site-packages directory")

if site_packages:
self._site_packages = site_packages[0]
else:
self._site_packages = dist_packages[0]
Comment on lines +775 to +778
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know where the site_packages property is used, so it isn't obvious to me, why only the first site_package is stored and why dist-packages are never stored when site_packages where found.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The site_packages property is used in the InstalledRepository class (see https://github.com/python-poetry/poetry/blob/master/poetry/repositories/installed_repository.py#L32).

We only need the "site-packages" directory where pip will install the dependencies which is, as far as I know, the site-packages in virtual environments and most OS and dist-packages for Debian-based distributions.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, ok I got it. I wonder if something with sysconfig would be more reliable? But as far as I can see, we must be able to detect if we are inside a virtual environment and of we are on a posix or nt system, to get the correct path (purelib seems to be the one we are interested in), are we?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for pointing me to sysconfig! Do you know if there are some systems where purelib and platlib are different? If there is none we could indeed use purelib and avoid the heuristics we are currently doing.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure. I've just discovered that module as well. There is something in PEP427 about the difference.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the end, it seems that all installed packages will reside in purelib and/or platlib, so to get populate the InstalledRepository we can just check both, I think. I'll test it out and see where it goes.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After testing it out it seems that using sysconfig won't help us. While it works in some cases, for Debian-based systems, for instance, it won't return the directory we need. We could use site.getsitepackages() for non-virtual environments but we would need to find a way to replicate it for virtual environments which would lead to using the same heuristic this PR already uses.

I think we can leave the PR as-is for now since I tested it on Ubuntu, MacOS and Windows without any issue and it fixes the issues I mentioned above.


return self._site_packages

@property
def sys_path(self): # type: () -> List[str]
Expand Down Expand Up @@ -1114,6 +1120,7 @@ def __init__(
os_name="posix",
is_venv=False,
pip_version="19.1",
sys_path=None,
**kwargs
):
super(MockEnv, self).__init__(**kwargs)
Expand All @@ -1124,6 +1131,7 @@ def __init__(
self._os_name = os_name
self._is_venv = is_venv
self._pip_version = Version.parse(pip_version)
self._sys_path = sys_path

@property
def version_info(self): # type: () -> Tuple[int]
Expand All @@ -1145,5 +1153,12 @@ def os(self): # type: () -> str
def pip_version(self):
return self._pip_version

@property
def sys_path(self):
if self._sys_path is None:
return super(MockEnv, self).sys_path

return self._sys_path

def is_venv(self): # type: () -> bool
return self._is_venv
6 changes: 2 additions & 4 deletions tests/masonry/builders/test_editable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@
def test_build_should_delegate_to_pip_for_non_pure_python_packages(tmp_dir, mocker):
move = mocker.patch("shutil.move")
tmp_dir = Path(tmp_dir)
env = MockEnv(path=tmp_dir, pip_version="18.1", execute=False)
env.site_packages.mkdir(parents=True)
env = MockEnv(path=tmp_dir, pip_version="18.1", execute=False, sys_path=[])
module_path = fixtures_dir / "extended"

builder = EditableBuilder(Factory().create_poetry(module_path), env, NullIO())
Expand All @@ -33,8 +32,7 @@ def test_build_should_delegate_to_pip_for_non_pure_python_packages(tmp_dir, mock
def test_build_should_temporarily_remove_the_pyproject_file(tmp_dir, mocker):
move = mocker.patch("shutil.move")
tmp_dir = Path(tmp_dir)
env = MockEnv(path=tmp_dir, pip_version="19.1", execute=False)
env.site_packages.mkdir(parents=True)
env = MockEnv(path=tmp_dir, pip_version="19.1", execute=False, sys_path=[])
module_path = fixtures_dir / "extended"

builder = EditableBuilder(Factory().create_poetry(module_path), env, NullIO())
Expand Down
49 changes: 47 additions & 2 deletions tests/utils/test_env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,20 @@
"""


class MockVirtualEnv(VirtualEnv):
def __init__(self, path, base=None, sys_path=None):
super(MockVirtualEnv, self).__init__(path, base=base)

self._sys_path = sys_path

@property
def sys_path(self):
if self._sys_path is not None:
return self._sys_path

return super(MockVirtualEnv, self).sys_path


@pytest.fixture()
def poetry(config):
poetry = Factory().create_poetry(
Expand Down Expand Up @@ -786,7 +800,7 @@ def test_env_site_packages_should_find_the_site_packages_directory_if_standard(t

site_packages.mkdir(parents=True)

env = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
env = MockVirtualEnv(Path(tmp_dir), Path(tmp_dir), sys_path=[str(site_packages)])

assert site_packages == env.site_packages

Expand All @@ -795,6 +809,37 @@ def test_env_site_packages_should_find_the_site_packages_directory_if_root(tmp_d
site_packages = Path(tmp_dir).joinpath("site-packages")
site_packages.mkdir(parents=True)

env = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
env = MockVirtualEnv(Path(tmp_dir), Path(tmp_dir), sys_path=[str(site_packages)])

assert site_packages == env.site_packages


def test_env_site_packages_should_find_the_dist_packages_directory_if_necessary(
tmp_dir,
):
site_packages = Path(tmp_dir).joinpath("dist-packages")
site_packages.mkdir(parents=True)

env = MockVirtualEnv(Path(tmp_dir), Path(tmp_dir), sys_path=[str(site_packages)])

assert site_packages == env.site_packages


def test_env_site_packages_should_prefer_site_packages_over_dist_packages(tmp_dir):
dist_packages = Path(tmp_dir).joinpath("dist-packages")
dist_packages.mkdir(parents=True)
site_packages = Path(tmp_dir).joinpath("site-packages")
site_packages.mkdir(parents=True)

env = MockVirtualEnv(
Path(tmp_dir), Path(tmp_dir), sys_path=[str(dist_packages), str(site_packages)]
)

assert site_packages == env.site_packages


def test_env_site_packages_should_raise_an_error_if_no_site_packages(tmp_dir):
env = MockVirtualEnv(Path(tmp_dir), Path(tmp_dir), sys_path=[])

with pytest.raises(RuntimeError):
env.site_packages