Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Unify find_packages of different repository types #6165

Merged
merged 2 commits into from
Aug 15, 2022
Merged

Unify find_packages of different repository types #6165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
56716f1
repositories: remove redundant normalizing (replacing "." by "-" is h…
radoering Aug 14, 2022
0a79930
refactor(repositories): unify find_packages of different repository t…
radoering Aug 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 40 additions & 58 deletions src/poetry/repositories/legacy_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@


if TYPE_CHECKING:
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.utils.link import Link
from poetry.core.semver.version_constraint import VersionConstraint

from poetry.config.config import Config

Expand All @@ -33,61 +33,6 @@ def __init__(

super().__init__(name, url.rstrip("/"), config, disable_cache)

def find_packages(self, dependency: Dependency) -> list[Package]:
packages = []
constraint, allow_prereleases = self._get_constraints_from_dependency(
dependency
)

key = dependency.name
if not constraint.is_any():
key = f"{key}:{constraint!s}"

ignored_pre_release_versions = []

if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get_page(f"/{dependency.name.replace('.', '-')}/")
if page is None:
return []

versions = []
for version in page.versions(dependency.name):
if version.is_unstable() and not allow_prereleases:
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_versions.append(version)
continue

if constraint.allows(version):
versions.append(version)

self._cache.store("matches").put(key, versions, 5)

for package_versions in (versions, ignored_pre_release_versions):
for version in package_versions:
package = Package(
dependency.name,
version,
source_type="legacy",
source_reference=self.name,
source_url=self._url,
)

packages.append(package)

self._log(
f"{len(packages)} packages found for {dependency.name} {constraint!s}",
level="debug",
)

if packages or not constraint.is_any():
# we have matching packages, or constraint is not (*)
break

return packages

def package(
self, name: str, version: str, extras: list[str] | None = None
) -> Package:
Expand Down Expand Up @@ -115,14 +60,51 @@ def package(
return package

def find_links_for_package(self, package: Package) -> list[Link]:
page = self._get_page(f"/{package.name.replace('.', '-')}/")
page = self._get_page(f"/{package.name}/")
if page is None:
return []

return list(page.links_for_version(package.name, package.version))

def _find_packages(self, name: str, constraint: VersionConstraint) -> list[Package]:
"""
Find packages on the remote server.
"""
versions: list[Version]

key = name
if not constraint.is_any():
key = f"{key}:{constraint!s}"

if self._cache.store("matches").has(key):
versions = self._cache.store("matches").get(key)
else:
page = self._get_page(f"/{name}/")
if page is None:
self._log(
f"No packages found for {name}",
level="debug",
)
return []

versions = [
version for version in page.versions(name) if constraint.allows(version)
]
self._cache.store("matches").put(key, versions, 5)

return [
Package(
name,
version,
source_type="legacy",
source_reference=self.name,
source_url=self._url,
)
for version in versions
]

def _get_release_info(self, name: str, version: str) -> dict[str, Any]:
page = self._get_page(f"/{canonicalize_name(name).replace('.', '-')}/")
page = self._get_page(f"/{canonicalize_name(name)}/")
if page is None:
raise PackageNotFound(f'No package named "{name}"')

Expand Down
99 changes: 42 additions & 57 deletions src/poetry/repositories/pypi_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from html5lib.html5parser import parse
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link
from poetry.core.semver.version import Version
from poetry.core.version.exceptions import InvalidVersion

from poetry.repositories.exceptions import PackageNotFound
Expand All @@ -26,7 +27,7 @@


if TYPE_CHECKING:
from poetry.core.packages.dependency import Dependency
from poetry.core.semver.version_constraint import VersionConstraint


class PyPiRepository(HTTPRepository):
Expand All @@ -43,62 +44,6 @@ def __init__(
self._base_url = url
self._fallback = fallback

def find_packages(self, dependency: Dependency) -> list[Package]:
"""
Find packages on the remote server.
"""
constraint, allow_prereleases = self._get_constraints_from_dependency(
dependency
)

try:
info = self.get_package_info(dependency.name)
except PackageNotFound:
self._log(
f"No packages found for {dependency.name} {constraint!s}",
level="debug",
)
return []

packages = []
ignored_pre_release_packages = []

for version, release in info["releases"].items():
if not release:
# Bad release
self._log(
f"No release information found for {dependency.name}-{version},"
" skipping",
level="debug",
)
continue

try:
package = Package(info["info"]["name"], version)
except InvalidVersion:
self._log(
f'Unable to parse version "{version}" for the'
f" {dependency.name} package, skipping",
level="debug",
)
continue

if package.is_prerelease() and not allow_prereleases:
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_packages.append(package)
continue

if constraint.allows(package.version):
packages.append(package)

self._log(
f"{len(packages)} packages found for {dependency.name} {constraint!s}",
level="debug",
)

return packages or ignored_pre_release_packages

def search(self, query: str) -> list[Package]:
results = []

Expand Down Expand Up @@ -160,6 +105,46 @@ def get_package_info(self, name: str) -> dict[str, Any]:
)
return package_info

def _find_packages(self, name: str, constraint: VersionConstraint) -> list[Package]:
"""
Find packages on the remote server.
"""
try:
info = self.get_package_info(name)
except PackageNotFound:
self._log(
f"No packages found for {name} {constraint!s}",
level="debug",
)
return []

packages = []

for version_string, release in info["releases"].items():
if not release:
# Bad release
self._log(
f"No release information found for {name}-{version_string},"
" skipping",
level="debug",
)
continue

try:
version = Version.parse(version_string)
except InvalidVersion:
self._log(
f'Unable to parse version "{version_string}" for the'
f" {name} package, skipping",
level="debug",
)
continue

if constraint.allows(version):
packages.append(Package(info["info"]["name"], version))

return packages

def _get_package_info(self, name: str) -> dict[str, Any]:
data = self._get(f"pypi/{name}/json")
if data is None:
Expand Down
45 changes: 25 additions & 20 deletions src/poetry/repositories/repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,30 +35,28 @@ def packages(self) -> list[Package]:

def find_packages(self, dependency: Dependency) -> list[Package]:
packages = []
ignored_pre_release_packages = []
constraint, allow_prereleases = self._get_constraints_from_dependency(
dependency
)
ignored_pre_release_packages = []

for package in self.packages:
if dependency.name == package.name:
if (
package.is_prerelease()
and not allow_prereleases
and not package.source_type
):
# If prereleases are not allowed and the package is a prerelease
# and is a standard package then we skip it
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_packages.append(package)
continue

if constraint.allows(package.version) or (
package.is_prerelease()
and constraint.allows(package.version.next_patch())
):
packages.append(package)
for package in self._find_packages(dependency.name, constraint):
if (
package.is_prerelease()
and not allow_prereleases
and not package.is_direct_origin()
):
if constraint.is_any():
# we need this when all versions of the package are pre-releases
ignored_pre_release_packages.append(package)
continue

packages.append(package)

self._log(
f"{len(packages)} packages found for {dependency.name} {constraint!s}",
level="debug",
)

return packages or ignored_pre_release_packages

Expand Down Expand Up @@ -114,6 +112,13 @@ def _get_constraints_from_dependency(

return constraint, allow_prereleases

def _find_packages(self, name: str, constraint: VersionConstraint) -> list[Package]:
return [
package
for package in self._packages
if package.name == name and constraint.allows(package.version)
]

def _log(self, msg: str, level: str = "info") -> None:
logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
getattr(logger, level)(f"<c1>Source ({self.name}):</c1> {msg}")
Expand Down