add initial support for single page repositories

This change allows users to make use of find-links style single page sources that do not present a PEP 503 simple API repository.
python-poetry · May 7, 2022 · 183cc08 · 183cc08
1 parent f645ab9
commit 183cc08
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 1 deletion.
diff --git a/src/poetry/factory.py b/src/poetry/factory.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import re
 
 from typing import TYPE_CHECKING
 from typing import Any
@@ -184,6 +185,7 @@ def create_legacy_repository(
  cls, source: dict[str, str], auth_config: Config, disable_cache: bool = False
  ) -> LegacyRepository:
  from poetry.repositories.legacy_repository import LegacyRepository
+ from poetry.repositories.single_page_repository import SinglePageRepository
  from poetry.utils.helpers import get_cert
  from poetry.utils.helpers import get_client_cert
 
@@ -196,7 +198,12 @@ def create_legacy_repository(
  name = source["name"]
  url = source["url"]
 
- return LegacyRepository(
+ repository_class = LegacyRepository
+
+ if re.match(r".*\.(htm|html)$", url):
+ repository_class = SinglePageRepository
+
+ return repository_class(
  name,
  url,
  config=auth_config,

diff --git a/src/poetry/repositories/single_page_repository.py b/src/poetry/repositories/single_page_repository.py
@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from poetry.repositories.legacy_repository import LegacyRepository
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
+
+
+class SinglePageRepository(LegacyRepository):
+ def _get_page(self, endpoint: str = None) -> SimpleRepositoryPage | None:
+ """
+ Single page repositories only have one page irrespective of endpoint.
+ """
+ response = self._get_response("")
+ if not response:
+ return None
+ return SimpleRepositoryPage(response.url, response.text)
diff --git a/tests/repositories/fixtures/single-page/jax_releases.html b/tests/repositories/fixtures/single-page/jax_releases.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"></head>
+<body>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp310-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp310-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp37-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp37-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp38-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp38-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp39-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp39-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.0.tar.gz">jax/jax-0.3.0.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.2.tar.gz">jax/jax-0.3.2.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.5.tar.gz">jax/jax-0.3.5.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.6.tar.gz">jax/jax-0.3.6.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.7.tar.gz">jax/jax-0.3.7.tar.gz</a><br>
+</body>
+</html>
diff --git a/tests/repositories/test_single_page_repository.py b/tests/repositories/test_single_page_repository.py
@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import re
+
+from pathlib import Path
+
+from poetry.core.packages.dependency import Dependency
+
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
+from poetry.repositories.single_page_repository import SinglePageRepository
+
+
+class MockSinglePageRepository(SinglePageRepository):
+
+ FIXTURES = Path(__file__).parent / "fixtures" / "single-page"
+
+ def __init__(self, page: str) -> None:
+ super().__init__(
+ "single-page",
+ url=f"http://single-page.foo.bar/{page}.html",
+ disable_cache=True,
+ )
+
+ def _get_page(self, endpoint: str = None) -> SimpleRepositoryPage | None:
+ fixture = self.FIXTURES / self.url.rsplit("/", 1)[-1]
+ if not fixture.exists():
+ return
+
+ with fixture.open(encoding="utf-8") as f:
+ return SimpleRepositoryPage(self._url, f.read())
+
+ def _download(self, url: str, dest: Path) -> None:
+ raise RuntimeError("Tests are not configured for downloads")
+
+
+def test_single_page_repository_get_page():
+ repo = MockSinglePageRepository("jax_releases")
+
+ page = repo._get_page("/ignored")
+ links = list(page.links)
+
+ assert len(links) == 21
+
+ for link in links:
+ assert re.match(r"^(jax|jaxlib)-0\.3\.\d.*\.(whl|tar\.gz)$", link.filename)
+ assert link.netloc == "storage.googleapis.com"
+ assert link.path.startswith("/jax-releases/")
+
+
+def test_single_page_repository_find_packages():
+ repo = MockSinglePageRepository("jax_releases")
+
+ dep = Dependency("jaxlib", "0.3.7")
+
+ packages = repo.find_packages(dep)
+
+ assert len(packages) == 1
+
+ package = packages[0]
+ assert package.name == dep.name
+ assert package.to_dependency().to_pep_508() == dep.to_pep_508()