add initial support for single page repositories

This change allows users to make use of find-links style single page sources that do not present a PEP 503 simple API repository.
python-poetry · May 7, 2022 · 183cc08 · 183cc08
1 parent f645ab9
commit 183cc08
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 1 deletion.
diff --git a/src/poetry/factory.py b/src/poetry/factory.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import re
 
 from typing import TYPE_CHECKING
 from typing import Any
@@ -184,6 +185,7 @@ def create_legacy_repository(
         cls, source: dict[str, str], auth_config: Config, disable_cache: bool = False
     ) -> LegacyRepository:
         from poetry.repositories.legacy_repository import LegacyRepository
+        from poetry.repositories.single_page_repository import SinglePageRepository
         from poetry.utils.helpers import get_cert
         from poetry.utils.helpers import get_client_cert
 
@@ -196,7 +198,12 @@ def create_legacy_repository(
         name = source["name"]
         url = source["url"]
 
-        return LegacyRepository(
+        repository_class = LegacyRepository
+
+        if re.match(r".*\.(htm|html)$", url):
+            repository_class = SinglePageRepository
+
+        return repository_class(
             name,
             url,
             config=auth_config,

diff --git a/src/poetry/repositories/single_page_repository.py b/src/poetry/repositories/single_page_repository.py
@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from poetry.repositories.legacy_repository import LegacyRepository
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
+
+
+class SinglePageRepository(LegacyRepository):
+    def _get_page(self, endpoint: str = None) -> SimpleRepositoryPage | None:
+        """
+        Single page repositories only have one page irrespective of endpoint.
+        """
+        response = self._get_response("")
+        if not response:
+            return None
+        return SimpleRepositoryPage(response.url, response.text)
diff --git a/tests/repositories/fixtures/single-page/jax_releases.html b/tests/repositories/fixtures/single-page/jax_releases.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"></head>
+<body>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp310-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp310-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp37-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp37-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp38-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp38-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp39-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp39-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.0.tar.gz">jax/jax-0.3.0.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.2.tar.gz">jax/jax-0.3.2.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.5.tar.gz">jax/jax-0.3.5.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.6.tar.gz">jax/jax-0.3.6.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.7.tar.gz">jax/jax-0.3.7.tar.gz</a><br>
+</body>
+</html>
diff --git a/tests/repositories/test_single_page_repository.py b/tests/repositories/test_single_page_repository.py
@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import re
+
+from pathlib import Path
+
+from poetry.core.packages.dependency import Dependency
+
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
+from poetry.repositories.single_page_repository import SinglePageRepository
+
+
+class MockSinglePageRepository(SinglePageRepository):
+
+    FIXTURES = Path(__file__).parent / "fixtures" / "single-page"
+
+    def __init__(self, page: str) -> None:
+        super().__init__(
+            "single-page",
+            url=f"http://single-page.foo.bar/{page}.html",
+            disable_cache=True,
+        )
+
+    def _get_page(self, endpoint: str = None) -> SimpleRepositoryPage | None:
+        fixture = self.FIXTURES / self.url.rsplit("/", 1)[-1]
+        if not fixture.exists():
+            return
+
+        with fixture.open(encoding="utf-8") as f:
+            return SimpleRepositoryPage(self._url, f.read())
+
+    def _download(self, url: str, dest: Path) -> None:
+        raise RuntimeError("Tests are not configured for downloads")
+
+
+def test_single_page_repository_get_page():
+    repo = MockSinglePageRepository("jax_releases")
+
+    page = repo._get_page("/ignored")
+    links = list(page.links)
+
+    assert len(links) == 21
+
+    for link in links:
+        assert re.match(r"^(jax|jaxlib)-0\.3\.\d.*\.(whl|tar\.gz)$", link.filename)
+        assert link.netloc == "storage.googleapis.com"
+        assert link.path.startswith("/jax-releases/")
+
+
+def test_single_page_repository_find_packages():
+    repo = MockSinglePageRepository("jax_releases")
+
+    dep = Dependency("jaxlib", "0.3.7")
+
+    packages = repo.find_packages(dep)
+
+    assert len(packages) == 1
+
+    package = packages[0]
+    assert package.name == dep.name
+    assert package.to_dependency().to_pep_508() == dep.to_pep_508()