dynamic listing of packages

aside from `jaxlib`, `jax` and `pocketfft` are now also included
jorenham · Apr 23, 2022 · ce1e02e · ce1e02e
1 parent d45dd37
commit ce1e02e
Show file tree

Hide file tree

Showing 11 changed files with 1,141 additions and 1,065 deletions.
diff --git a/docs/index.html b/docs/index.html
@@ -1,10 +1,13 @@
 <!DOCTYPE html>
 <html lang="en">
-  <head>
+<head>
     <meta name="pypi:repository-version" content="1.0">
     <title>Simple index</title>
-  </head>
-  <body>
-    <a href="jaxlib/">jaxlib</a>
-  </body>
+</head>
+<body>
+
+<a href="jaxlib/">jaxlib</a><br>
+<a href="jax/">jax</a><br>
+<a href="pocketfft/">pocketfft</a><br>
+</body>
 </html>
diff --git a/docs/jax/index.html b/docs/jax/index.html
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta name="pypi:repository-version" content="1.0">
+    <title>Links for jax</title>
+</head>
+<body>
+<h1>Links for jax</h1>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.0.tar.gz">jax/jax-0.3.0.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.2.tar.gz">jax/jax-0.3.2.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.5.tar.gz">jax/jax-0.3.5.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.6.tar.gz">jax/jax-0.3.6.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.7.tar.gz">jax/jax-0.3.7.tar.gz</a><br>
+</body>
+</html>
diff --git a/docs/jaxlib/index.html b/docs/jaxlib/index.html
diff --git a/docs/pocketfft/index.html b/docs/pocketfft/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta name="pypi:repository-version" content="1.0">
+    <title>Links for pocketfft</title>
+</head>
+<body>
+<h1>Links for pocketfft</h1>
+<a href="https://storage.googleapis.com/jax-releases/mirror/pocketfft/pocketfft-53e9dd4d12f986207c96d97c5183f5a72239c76e.tar.gz">mirror/pocketfft/pocketfft-53e9dd4d12f986207c96d97c5183f5a72239c76e.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/mirror/pocketfft/pocketfft-f800d91ba695b6e19ae2687dd60366900b928002.tar.gz">mirror/pocketfft/pocketfft-f800d91ba695b6e19ae2687dd60366900b928002.tar.gz</a><br>
+</body>
+</html>
diff --git a/jax_pep503/build.py b/jax_pep503/build.py
@@ -1,20 +1,32 @@
+import asyncio
 import pathlib
 
 from fastapi.testclient import TestClient
+from mainpy import main
 
-from .main import app
+from jax_pep503.main import app, get_package_links
 
 HTML_DIR = pathlib.Path('./docs')
-URL_PATHS = '/', '/jaxlib/'
 
 
 client = TestClient(app)
 
 
-def build_index(html_dir: pathlib.Path = HTML_DIR):
+def build_index():
+    asyncio.run(_build_index())
+
+
+@main
+async def _build_index(html_dir: pathlib.Path = HTML_DIR):
     html_dir.mkdir(exist_ok=True)
 
-    for url_path in URL_PATHS:
+    url_paths = ['/']
+
+    package_links = await get_package_links()
+    for package in package_links:
+        url_paths.append(f'/{package}/')
+
+    for url_path in url_paths:
         response = client.get(url_path)
         response.raise_for_status()
 
@@ -25,7 +37,3 @@ def build_index(html_dir: pathlib.Path = HTML_DIR):
 
         print(f'{url_path} -> {html_path}')
         html_path.write_bytes(response.content)
-
-
-if __name__ == '__main__':
-    build_index()
diff --git a/jax_pep503/main.py b/jax_pep503/main.py
@@ -1,6 +1,8 @@
+import collections
 from datetime import datetime, timedelta
 import re
-from typing import Final
+from pathlib import Path
+from typing import Final, TypeAlias
 
 from fastapi import FastAPI, Request
 from fastapi.responses import HTMLResponse
@@ -10,62 +12,95 @@
 import yarl
 
 
-VERSION: Final = "1.0"
-JAX_URL: Final = yarl.URL("https://storage.googleapis.com/jax-releases/")
-PATTERN_XML_KEY = re.compile(r'<Key>(.*?)</Key>')
-RESCRAPE_INTERVAL = timedelta(days=1)
+JAX_URL: Final = yarl.URL('https://storage.googleapis.com/jax-releases/')
+PATTERN_XML_KEY: Final = re.compile(r'<Key>(.*?)</Key>')
+RESCRAPE_INTERVAL: Final = timedelta(days=1)
+RELEASE_SUFFIXES: Final = frozenset({'.whl', '.gz'})
+
 
 app: Final = FastAPI()
-templates: Final = Jinja2Templates(directory="templates")
+templates: Final = Jinja2Templates(directory='templates')
+
+
+_HTMLAttrs: TypeAlias = dict[str, str]
+_Links: TypeAlias = dict[str, _HTMLAttrs]
 
 
 _last_scrape = datetime.min
-_jaxlib_links = None
+_package_links = None
+
+
+async def get_package_links() -> _Links:
+    links = {}
+    for package_name in (await get_links()):
+        links[package_name] = {'href': f'{package_name}/'}
+    return links
+
+
+async def get_package_release_links(package_name: str) -> _Links:
+    return (await get_links())[package_name]
 
 
-async def get_jaxlib_links() -> dict[str, str]:
-    global _last_scrape, _jaxlib_links
+async def get_links() -> dict[str, _Links]:
+    global _last_scrape, _package_links
 
-    if not _jaxlib_links or datetime.now() - _last_scrape > RESCRAPE_INTERVAL:
-        _jaxlib_links = await _get_jaxlib_links()
+    if not _package_links or datetime.now() - _last_scrape > RESCRAPE_INTERVAL:
+        _package_links = await _get_links()
         _last_scrape = datetime.now()
 
-    return _jaxlib_links
+    return _package_links
 
 
-async def _get_jaxlib_links() -> dict[str, str]:
-    # TODO cache
-    links = {}
+async def _get_links() -> dict[str, _Links]:
+    links = collections.defaultdict(dict)
 
     async with httpx.AsyncClient() as client:
         r: httpx.Response = await client.get(str(JAX_URL))
         r.raise_for_status()
 
         xml = r.text
-        for match in re.findall(PATTERN_XML_KEY, xml):
-            if not match.endswith('.whl'):
+        for release in re.findall(PATTERN_XML_KEY, xml):
+            if Path(release).suffix not in RELEASE_SUFFIXES:
+                continue
+
+            url = JAX_URL / release
+
+            if not (name := url.name):
                 continue
 
-            links[match] = str(JAX_URL / match)
+            package_name, tag, *tail = name.split('-')
+
+            attrs = {'href': str(url)}
+
+            if tail:
+                version_raw = tail[0]
+                assert version_raw.startswith('cp')
+
+                version = int(version_raw[2]), int(version_raw[3:])
+                version_str = '.'.join(map(str, version))
+
+                attrs['data-requires-python'] = version_str
+
+            links[package_name][release] = attrs
 
     return links
 
 
-@app.get("/", response_class=HTMLResponse)
+def render_listing(request: Request, title: str, links: _Links, **context):
+    context = {'request': request, 'title': title, 'links': links} | context
+    return templates.TemplateResponse('listing.html', context)
+
+
+@app.get('/', response_class=HTMLResponse)
 async def index(request: Request):
-    return templates.TemplateResponse("index.html", {
-        "request": request,
-        "version": VERSION,
-    })
-
-
-@app.get("/jaxlib/", response_class=HTMLResponse)
-async def package_jaxlib(request: Request):
-    links = await get_jaxlib_links()
-
-    return templates.TemplateResponse("package.html", {
-        "request": request,
-        "version": VERSION,
-        "name": "jaxlib",
-        "links": links,
-    })
+    links = await get_package_links()
+
+    return render_listing(request, title='Simple index', links=links)
+
+
+@app.get('/{name}/', response_class=HTMLResponse)
+async def package(request: Request, name: str):
+    title = f'Links for {name}'
+    links = await get_package_release_links(name)
+
+    return render_listing(request, title=title, heading=title, links=links)
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "jax_pep503"
-version = "0.2.0"
+version = "0.3.0"
 description = "PEP 503 compliant repository for Jax"
 authors = ["Joren Hammudoglu <jhammudoglu@gmail.com>"]
 homepage = "https://jorenham.github.io/jax_pep503/"
@@ -21,6 +21,7 @@ httpx = ">=0.21,<1.0"
 Jinja2 = ">=3.0,<4.0"
 yarl = ">=1.7,<2.0"
 requests = "^2.27"
+mainpy = "^1.0.0"
 
 [tool.poetry.dev-dependencies]
 

diff --git a/templates/index.html b/templates/index.html
diff --git a/templates/listing.html b/templates/listing.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta name="pypi:repository-version" content="1.0">{# PEP 629 #}
+    <title>{{ title }}</title>
+</head>
+<body>
+{% if heading %}<h1>{{ heading }}</h1>{% endif %}
+{% for name, attrs in links.items() -%}
+    <a{{ attrs|xmlattr }}>{{ name }}</a><br>
+{% endfor -%}
+</body>
+</html>
diff --git a/templates/package.html b/templates/package.html