Merge pull request #211 from lsst-sqre/tickets/DM-41528

DM-41528: Reorganize the JupyterHub build

Dockerfile.hub

@@ -17,25 +17,25 @@
 FROM jupyterhub/jupyterhub:4.0.2 as base-image
 
 # Update system packages
-COPY spawner/scripts/install-base-packages.sh .
+COPY hub/scripts/install-base-packages.sh .
 RUN ./install-base-packages.sh && rm install-base-packages.sh
 
 FROM base-image as dependencies-image
 
 # Install system packages only needed for building dependencies.
-COPY spawner/scripts/install-dependency-packages.sh .
+COPY hub/scripts/install-dependency-packages.sh .
 RUN ./install-dependency-packages.sh && rm install-dependency-packages.sh
 
 # Install the dependencies of the extra JupyterHub modules.
-COPY spawner/requirements/main.txt ./requirements.txt
+COPY hub/requirements/main.txt ./requirements.txt
 RUN pip install --quiet --no-cache-dir -r requirements.txt
 
 FROM dependencies-image as runtime-image
 
 # Install the extra JupyterHub modules.
 COPY . /app
 WORKDIR /app
-RUN pip install --no-cache-dir spawner
+RUN pip install --no-cache-dir ./authenticator ./spawner
 
 # Create a non-root user to run JupyterHub. Upstream uses 768 as the UID
 # and GID. This value must be kept in sync with the Phalanx nublado

authenticator/LICENSE

@@ -0,0 +1 @@
+../LICENSE

authenticator/README.md

@@ -0,0 +1,6 @@
+# Gafaelfawr JupyterHub authenticator
+
+This is an implementation of the JupyterHub `Authenticator` class that authenticates a user using [Gafaelfawr](https://gafaelfawr.lsst.io), assuming authentication is configured using [Phalanx](https://phalanx.lsst.io).
+It is, in theory, generic, and is maintained as a stand-alone Python module, but is normally installed in a Docker image with JupyterHub as part of the Rubin Science Platform JupyterHub build.
+
+For more details about this architecture, see [SQR-066](https://sqr-066.lsst.io/).

authenticator/pyproject.toml

@@ -0,0 +1,83 @@
+[project]
+# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
+name = "rubin.nublado.authenticator"
+description = "JupyterHub Authenticator class supporting Gafaelfawr."
+license = { file = "LICENSE" }
+readme = "README.md"
+keywords = ["rubin", "lsst"]
+# https://pypi.org/classifiers/
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Natural Language :: English",
+    "Operating System :: POSIX",
+    "Typing :: Typed",
+]
+requires-python = ">=3.10"
+dependencies = ["jupyterhub<5", "tornado<7"]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+dev = [
+    # Testing
+    "pytest",
+    "pytest-asyncio",
+    "pytest-cov",
+    "pytest-sugar",
+]
+
+[project.urls]
+Homepage = "https://nublado.lsst.io"
+Source = "https://github.com/lsst-sqre/nublado"
+
+[build-system]
+requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = 'setuptools.build_meta'
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools_scm]
+root = ".."
+
+[tool.coverage.run]
+parallel = true
+branch = true
+source = ["rubin.nublado.authenticator"]
+
+[tool.coverage.paths]
+source = ["src"]
+
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:"
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "strict"
+filterwarnings = [
+    # Will probably be fixed with JupyterHub v4.
+    "ignore:'pipes' is deprecated:DeprecationWarning:jupyterhub.spawner"
+]
+# The python_files setting is not for test detection (pytest will pick up any
+# test files named *_test.py without this setting) but to enable special
+# assert processing in any non-test supporting files under tests.  We
+# conventionally put test support functions under tests.support and may
+# sometimes use assert in test fixtures in conftest.py, and pytest only
+# enables magical assert processing (showing a full diff on assert failures
+# with complex data structures rather than only the assert message) in files
+# listed in python_files.
+python_files = ["tests/*.py", "tests/*/*.py"]

authenticator/src/rubin/nublado/authenticator/__init__.py

@@ -0,0 +1,5 @@
+"""Gafaelfawr authenticator for JupyterHub."""
+
+from ._internals import GafaelfawrAuthenticator
+
+__all__ = ["GafaelfawrAuthenticator"]

spawner/src/rsp_restspawner/auth.py → authenticator/src/rubin/nublado/authenticator/_internals.py

@@ -30,6 +30,50 @@ def _build_auth_info(headers: HTTPHeaders) -> AuthInfo:
     return {"name": username, "auth_state": {"token": token}}
 
 
+class _GafaelfawrLogoutHandler(LogoutHandler):
+    """Logout handler for Gafaelfawr authentication.
+
+    A logout should always stop all running servers, and then redirect to the
+    RSP logout page.
+    """
+
+    @property
+    def shutdown_on_logout(self) -> bool:
+        """Unconditionally true for Gafaelfawr logout."""
+        return True
+
+    async def render_logout_page(self) -> None:
+        self.redirect("/logout", permanent=False)
+
+
+class _GafaelfawrLoginHandler(BaseHandler):
+    """Login handler for Gafaelfawr authentication.
+
+    This retrieves the authentication token from the headers, makes an API
+    call to get its metadata, constructs an authentication state, and then
+    redirects to the next URL.
+    """
+
+    async def get(self) -> None:
+        """Handle GET to the login page."""
+        auth_info = _build_auth_info(self.request.headers)
+
+        # Store the ancillary user information in the user database and create
+        # or return the user object. This call is unfortunately undocumented,
+        # but it's what BaseHandler calls to record the auth_state information
+        # after a form-based login. Hopefully this is a stable interface.
+        user = await self.auth_to_user(auth_info)
+
+        # Tell JupyterHub to set its login cookie (also undocumented).
+        self.set_login_cookie(user)
+
+        # Redirect to the next URL, which is under the control of JupyterHub
+        # and opaque to the authenticator. In practice, it will normally be
+        # whatever URL the user was trying to go to when JupyterHub decided
+        # they needed to be authenticated.
+        self.redirect(self.get_next_url(user))
+
+
 class GafaelfawrAuthenticator(Authenticator):
     """JupyterHub authenticator using Gafaelfawr headers.
 
@@ -54,11 +98,10 @@ class GafaelfawrAuthenticator(Authenticator):
     ``auto_login`` setting on the configured authenticator. This setting tells
     the built-in login page to, instead of presenting a login form, redirect
     the user to whatever URL is returned by ``login_url``. In our case, this
-    will be ``/hub/gafaelfawr/login``, served by `GafaelfawrLoginHandler`.
-    This simple handler will read the token from the header, retrieve its
-    metadata, create the session and cookie, and then make the same redirect
-    call the login form handler would normally have made after the
-    ``authenticate`` method returned.
+    will be ``/hub/gafaelfawr/login``. This simple handler will read the token
+    from the header, retrieve its metadata, create the session and cookie, and
+    then make the same redirect call the login form handler would normally
+    have made after the ``authenticate`` method returned.
 
     In this model, the ``authenticate`` method is not used, since the login
     handler never receives a form submission.
@@ -107,8 +150,8 @@ async def authenticate(
     def get_handlers(self, app: JupyterHub) -> list[Route]:
         """Register the header-only login and the logout handlers."""
         return [
-            ("/gafaelfawr/login", GafaelfawrLoginHandler),
-            ("/logout", GafaelfawrLogoutHandler),
+            ("/gafaelfawr/login", _GafaelfawrLoginHandler),
+            ("/logout", _GafaelfawrLogoutHandler),
         ]
 
     def login_url(self, base_url: str) -> str:
@@ -152,47 +195,3 @@ async def refresh_user(
             return True
         else:
             return _build_auth_info(handler.request.headers)
-
-
-class GafaelfawrLogoutHandler(LogoutHandler):
-    """Logout handler for Gafaelfawr authentication.
-
-    A logout should always stop all running servers, and then redirect to the
-    RSP logout page.
-    """
-
-    @property
-    def shutdown_on_logout(self) -> bool:
-        """Unconditionally true for Gafaelfawr logout."""
-        return True
-
-    async def render_logout_page(self) -> None:
-        self.redirect("/logout", permanent=False)
-
-
-class GafaelfawrLoginHandler(BaseHandler):
-    """Login handler for Gafaelfawr authentication.
-
-    This retrieves the authentication token from the headers, makes an API
-    call to get its metadata, constructs an authentication state, and then
-    redirects to the next URL.
-    """
-
-    async def get(self) -> None:
-        """Handle GET to the login page."""
-        auth_info = _build_auth_info(self.request.headers)
-
-        # Store the ancillary user information in the user database and create
-        # or return the user object. This call is unfortunately undocumented,
-        # but it's what BaseHandler calls to record the auth_state information
-        # after a form-based login. Hopefully this is a stable interface.
-        user = await self.auth_to_user(auth_info)
-
-        # Tell JupyterHub to set its login cookie (also undocumented).
-        self.set_login_cookie(user)
-
-        # Redirect to the next URL, which is under the control of JupyterHub
-        # and opaque to the authenticator. In practice, it will normally be
-        # whatever URL the user was trying to go to when JupyterHub decided
-        # they needed to be authenticated.
-        self.redirect(self.get_next_url(user))

spawner/tests/auth_test.py → authenticator/tests/authenticator_test.py

@@ -13,20 +13,20 @@
 from tornado import web
 from tornado.httputil import HTTPHeaders
 
-from rsp_restspawner import GafaelfawrAuthenticator
-from rsp_restspawner.auth import (
-    GafaelfawrLoginHandler,
-    GafaelfawrLogoutHandler,
+from rubin.nublado.authenticator import GafaelfawrAuthenticator
+from rubin.nublado.authenticator._internals import (
     _build_auth_info,
+    _GafaelfawrLoginHandler,
+    _GafaelfawrLogoutHandler,
 )
 
 
 @pytest.mark.asyncio
 async def test_authenticator() -> None:
     authenticator = GafaelfawrAuthenticator()
     assert authenticator.get_handlers(MagicMock()) == [
-        ("/gafaelfawr/login", GafaelfawrLoginHandler),
-        ("/logout", GafaelfawrLogoutHandler),
+        ("/gafaelfawr/login", _GafaelfawrLoginHandler),
+        ("/logout", _GafaelfawrLogoutHandler),
     ]
 
     assert authenticator.login_url("/hub") == "/hub/gafaelfawr/login"

hub/README.md

@@ -0,0 +1,11 @@
+# RSP JupyterHub build
+
+This directory holds the build machinery for the modified JupyterHub build used by the Notebook Aspect of the Rubin Science Platform.
+
+The RSP JupyterHub is a combination of the standard JupyterHub build plus a custom authenticator and a custom spawner.
+The authenticator and spawner are maintained as separate Python library packages with normal floating dependencies.
+However, for a reproducible JupyterHub image build, we want to pin all Python dependencies and add some supplemental scripts.
+
+This directory contains the pinned Python dependencies and the supplemental scripts.
+It is used by `Dockerfile.hub` when building the JupyterHub image.
+The pinned dependencies here are only used for that image, and may differ from the pinned dependencies for the Nublado controller.

spawner/requirements/main.in → hub/requirements/main.in

@@ -11,7 +11,7 @@
 # same version that Dockerfile uses.
 jupyterhub==4.0.2
 
-# Dependencies used directly by the code added by this package.
+# Dependencies used directly by the authenticator and spawner.
 httpx
 httpx-sse
 PyYAML
@@ -31,6 +31,3 @@ kubernetes_asyncio
 # JupyterHub image currently uses.
 exceptiongroup
 ruamel.yaml.clib
-
-# anyio 4 currently conflicts with FastAPI.
-anyio<4