DM-41528: Incorporate the REST spawner and JupyterHub build

.github/workflows/ci.yaml

@@ -124,11 +124,23 @@ jobs:
           fetch-depth: 0
 
       - uses: lsst-sqre/build-and-push-to-ghcr@v1
-        id: build
+        id: build1
         with:
+          dockerfile: Dockerfile.controller
           image: ${{ github.repository }}-controller
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Report result
         run: |
-          echo Pushed ghcr.io/${{ github.repository }}-controller:${{ steps.build.outputs.tag }}
+          echo Pushed ghcr.io/${{ github.repository }}-controller:${{ steps.build1.outputs.tag }}
+
+      - uses: lsst-sqre/build-and-push-to-ghcr@v1
+        id: build2
+        with:
+          dockerfile: Dockerfile.controller
+          image: ${{ github.repository }}-jupyterhub
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Report result
+        run: |
+          echo Pushed ghcr.io/${{ github.repository }}-jupyterhub:${{ steps.build2.outputs.tag }}

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.3
+    rev: v0.1.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -17,6 +17,7 @@ repos:
     rev: 23.10.1
     hooks:
       - id: black
+        args: [--config, pyproject.toml]
 
   - repo: https://github.com/adamchainz/blacken-docs
     rev: 1.16.0

Dockerfile → Dockerfile.controller

@@ -1,12 +1,14 @@
+# Docker build instructions for the Nublado controller.
+#
 # This Dockerfile has four stages:
 #
 # base-image
 #   Updates the base Python image with security patches and common system
 #   packages. This image becomes the base of all other images.
 # dependencies-image
-#   Installs third-party dependencies (requirements/main.txt) into a virtual
-#   environment. This virtual environment is ideal for copying across build
-#   stages.
+#   Installs third-party dependencies (controller/requirements/main.txt)
+#   into a virtual environment. This virtual environment is ideal for
+#   copying across build stages.
 # install-image
 #   Installs the app into the virtual environment.
 # runtime-image
@@ -17,13 +19,13 @@
 FROM python:3.11.5-slim-bullseye as base-image
 
 # Update system packages
-COPY scripts/install-base-packages.sh .
+COPY controller/scripts/install-base-packages.sh .
 RUN ./install-base-packages.sh && rm ./install-base-packages.sh
 
 FROM base-image AS dependencies-image
 
 # Install system packages only needed for building dependencies.
-COPY scripts/install-dependency-packages.sh .
+COPY controller/scripts/install-dependency-packages.sh .
 RUN ./install-dependency-packages.sh
 
 # Create a Python virtual environment
@@ -56,7 +58,7 @@ RUN useradd --create-home appuser
 COPY --from=install-image /opt/venv /opt/venv
 
 # Copy the startup script
-COPY scripts/start.sh /start.sh
+COPY controller/scripts/start.sh /start.sh
 
 # Make sure we use the virtualenv
 ENV PATH="/opt/venv/bin:$PATH"

Dockerfile.hub

@@ -0,0 +1,49 @@
+# Docker build instructions for the custom Nublado JupyterHub build.
+#
+# This Dockerfile has three stages:
+#
+# base-image
+#   Updates the base Python image with security patches and common system
+#   packages. This image becomes the base of all other images.
+# dependencies-image
+#   Installs third-party dependencies (spawner/requirements/main.txt) into
+#   a virtual environment. This virtual environment is ideal for copying
+#   across build stages.
+# runtime-image
+#   - Copies the virtual environment into place.
+#   - Installs the additional JupyterHub plugins.
+#   - Runs a non-root user.
+
+FROM jupyterhub/jupyterhub:4.0.2 as base-image
+
+# Update system packages
+COPY spawner/scripts/install-base-packages.sh .
+RUN ./install-base-packages.sh && rm install-base-packages.sh
+
+FROM base-image as dependencies-image
+
+# Install system packages only needed for building dependencies.
+COPY spawner/scripts/install-dependency-packages.sh .
+RUN ./install-dependency-packages.sh && rm install-dependency-packages.sh
+
+# Install the dependencies of the extra JupyterHub modules.
+COPY spawner/requirements/main.txt ./requirements.txt
+RUN pip install --quiet --no-cache-dir -r requirements.txt
+
+FROM dependencies-image as runtime-image
+
+# Install the extra JupyterHub modules.
+COPY . /app
+WORKDIR /app
+RUN pip install --no-cache-dir spawner
+
+# Create a non-root user to run JupyterHub. Upstream uses 768 as the UID
+# and GID. This value must be kept in sync with the Phalanx nublado
+# application values.yaml.
+RUN groupadd --gid 768 jovyan
+RUN useradd --create-home jovyan --uid 768 --gid 768
+WORKDIR /home/jovyan
+
+USER jovyan
+EXPOSE 8000
+EXPOSE 8081

controller/LICENSE

@@ -0,0 +1 @@
+../LICENSE

controller/requirements/dev.txt

@@ -19,7 +19,7 @@ anyio==3.7.1 \
     --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5
     # via
     #   -c controller/requirements/main.txt
-    #   httpcore
+    #   httpx
 asgi-lifespan==2.1.0 \
     --hash=sha256:5e2effaf0bfe39829cf2d64e7ecc47c7d86d676a6599f7afba378c31f5e3a308 \
     --hash=sha256:ed840706680e28428c01e14afb3875d7d76d3206f3d5b2f2294e059b5c23804f
@@ -247,15 +247,15 @@ h11==0.14.0 \
     # via
     #   -c controller/requirements/main.txt
     #   httpcore
-httpcore==0.18.0 \
-    --hash=sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9 \
-    --hash=sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced
+httpcore==1.0.1 \
+    --hash=sha256:c5e97ef177dca2023d0b9aad98e49507ef5423e9f1d94ffe2cfe250aa28e63b0 \
+    --hash=sha256:fce1ddf9b606cfb98132ab58865c3728c52c8e4c3c46e2aabb3674464a186e92
     # via
     #   -c controller/requirements/main.txt
     #   httpx
-httpx==0.25.0 \
-    --hash=sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100 \
-    --hash=sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875
+httpx==0.25.1 \
+    --hash=sha256:fec7d6cc5c27c578a391f7e87b9aa7d3d8fbcd034f6399f9f79b45bcc12a866a \
+    --hash=sha256:ffd96d5cf901e63863d9f1b4b6807861dbea4d301613415d9e6e57ead15fc5d0
     # via
     #   -c controller/requirements/main.txt
     #   respx
@@ -802,7 +802,6 @@ sniffio==1.3.0 \
     #   -c controller/requirements/main.txt
     #   anyio
     #   asgi-lifespan
-    #   httpcore
     #   httpx
 snowballstemmer==2.2.0 \
     --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \

controller/requirements/main.txt

@@ -110,7 +110,7 @@ anyio==3.7.1 \
     --hash=sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5
     # via
     #   fastapi
-    #   httpcore
+    #   httpx
     #   starlette
     #   watchfiles
 async-timeout==4.0.3 \
@@ -402,9 +402,9 @@ google-auth==2.23.4 \
     --hash=sha256:79905d6b1652187def79d491d6e23d0cbb3a21d3c7ba0dbaa9c8a01906b13ff3 \
     --hash=sha256:d4bbc92fe4b8bfd2f3e8d88e5ba7085935da208ee38a134fc280e7ce682a05f2
     # via google-api-core
-google-cloud-artifact-registry==1.8.4 \
-    --hash=sha256:2deba5c9c0065cf8e2e63c547e93a4921867d57261d57d4a756be1031529e228 \
-    --hash=sha256:a5066555fe8324cc6e8498014b054b1d3e87a8f9e1b0fa1e47ba3bbc3daaa581
+google-cloud-artifact-registry==1.9.0 \
+    --hash=sha256:3484d2a2c04fadfa480be16a554d01e47e4c4f4b5a123cd3725e865b4dd7f325 \
+    --hash=sha256:aeee14226b1636e7bc043476cff756876068872bda3246a3b124c39d3f4e3be8
     # via -r controller/requirements/main.in
 googleapis-common-protos[grpc]==1.61.0 \
     --hash=sha256:22f1915393bb3245343f6efe87f6fe868532efc12aa26b391b15132e1279f1c0 \
@@ -487,9 +487,9 @@ h11==0.14.0 \
     # via
     #   httpcore
     #   uvicorn
-httpcore==0.18.0 \
-    --hash=sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9 \
-    --hash=sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced
+httpcore==1.0.1 \
+    --hash=sha256:c5e97ef177dca2023d0b9aad98e49507ef5423e9f1d94ffe2cfe250aa28e63b0 \
+    --hash=sha256:fce1ddf9b606cfb98132ab58865c3728c52c8e4c3c46e2aabb3674464a186e92
     # via httpx
 httptools==0.6.1 \
     --hash=sha256:00d5d4b68a717765b1fabfd9ca755bd12bf44105eeb806c03d1962acd9b8e563 \
@@ -529,9 +529,9 @@ httptools==0.6.1 \
     --hash=sha256:e57997ac7fb7ee43140cc03664de5f268813a481dff6245e0075925adc6aa185 \
     --hash=sha256:fe467eb086d80217b7584e61313ebadc8d187a4d95bb62031b7bab4b205c3ba3
     # via uvicorn
-httpx==0.25.0 \
-    --hash=sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100 \
-    --hash=sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875
+httpx==0.25.1 \
+    --hash=sha256:fec7d6cc5c27c578a391f7e87b9aa7d3d8fbcd034f6399f9f79b45bcc12a866a \
+    --hash=sha256:ffd96d5cf901e63863d9f1b4b6807861dbea4d301613415d9e6e57ead15fc5d0
     # via
     #   -r controller/requirements/main.in
     #   safir
@@ -948,7 +948,6 @@ sniffio==1.3.0 \
     --hash=sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384
     # via
     #   anyio
-    #   httpcore
     #   httpx
 sse-starlette==1.6.5 \
     --hash=sha256:68b6b7eb49be0c72a2af80a055994c13afcaa4761b29226beb208f954c25a642 \