GitHub webhook handler to refresh notebookrunner flocks

src/mobu/config.py

@@ -65,6 +65,17 @@ class Configuration(BaseSettings):
         examples=["gt-vilSCi1ifK_MyuaQgMD2dQ.d6SIJhowv5Hs3GvujOyUig"],
     )
 
+    github_webhook_secret: str | None = Field(
+        None,
+        title="Github webhook secret",
+        description=(
+            "Any repo that wants mobu to automatically respawn labs when"
+            " notebooks change must use this secret in its webhook"
+            " configuration in GitHub."
+        ),
+        validation_alias="MOBU_GITHUB_WEBHOOK_SECRET",
+    )
+
     name: str = Field(
         "mobu",
         title="Name of application",

src/mobu/dependencies/context.py

@@ -12,6 +12,7 @@
 from fastapi import Depends, Request
 from safir.dependencies.gafaelfawr import auth_logger_dependency
 from safir.dependencies.http_client import http_client_dependency
+from safir.dependencies.logger import logger_dependency
 from structlog.stdlib import BoundLogger
 
 from ..factory import Factory, ProcessContext
@@ -21,6 +22,7 @@
     "ContextDependency",
     "RequestContext",
     "context_dependency",
+    "anonymous_context_dependency",
 ]
 
 
@@ -90,3 +92,11 @@ async def aclose(self) -> None:
 
 context_dependency = ContextDependency()
 """The dependency that will return the per-request context."""
+
+
+async def anonymous_context_dependency(
+    request: Request,
+    logger: Annotated[BoundLogger, Depends(logger_dependency)],
+) -> RequestContext:
+    """Per-request context for non-gafaelfawr-auth'd requests."""
+    return await context_dependency(request=request, logger=logger)

src/mobu/handlers/external.py

@@ -1,23 +1,30 @@
 """Handlers for the app's external root, ``/mobu/``."""
 
+import asyncio
 import json
 from collections.abc import Iterator
 from pathlib import Path
 from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, Response
 from fastapi.responses import JSONResponse, StreamingResponse
+from gidgethub.sansio import Event
 from safir.datetime import current_datetime
 from safir.metadata import get_metadata
 from safir.models import ErrorModel
 from safir.slack.webhook import SlackRouteErrorHandler
 
 from ..config import config
-from ..dependencies.context import RequestContext, context_dependency
+from ..dependencies.context import (
+    RequestContext,
+    anonymous_context_dependency,
+    context_dependency,
+)
 from ..models.flock import FlockConfig, FlockData, FlockSummary
 from ..models.index import Index
 from ..models.monkey import MonkeyData
 from ..models.solitary import SolitaryConfig, SolitaryResult
+from .github_webhooks import webhook_router
 
 external_router = APIRouter(route_class=SlackRouteErrorHandler)
 """FastAPI router for all external handlers."""
@@ -246,3 +253,33 @@ async def get_summary(
     context: Annotated[RequestContext, Depends(context_dependency)],
 ) -> list[FlockSummary]:
     return context.manager.summarize_flocks()
+
+
+@external_router.post(
+    "/github/webhook",
+    summary="GitHub webhooks",
+    description="This endpoint receives webhook events from GitHub.",
+    status_code=202,
+)
+async def post_github_webhook(
+    context: Annotated[RequestContext, Depends(anonymous_context_dependency)],
+) -> Response:
+    """Process GitHub webhook events.
+
+    This should be exposed via a Gafaelfawr anonymous ingress.
+    """
+    webhook_secret = config.github_webhook_secret
+    body = await context.request.body()
+    event = Event.from_http(
+        context.request.headers, body, secret=webhook_secret
+    )
+
+    # Bind the X-GitHub-Delivery header to the logger context; this
+    # identifies the webhook request in GitHub's API and UI for
+    # diagnostics
+    logger = context.logger.bind(github_delivery=event.delivery_id)
+
+    logger.debug("Received GitHub webhook", payload=event.data)
+    # Give GitHub some time to reach internal consistency.
+    await asyncio.sleep(1)
+    await webhook_router.dispatch(event=event, logger=logger, context=context)

src/mobu/handlers/github_webhooks.py

@@ -0,0 +1,42 @@
+"""Github webhook handlers."""
+
+from gidgethub import routing
+from gidgethub.sansio import Event
+from structlog.stdlib import BoundLogger
+
+from ..dependencies.context import RequestContext
+
+__all__ = ["webhook_router"]
+
+webhook_router = routing.Router()
+
+
+@webhook_router.register("push")
+async def handle_push(
+    event: Event, logger: BoundLogger, context: RequestContext
+) -> None:
+    """Handle a push event."""
+    ref = event.data["ref"]
+    url = event.data["repository"]["clone_url"]
+    logger = logger.bind(ref=ref, url=url)
+
+    prefix, branch = ref.rsplit("/", 1)
+    if prefix != "refs/heads":
+        logger.debug(
+            "github webhook ignored: ref is not a branch",
+        )
+        return
+
+    flocks = context.manager.list_flocks_for_repo(
+        repo_url=url, repo_branch=branch
+    )
+    if not flocks:
+        logger.debug(
+            "github webhook ignored: no flocks match repo and branch",
+        )
+        return
+
+    for flock in flocks:
+        context.manager.refresh_flock(flock)
+
+    logger.info("github webhook handled")

src/mobu/services/flock.py

@@ -12,6 +12,10 @@
 from structlog.stdlib import BoundLogger
 
 from ..exceptions import MonkeyNotFoundError
+from ..models.business.notebookrunner import (
+    NotebookRunnerConfig,
+    NotebookRunnerOptions,
+)
 from ..models.flock import FlockConfig, FlockData, FlockSummary
 from ..models.user import AuthenticatedUser, User, UserSpec
 from ..storage.gafaelfawr import GafaelfawrStorage
@@ -136,6 +140,19 @@ def signal_refresh(self) -> None:
         for monkey in self._monkeys.values():
             monkey.signal_refresh()
 
+    def uses_repo(self, repo_url: str, repo_branch: str) -> bool:
+        match self._config:
+            case FlockConfig(
+                business=NotebookRunnerConfig(
+                    options=NotebookRunnerOptions(
+                        repo_url=url,
+                        repo_branch=branch,
+                    )
+                )
+            ) if (url, branch) == (repo_url, repo_branch):
+                return True
+        return False
+
     def _create_monkey(self, user: AuthenticatedUser) -> Monkey:
         """Create a monkey that will run as a given user."""
         return Monkey(

src/mobu/services/manager.py

@@ -118,6 +118,15 @@ def get_flock(self, name: str) -> Flock:
             raise FlockNotFoundError(name)
         return flock
 
+    def list_flocks_for_repo(
+        self, repo_url: str, repo_branch: str
+    ) -> list[str]:
+        return [
+            name
+            for name, flock in self._flocks.items()
+            if flock.uses_repo(repo_url=repo_url, repo_branch=repo_branch)
+        ]
+
     def list_flocks(self) -> list[str]:
         """List all flocks.
 

tests/conftest.py

@@ -19,7 +19,7 @@
 from mobu.config import config
 from mobu.services.business.gitlfs import GitLFSBusiness
 
-from .support.constants import TEST_BASE_URL
+from .support.constants import TEST_BASE_URL, TEST_GITHUB_WEBHOOK_SECRET
 from .support.gafaelfawr import make_gafaelfawr_token
 from .support.gitlfs import (
     no_git_lfs_data,
@@ -47,6 +47,7 @@ def _configure() -> Iterator[None]:
     """
     config.environment_url = HttpUrl("https://test.example.com")
     config.gafaelfawr_token = make_gafaelfawr_token()
+    config.github_webhook_secret = TEST_GITHUB_WEBHOOK_SECRET
     yield
     config.environment_url = None
     config.gafaelfawr_token = None
@@ -83,6 +84,18 @@ async def client(app: FastAPI) -> AsyncIterator[AsyncClient]:
         yield client
 
 
+@pytest_asyncio.fixture
+async def anon_client(app: FastAPI) -> AsyncIterator[AsyncClient]:
+    """Return an anonymous ``httpx.AsyncClient`` configured to talk to the test
+    app.
+    """
+    async with AsyncClient(
+        transport=ASGITransport(app=app),  # type: ignore[arg-type]
+        base_url=TEST_BASE_URL,
+    ) as client:
+        yield client
+
+
 @pytest.fixture
 def jupyter(respx_mock: respx.Router) -> Iterator[MockJupyter]:
     """Mock out JupyterHub and Jupyter labs."""

tests/handlers/github_test.py

@@ -0,0 +1,155 @@
+"""Test the github webhook handler."""
+
+import hashlib
+import hmac
+from dataclasses import dataclass
+from pathlib import Path
+from string import Template
+
+import pytest
+import respx
+from httpx import AsyncClient
+from pytest_mock import MockerFixture
+
+import mobu.services.flock
+
+from ..support.constants import TEST_GITHUB_WEBHOOK_SECRET
+from ..support.gafaelfawr import mock_gafaelfawr
+
+
+@dataclass(frozen=True)
+class GithubRequest:
+    payload: str
+    headers: dict[str, str]
+
+
+def webhook_request(org: str, repo: str, ref: str) -> GithubRequest:
+    """Build a Github webhook request and headers with the right hash."""
+    data_path = Path(__file__).parent.parent / "support" / "data"
+    template = (data_path / "github_webhook.tmpl.json").read_text()
+    payload = Template(template).substitute(
+        org=org,
+        repo=repo,
+        ref=ref,
+    )
+
+    # https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries#python-example
+    hash_object = hmac.new(
+        TEST_GITHUB_WEBHOOK_SECRET.encode("utf-8"),
+        msg=payload.encode("utf-8"),
+        digestmod=hashlib.sha256,
+    )
+    sig = "sha256=" + hash_object.hexdigest()
+
+    headers = {
+        "Accept": "*/*",
+        "Content-Type": "application/json",
+        "User-Agent": "GitHub-Hookshot/c9d6c0a",
+        "X-GitHub-Delivery": "d2d3c948-1d61-11ef-848a-c578f23615c9",
+        "X-GitHub-Event": "push",
+        "X-GitHub-Hook-ID": "479971864",
+        "X-GitHub-Hook-Installation-Target-ID": "804427678",
+        "X-GitHub-Hook-Installation-Target-Type": "repository",
+        "X-Hub-Signature-256": sig,
+    }
+
+    return GithubRequest(payload=payload, headers=headers)
+
+
+@pytest.mark.asyncio
+async def test_handle_webhook(
+    client: AsyncClient, respx_mock: respx.Router, mocker: MockerFixture
+) -> None:
+    configs = [
+        {
+            "name": "test-notebook",
+            "count": 1,
+            "user_spec": {"username_prefix": "testuser-notebook"},
+            "scopes": ["exec:notebook"],
+            "business": {
+                "type": "NotebookRunner",
+                "options": {
+                    "repo_url": "https://github.com/lsst-sqre/some-repo.git",
+                    "repo_branch": "main",
+                },
+            },
+        },
+        {
+            "name": "test-notebook-branch",
+            "count": 1,
+            "user_spec": {"username_prefix": "testuser-notebook-branch"},
+            "scopes": ["exec:notebook"],
+            "business": {
+                "type": "NotebookRunner",
+                "options": {
+                    "repo_url": "https://github.com/lsst-sqre/some-repo.git",
+                    "repo_branch": "some-branch",
+                },
+            },
+        },
+        {
+            "name": "test-other-notebook",
+            "count": 1,
+            "user_spec": {"username_prefix": "testuser-other-notebook"},
+            "scopes": ["exec:notebook"],
+            "business": {
+                "type": "NotebookRunner",
+                "options": {
+                    "repo_url": "https://github.com/lsst-sqre/some-other-repo.git",
+                    "repo_branch": "main",
+                },
+            },
+        },
+        {
+            "name": "test-non-notebook",
+            "count": 1,
+            "user_spec": {"username_prefix": "testuser-non-notebook"},
+            "scopes": ["exec:notebook"],
+            "business": {"type": "EmptyLoop"},
+        },
+    ]
+
+    # Don't actually do any business
+    mocker.patch.object(mobu.services.flock.Monkey, "start")  # type: ignore[attr-defined]
+    mocker.patch.object(mobu.services.flock.Monkey, "stop")  # type: ignore[attr-defined]
+
+    mock_gafaelfawr(respx_mock)
+
+    # Start the flocks
+    for config in configs:
+        r = await client.put("/mobu/flocks", json=config)
+        assert r.status_code == 201
+
+    # Post a webhook event like GitHub would
+    request = webhook_request(
+        org="lsst-sqre",
+        repo="some-repo",
+        ref="refs/heads/main",
+    )
+    await client.post(
+        "/mobu/github/webhook",
+        headers=request.headers,
+        content=request.payload,
+    )
+
+    # Only the business for the correct branch and repo should be refreshing
+    r = await client.get(
+        "/mobu/flocks/test-notebook/monkeys/testuser-notebook1"
+    )
+    assert r.json()["business"]["refreshing"] is True
+
+    # The other businesses should not be refreshing
+    r = await client.get(
+        "/mobu/flocks/test-notebook-branch/monkeys/testuser-notebook-branch1"
+    )
+    assert r.json()["business"]["refreshing"] is False
+
+    r = await client.get(
+        "/mobu/flocks/test-other-notebook/monkeys/testuser-other-notebook1"
+    )
+    assert r.json()["business"]["refreshing"] is False
+
+    r = await client.get(
+        "/mobu/flocks/test-non-notebook/monkeys/testuser-non-notebook1"
+    )
+    assert r.json()["business"]["refreshing"] is False