diff --git a/src/mobu/config.py b/src/mobu/config.py index 3ab3fe22..86f894a7 100644 --- a/src/mobu/config.py +++ b/src/mobu/config.py @@ -65,6 +65,17 @@ class Configuration(BaseSettings): examples=["gt-vilSCi1ifK_MyuaQgMD2dQ.d6SIJhowv5Hs3GvujOyUig"], ) + github_webhook_secret: str | None = Field( + None, + title="Github webhook secret", + description=( + "Any repo that wants mobu to automatically respawn labs when" + " notebooks change must use this secret in its webhook" + " configuration in GitHub." + ), + validation_alias="MOBU_GITHUB_WEBHOOK_SECRET", + ) + name: str = Field( "mobu", title="Name of application", diff --git a/src/mobu/dependencies/context.py b/src/mobu/dependencies/context.py index e3de4768..6377b4bd 100644 --- a/src/mobu/dependencies/context.py +++ b/src/mobu/dependencies/context.py @@ -12,6 +12,7 @@ from fastapi import Depends, Request from safir.dependencies.gafaelfawr import auth_logger_dependency from safir.dependencies.http_client import http_client_dependency +from safir.dependencies.logger import logger_dependency from structlog.stdlib import BoundLogger from ..factory import Factory, ProcessContext @@ -21,6 +22,7 @@ "ContextDependency", "RequestContext", "context_dependency", + "anonymous_context_dependency", ] @@ -90,3 +92,11 @@ async def aclose(self) -> None: context_dependency = ContextDependency() """The dependency that will return the per-request context.""" + + +async def anonymous_context_dependency( + request: Request, + logger: Annotated[BoundLogger, Depends(logger_dependency)], +) -> RequestContext: + """Per-request context for non-gafaelfawr-auth'd requests.""" + return await context_dependency(request=request, logger=logger) diff --git a/src/mobu/handlers/external.py b/src/mobu/handlers/external.py index 7c93bc11..114e7aa3 100644 --- a/src/mobu/handlers/external.py +++ b/src/mobu/handlers/external.py @@ -1,5 +1,6 @@ """Handlers for the app's external root, ``/mobu/``.""" +import asyncio import json from collections.abc import Iterator from pathlib import Path @@ -7,17 +8,23 @@ from fastapi import APIRouter, Depends, Response from fastapi.responses import JSONResponse, StreamingResponse +from gidgethub.sansio import Event from safir.datetime import current_datetime from safir.metadata import get_metadata from safir.models import ErrorModel from safir.slack.webhook import SlackRouteErrorHandler from ..config import config -from ..dependencies.context import RequestContext, context_dependency +from ..dependencies.context import ( + RequestContext, + anonymous_context_dependency, + context_dependency, +) from ..models.flock import FlockConfig, FlockData, FlockSummary from ..models.index import Index from ..models.monkey import MonkeyData from ..models.solitary import SolitaryConfig, SolitaryResult +from .github_webhooks import webhook_router external_router = APIRouter(route_class=SlackRouteErrorHandler) """FastAPI router for all external handlers.""" @@ -246,3 +253,33 @@ async def get_summary( context: Annotated[RequestContext, Depends(context_dependency)], ) -> list[FlockSummary]: return context.manager.summarize_flocks() + + +@external_router.post( + "/github/webhook", + summary="GitHub webhooks", + description="This endpoint receives webhook events from GitHub.", + status_code=202, +) +async def post_github_webhook( + context: Annotated[RequestContext, Depends(anonymous_context_dependency)], +) -> Response: + """Process GitHub webhook events. + + This should be exposed via a Gafaelfawr anonymous ingress. + """ + webhook_secret = config.github_webhook_secret + body = await context.request.body() + event = Event.from_http( + context.request.headers, body, secret=webhook_secret + ) + + # Bind the X-GitHub-Delivery header to the logger context; this + # identifies the webhook request in GitHub's API and UI for + # diagnostics + logger = context.logger.bind(github_delivery=event.delivery_id) + + logger.debug("Received GitHub webhook", payload=event.data) + # Give GitHub some time to reach internal consistency. + await asyncio.sleep(1) + await webhook_router.dispatch(event=event, logger=logger, context=context) diff --git a/src/mobu/handlers/github_webhooks.py b/src/mobu/handlers/github_webhooks.py new file mode 100644 index 00000000..96595b4e --- /dev/null +++ b/src/mobu/handlers/github_webhooks.py @@ -0,0 +1,42 @@ +"""Github webhook handlers.""" + +from gidgethub import routing +from gidgethub.sansio import Event +from structlog.stdlib import BoundLogger + +from ..dependencies.context import RequestContext + +__all__ = ["webhook_router"] + +webhook_router = routing.Router() + + +@webhook_router.register("push") +async def handle_push( + event: Event, logger: BoundLogger, context: RequestContext +) -> None: + """Handle a push event.""" + ref = event.data["ref"] + url = event.data["repository"]["clone_url"] + logger = logger.bind(ref=ref, url=url) + + prefix, branch = ref.rsplit("/", 1) + if prefix != "refs/heads": + logger.debug( + "github webhook ignored: ref is not a branch", + ) + return + + flocks = context.manager.list_flocks_for_repo( + repo_url=url, repo_branch=branch + ) + if not flocks: + logger.debug( + "github webhook ignored: no flocks match repo and branch", + ) + return + + for flock in flocks: + context.manager.refresh_flock(flock) + + logger.info("github webhook handled") diff --git a/src/mobu/services/flock.py b/src/mobu/services/flock.py index af820559..9b905e98 100644 --- a/src/mobu/services/flock.py +++ b/src/mobu/services/flock.py @@ -12,6 +12,10 @@ from structlog.stdlib import BoundLogger from ..exceptions import MonkeyNotFoundError +from ..models.business.notebookrunner import ( + NotebookRunnerConfig, + NotebookRunnerOptions, +) from ..models.flock import FlockConfig, FlockData, FlockSummary from ..models.user import AuthenticatedUser, User, UserSpec from ..storage.gafaelfawr import GafaelfawrStorage @@ -136,6 +140,19 @@ def signal_refresh(self) -> None: for monkey in self._monkeys.values(): monkey.signal_refresh() + def uses_repo(self, repo_url: str, repo_branch: str) -> bool: + match self._config: + case FlockConfig( + business=NotebookRunnerConfig( + options=NotebookRunnerOptions( + repo_url=url, + repo_branch=branch, + ) + ) + ) if (url, branch) == (repo_url, repo_branch): + return True + return False + def _create_monkey(self, user: AuthenticatedUser) -> Monkey: """Create a monkey that will run as a given user.""" return Monkey( diff --git a/src/mobu/services/manager.py b/src/mobu/services/manager.py index 2661fedf..08bd04a3 100644 --- a/src/mobu/services/manager.py +++ b/src/mobu/services/manager.py @@ -118,6 +118,15 @@ def get_flock(self, name: str) -> Flock: raise FlockNotFoundError(name) return flock + def list_flocks_for_repo( + self, repo_url: str, repo_branch: str + ) -> list[str]: + return [ + name + for name, flock in self._flocks.items() + if flock.uses_repo(repo_url=repo_url, repo_branch=repo_branch) + ] + def list_flocks(self) -> list[str]: """List all flocks. diff --git a/tests/conftest.py b/tests/conftest.py index a377664d..99ad64ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,7 @@ from mobu.config import config from mobu.services.business.gitlfs import GitLFSBusiness -from .support.constants import TEST_BASE_URL +from .support.constants import TEST_BASE_URL, TEST_GITHUB_WEBHOOK_SECRET from .support.gafaelfawr import make_gafaelfawr_token from .support.gitlfs import ( no_git_lfs_data, @@ -47,6 +47,7 @@ def _configure() -> Iterator[None]: """ config.environment_url = HttpUrl("https://test.example.com") config.gafaelfawr_token = make_gafaelfawr_token() + config.github_webhook_secret = TEST_GITHUB_WEBHOOK_SECRET yield config.environment_url = None config.gafaelfawr_token = None @@ -83,6 +84,18 @@ async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: yield client +@pytest_asyncio.fixture +async def anon_client(app: FastAPI) -> AsyncIterator[AsyncClient]: + """Return an anonymous ``httpx.AsyncClient`` configured to talk to the test + app. + """ + async with AsyncClient( + transport=ASGITransport(app=app), # type: ignore[arg-type] + base_url=TEST_BASE_URL, + ) as client: + yield client + + @pytest.fixture def jupyter(respx_mock: respx.Router) -> Iterator[MockJupyter]: """Mock out JupyterHub and Jupyter labs.""" diff --git a/tests/handlers/github_test.py b/tests/handlers/github_test.py new file mode 100644 index 00000000..acd8e4c8 --- /dev/null +++ b/tests/handlers/github_test.py @@ -0,0 +1,155 @@ +"""Test the github webhook handler.""" + +import hashlib +import hmac +from dataclasses import dataclass +from pathlib import Path +from string import Template + +import pytest +import respx +from httpx import AsyncClient +from pytest_mock import MockerFixture + +import mobu.services.flock + +from ..support.constants import TEST_GITHUB_WEBHOOK_SECRET +from ..support.gafaelfawr import mock_gafaelfawr + + +@dataclass(frozen=True) +class GithubRequest: + payload: str + headers: dict[str, str] + + +def webhook_request(org: str, repo: str, ref: str) -> GithubRequest: + """Build a Github webhook request and headers with the right hash.""" + data_path = Path(__file__).parent.parent / "support" / "data" + template = (data_path / "github_webhook.tmpl.json").read_text() + payload = Template(template).substitute( + org=org, + repo=repo, + ref=ref, + ) + + # https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries#python-example + hash_object = hmac.new( + TEST_GITHUB_WEBHOOK_SECRET.encode("utf-8"), + msg=payload.encode("utf-8"), + digestmod=hashlib.sha256, + ) + sig = "sha256=" + hash_object.hexdigest() + + headers = { + "Accept": "*/*", + "Content-Type": "application/json", + "User-Agent": "GitHub-Hookshot/c9d6c0a", + "X-GitHub-Delivery": "d2d3c948-1d61-11ef-848a-c578f23615c9", + "X-GitHub-Event": "push", + "X-GitHub-Hook-ID": "479971864", + "X-GitHub-Hook-Installation-Target-ID": "804427678", + "X-GitHub-Hook-Installation-Target-Type": "repository", + "X-Hub-Signature-256": sig, + } + + return GithubRequest(payload=payload, headers=headers) + + +@pytest.mark.asyncio +async def test_handle_webhook( + client: AsyncClient, respx_mock: respx.Router, mocker: MockerFixture +) -> None: + configs = [ + { + "name": "test-notebook", + "count": 1, + "user_spec": {"username_prefix": "testuser-notebook"}, + "scopes": ["exec:notebook"], + "business": { + "type": "NotebookRunner", + "options": { + "repo_url": "https://github.com/lsst-sqre/some-repo.git", + "repo_branch": "main", + }, + }, + }, + { + "name": "test-notebook-branch", + "count": 1, + "user_spec": {"username_prefix": "testuser-notebook-branch"}, + "scopes": ["exec:notebook"], + "business": { + "type": "NotebookRunner", + "options": { + "repo_url": "https://github.com/lsst-sqre/some-repo.git", + "repo_branch": "some-branch", + }, + }, + }, + { + "name": "test-other-notebook", + "count": 1, + "user_spec": {"username_prefix": "testuser-other-notebook"}, + "scopes": ["exec:notebook"], + "business": { + "type": "NotebookRunner", + "options": { + "repo_url": "https://github.com/lsst-sqre/some-other-repo.git", + "repo_branch": "main", + }, + }, + }, + { + "name": "test-non-notebook", + "count": 1, + "user_spec": {"username_prefix": "testuser-non-notebook"}, + "scopes": ["exec:notebook"], + "business": {"type": "EmptyLoop"}, + }, + ] + + # Don't actually do any business + mocker.patch.object(mobu.services.flock.Monkey, "start") # type: ignore[attr-defined] + mocker.patch.object(mobu.services.flock.Monkey, "stop") # type: ignore[attr-defined] + + mock_gafaelfawr(respx_mock) + + # Start the flocks + for config in configs: + r = await client.put("/mobu/flocks", json=config) + assert r.status_code == 201 + + # Post a webhook event like GitHub would + request = webhook_request( + org="lsst-sqre", + repo="some-repo", + ref="refs/heads/main", + ) + await client.post( + "/mobu/github/webhook", + headers=request.headers, + content=request.payload, + ) + + # Only the business for the correct branch and repo should be refreshing + r = await client.get( + "/mobu/flocks/test-notebook/monkeys/testuser-notebook1" + ) + assert r.json()["business"]["refreshing"] is True + + # The other businesses should not be refreshing + r = await client.get( + "/mobu/flocks/test-notebook-branch/monkeys/testuser-notebook-branch1" + ) + assert r.json()["business"]["refreshing"] is False + + r = await client.get( + "/mobu/flocks/test-other-notebook/monkeys/testuser-other-notebook1" + ) + assert r.json()["business"]["refreshing"] is False + + r = await client.get( + "/mobu/flocks/test-non-notebook/monkeys/testuser-non-notebook1" + ) + assert r.json()["business"]["refreshing"] is False diff --git a/tests/support/constants.py b/tests/support/constants.py index 69694516..47881223 100644 --- a/tests/support/constants.py +++ b/tests/support/constants.py @@ -2,3 +2,6 @@ TEST_BASE_URL = "https://example.com" """Base URL used for the test `httpx.AsyncClient`.""" + +TEST_GITHUB_WEBHOOK_SECRET = "some-webhook-secret" +"""Webhook secret used for hashing test github webhook payloads.""" diff --git a/tests/support/data/github_webhook.tmpl.json b/tests/support/data/github_webhook.tmpl.json new file mode 100644 index 00000000..8978e531 --- /dev/null +++ b/tests/support/data/github_webhook.tmpl.json @@ -0,0 +1,214 @@ +{ + "ref": "$ref", + "before": "42ec8ee4ffcb3dff4241d59a80aa875784d441aa", + "after": "7917992fb012d1858bfc344034679b2aa5d81868", + "repository": { + "id": 804427678, + "node_id": "R_kgDOL_KXng", + "name": "$repo", + "full_name": "$org/$repo", + "private": false, + "owner": { + "name": "someone", + "email": "someone@somewhere.org", + "login": "some-login", + "id": 10158560, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjEwMTU4NTYw", + "avatar_url": "https://avatars.githubusercontent.com/u/10158560?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/$org", + "html_url": "https://github.com/$org", + "followers_url": "https://api.github.com/users/$org/followers", + "following_url": "https://api.github.com/users/$org/following{/other_user}", + "gists_url": "https://api.github.com/users/$org/gists{/gist_id}", + "starred_url": "https://api.github.com/users/$org/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/$org/subscriptions", + "organizations_url": "https://api.github.com/users/$org/orgs", + "repos_url": "https://api.github.com/users/$org/repos", + "events_url": "https://api.github.com/users/$org/events{/privacy}", + "received_events_url": "https://api.github.com/users/$org/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://github.com/$org/$repo", + "description": "Stuff with which to test mobu", + "fork": false, + "url": "https://github.com/$org/$repo", + "forks_url": "https://api.github.com/repos/$org/$repo/forks", + "keys_url": "https://api.github.com/repos/$org/$repo/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/$org/$repo/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/$org/$repo/teams", + "hooks_url": "https://api.github.com/repos/$org/$repo/hooks", + "issue_events_url": "https://api.github.com/repos/$org/$repo/issues/events{/number}", + "assignees_url": "https://api.github.com/repos/$org/$repo/assignees{/user}", + "events_url": "https://api.github.com/repos/$org/$repo/events", + "branches_url": "https://api.github.com/repos/$org/$repo/branches{/branch}", + "tags_url": "https://api.github.com/repos/$org/$repo/tags", + "blobs_url": "https://api.github.com/repos/$org/$repo/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/$org/$repo/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/$org/$repo/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/$org/$repo/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/$org/$repo/statuses/{sha}", + "languages_url": "https://api.github.com/repos/$org/$repo/languages", + "stargazers_url": "https://api.github.com/repos/$org/$repo/stargazers", + "contributors_url": "https://api.github.com/repos/$org/$repo/contributors", + "subscribers_url": "https://api.github.com/repos/$org/$repo/subscribers", + "subscription_url": "https://api.github.com/repos/$org/$repo/subscription", + "commits_url": "https://api.github.com/repos/$org/$repo/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/$org/$repo/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/$org/$repo/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/$org/$repo/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/$org/$repo/contents/{+path}", + "compare_url": "https://api.github.com/repos/$org/$repo/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/$org/$repo/merges", + "archive_url": "https://api.github.com/repos/$org/$repo/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/$org/$repo/downloads", + "issues_url": "https://api.github.com/repos/$org/$repo/issues{/number}", + "pulls_url": "https://api.github.com/repos/$org/$repo/pulls{/number}", + "milestones_url": "https://api.github.com/repos/$org/$repo/milestones{/number}", + "notifications_url": "https://api.github.com/repos/$org/$repo/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/$org/$repo/labels{/name}", + "releases_url": "https://api.github.com/repos/$org/$repo/releases{/id}", + "deployments_url": "https://api.github.com/repos/$org/$repo/deployments", + "created_at": 1716390526, + "updated_at": "2024-05-28T21:14:02Z", + "pushed_at": 1716949144, + "git_url": "git://github.com/$org/$repo.git", + "ssh_url": "git@github.com:$org/$repo.git", + "clone_url": "https://github.com/$org/$repo.git", + "svn_url": "https://github.com/$org/$repo", + "homepage": null, + "size": 13, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Jupyter Notebook", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "has_discussions": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": null, + "allow_forking": true, + "is_template": false, + "web_commit_signoff_required": false, + "topics": [ + + ], + "visibility": "public", + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "main", + "stargazers": 0, + "master_branch": "main", + "organization": "$org", + "custom_properties": { + + } + }, + "pusher": { + "name": "some-pusher", + "email": "330402+some-pusher@users.noreply.github.com" + }, + "organization": { + "login": "$org", + "id": 10158560, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjEwMTU4NTYw", + "url": "https://api.github.com/orgs/$org", + "repos_url": "https://api.github.com/orgs/$org/repos", + "events_url": "https://api.github.com/orgs/$org/events", + "hooks_url": "https://api.github.com/orgs/$org/hooks", + "issues_url": "https://api.github.com/orgs/$org/issues", + "members_url": "https://api.github.com/orgs/$org/members{/member}", + "public_members_url": "https://api.github.com/orgs/$org/public_members{/member}", + "avatar_url": "https://avatars.githubusercontent.com/u/10158560?v=4", + "description": "Rubin Observatory Science Quality and Reliability Engineering team" + }, + "sender": { + "login": "some-sender", + "id": 330402, + "node_id": "MDQ6VXNlcjMzMDQwMg==", + "avatar_url": "https://avatars.githubusercontent.com/u/330402?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/some-sender", + "html_url": "https://github.com/some-sender", + "followers_url": "https://api.github.com/users/some-sender/followers", + "following_url": "https://api.github.com/users/some-sender/following{/other_user}", + "gists_url": "https://api.github.com/users/some-sender/gists{/gist_id}", + "starred_url": "https://api.github.com/users/some-sender/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/some-sender/subscriptions", + "organizations_url": "https://api.github.com/users/some-sender/orgs", + "repos_url": "https://api.github.com/users/some-sender/repos", + "events_url": "https://api.github.com/users/some-sender/events{/privacy}", + "received_events_url": "https://api.github.com/users/some-sender/received_events", + "type": "User", + "site_admin": false + }, + "created": false, + "deleted": false, + "forced": false, + "base_ref": null, + "compare": "https://github.com/$org/$repo/compare/42ec8ee4ffcb...7917992fb012", + "commits": [ + { + "id": "7917992fb012d1858bfc344034679b2aa5d81868", + "tree_id": "de0a1710ea36ef179af29acb9bda3f6edb31f56c", + "distinct": true, + "message": "refresh", + "timestamp": "2024-05-28T21:19:03-05:00", + "url": "https://github.com/$org/$repo/commit/7917992fb012d1858bfc344034679b2aa5d81868", + "author": { + "name": "Some Body", + "email": "somebody@somewhere.org", + "username": "some-sender" + }, + "committer": { + "name": "Some Body", + "email": "somebody@somewhere.org", + "username": "some-sender" + }, + "added": [ + + ], + "removed": [ + + ], + "modified": [ + "simple_notebook.ipynb" + ] + } + ], + "head_commit": { + "id": "7917992fb012d1858bfc344034679b2aa5d81868", + "tree_id": "de0a1710ea36ef179af29acb9bda3f6edb31f56c", + "distinct": true, + "message": "refresh", + "timestamp": "2024-05-28T21:19:03-05:00", + "url": "https://github.com/$org/$repo/commit/7917992fb012d1858bfc344034679b2aa5d81868", + "author": { + "name": "Some Body", + "email": "somebody@somewhere.org", + "username": "some-sender" + }, + "committer": { + "name": "Some Body", + "email": "somebody@somewhere.org", + "username": "some-sender" + }, + "added": [ + + ], + "removed": [ + + ], + "modified": [ + "simple_notebook.ipynb" + ] + } +}