Implement a simple github webook server to build the doc.

README.md

@@ -50,3 +50,15 @@ of Sphinx we're using where::
     3.10      2.3.1    2.3.1  2.3.1    2.3.1  2.3.1  2.3.1  2.3.1    2.3.1  2.3.1
     3.11      2.3.1    2.3.1  2.3.1    2.3.1  2.3.1  2.3.1  2.3.1    2.3.1  2.3.1
     ========  =======  =====  =======  =====  =====  =====  =======  =====  =====
+
+
+## The github hook server
+
+`build_docs_server.py` is a simple HTTP server handling Github Webhooks
+requests to build the doc when needed. It only needs `push` events.
+
+Its logging can be configured by giving a yaml file path to the
+`--logging-config` argument.
+
+By default the loglevel is `DEBUG` on `stderr`, the default config can
+be found in the code so one can bootstrap a different config from it.

build_docs.py

@@ -40,6 +40,7 @@
 from string import Template
 from textwrap import indent
 
+import zc.lockfile
 import jinja2
 
 HERE = Path(__file__).resolve().parent
@@ -110,9 +111,7 @@ def title(self):
     Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"),
     Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"),
     Version("3.9", "3.9", "stable", sphinx_version="2.4.4"),
-    Version(
-        "3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"]
-    ),
+    Version("3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"]),
     Version(
         "3.11", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"]
     ),
@@ -174,6 +173,7 @@ def run(cmd) -> subprocess.CompletedProcess:
         stdout=subprocess.PIPE,
         encoding="utf-8",
         errors="backslashreplace",
+        check=False,
     )
     if result.returncode:
         # Log last 20 lines, those are likely the interesting ones.
@@ -372,13 +372,13 @@ def setup_switchers(html_root):
         script = """    <script type="text/javascript" src="{}_static/switchers.js"></script>\n""".format(
             "../" * depth
         )
-        with edit(file) as (i, o):
-            for line in i:
+        with edit(file) as (ifile, ofile):
+            for line in ifile:
                 if line == script:
                     continue
                 if line == "  </body>\n":
-                    o.write(script)
-                o.write(line)
+                    ofile.write(script)
+                ofile.write(line)
 
 
 def build_one(
@@ -750,12 +750,75 @@ def setup_logging(log_directory):
     logging.getLogger().setLevel(logging.DEBUG)
 
 
+def build_and_publish(
+    build_root,
+    www_root,
+    version,
+    language,
+    quick,
+    group,
+    log_directory,
+    skip_cache_invalidation,
+    theme,
+):
+    """Build and publish a Python doc, for a language, and a version.
+
+    Also ensures that a single process is doing it by using a `.lock`
+    file per language / version pair.
+    """
+    try:
+        lock = zc.lockfile.LockFile(
+            os.path.join(
+                HERE,
+                "{version}-{lang}.lock".format(version=version.name, lang=language.tag),
+            )
+        )
+
+        try:
+            venv = build_venv(build_root, version, theme)
+            build_one(
+                version,
+                quick,
+                venv,
+                build_root,
+                group,
+                log_directory,
+                language,
+            )
+            copy_build_to_webroot(
+                build_root,
+                version,
+                language,
+                group,
+                quick,
+                skip_cache_invalidation,
+                www_root,
+            )
+        except Exception as err:
+            logging.exception(
+                "Exception while building %s version %s",
+                language.tag,
+                version.name,
+            )
+            if sentry_sdk:
+                sentry_sdk.capture_exception(err)
+
+    except zc.lockfile.LockError:
+        logging.info(
+            "Skipping build of %s/%s (build already running)",
+            language.tag,
+            version.name,
+        )
+    else:
+        lock.close()
+
+
 def main():
     args = parse_args()
     languages_dict = {language.tag: language for language in LANGUAGES}
     if args.version:
         version_info()
-        exit(0)
+        sys.exit(0)
     if args.log_directory:
         args.log_directory = os.path.abspath(args.log_directory)
     if args.build_root:
@@ -782,34 +845,17 @@ def main():
                     scope.set_tag("version", version.name)
                     scope.set_tag("language", language_tag)
             language = languages_dict[language_tag]
-            try:
-                venv = build_venv(args.build_root, version, args.theme)
-                build_one(
-                    version,
-                    args.quick,
-                    venv,
-                    args.build_root,
-                    args.group,
-                    args.log_directory,
-                    language,
-                )
-                copy_build_to_webroot(
-                    args.build_root,
-                    version,
-                    language,
-                    args.group,
-                    args.quick,
-                    args.skip_cache_invalidation,
-                    args.www_root,
-                )
-            except Exception as err:
-                logging.exception(
-                    "Exception while building %s version %s",
-                    language_tag,
-                    version.name,
-                )
-                if sentry_sdk:
-                    sentry_sdk.capture_exception(err)
+            build_and_publish(
+                args.build_root,
+                args.www_root,
+                version,
+                language,
+                args.quick,
+                args.group,
+                args.log_directory,
+                args.skip_cache_invalidation,
+                args.theme,
+            )
     build_sitemap(args.www_root)
     build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation)
 

build_docs_server.py

@@ -0,0 +1,169 @@
+"""Github hook server.
+
+This is a simple HTTP server handling Github Webhooks requests to
+build the doc when needed.
+
+It needs a GH_SECRET environment variable to be able to receive hooks
+on `/hook/github`.
+
+Its logging can be configured by giving a yaml file path to the
+`--logging-config` argument.
+
+By default the loglevel is `DEBUG` on `stderr`, the default config can
+be found in the code so one can bootstrap a different config from it.
+"""
+
+from pathlib import Path
+import argparse
+import asyncio
+import logging.config
+import os
+
+from aiohttp import web
+from gidgethub import sansio
+import yaml
+
+from build_docs import VERSIONS
+
+
+__version__ = "0.0.1"
+
+DEFAULT_LOGGING_CONFIG = """
+---
+
+version: 1
+disable_existing_loggers: false
+formatters:
+  normal:
+    format: '%(asctime)s - %(levelname)s - %(message)s'
+handlers:
+  stderr:
+    class: logging.StreamHandler
+    stream: ext://sys.stderr
+    level: DEBUG
+    formatter: normal
+loggers:
+  build_docs_server:
+    level: DEBUG
+    handlers: [stderr]
+  aiohttp.access:
+    level: DEBUG
+    handlers: [stderr]
+  aiohttp.client:
+    level: DEBUG
+    handlers: [stderr]
+  aiohttp.internal:
+    level: DEBUG
+    handlers: [stderr]
+  aiohttp.server:
+    level: DEBUG
+    handlers: [stderr]
+  aiohttp.web:
+    level: DEBUG
+    handlers: [stderr]
+  aiohttp.websocket:
+    level: DEBUG
+    handlers: [stderr]
+"""
+
+logger = logging.getLogger("build_docs_server")
+
+
+async def version(request):
+    return web.json_response(
+        {
+            "name": "docs.python.org Github handler",
+            "version": __version__,
+            "source": "https://github.com/python/docsbuild-scripts",
+        }
+    )
+
+
+async def child_waiter(app):
+    while True:
+        try:
+            status = os.waitid(os.P_ALL, 0, os.WNOHANG | os.WEXITED)
+            logger.debug("Child completed with status %s", str(status))
+        except ChildProcessError:
+            await asyncio.sleep(600)
+
+
+async def start_child_waiter(app):
+    app["child_waiter"] = asyncio.ensure_future(child_waiter(app))
+
+
+async def stop_child_waiter(app):
+    app["child_waiter"].cancel()
+
+
+async def hook(request):
+    body = await request.read()
+    event = sansio.Event.from_http(
+        request.headers, body, secret=os.environ.get("GH_SECRET")
+    )
+    if event.event != "push":
+        logger.debug("Received a %s event, nothing to do.", event.event)
+        return web.Response()
+    touched_files = (
+        set(event.data["head_commit"]["added"])
+        | set(event.data["head_commit"]["modified"])
+        | set(event.data["head_commit"]["removed"])
+    )
+    if not any("Doc" in touched_file for touched_file in touched_files):
+        logger.debug("No documentation file modified, ignoring.")
+        return web.Response()  # Nothing to do
+    branch = event.data["ref"].split("/")[-1]
+    known_branches = {version.branch for version in VERSION}
+    if branch not in known_branches:
+        logger.warning("Ignoring a change in branch %s (unknown branch)", branch)
+        return web.Response()  # Nothing to do
+    logger.debug("Forking a build for branch %s", branch)
+    pid = os.fork()
+    if pid == 0:
+        os.execl(
+            "/usr/bin/env",
+            "/usr/bin/env",
+            "python",
+            "build_docs.py",
+            "--branch",
+            branch,
+        )
+    else:
+        return web.Response()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--path", help="Unix socket to listen for connections.")
+    parser.add_argument("--port", help="Local port to listen for connections.")
+    parser.add_argument(
+        "--logging-config",
+        help="yml file containing a Python logging dictconfig, see README.md",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    logging.config.dictConfig(
+        yaml.load(
+            Path(args.logging_config).read_text()
+            if args.logging_config
+            else DEFAULT_LOGGING_CONFIG,
+            Loader=yaml.SafeLoader,
+        )
+    )
+    app = web.Application()
+    app.on_startup.append(start_child_waiter)
+    app.on_cleanup.append(stop_child_waiter)
+    app.add_routes(
+        [
+            web.get("/", version),
+            web.post("/hooks/github", hook),
+        ]
+    )
+    web.run_app(app, path=args.path, port=args.port)
+
+
+if __name__ == "__main__":
+    main()

requirements.in

@@ -1,2 +1,6 @@
-sentry-sdk
+aiohttp
+gidgethub
 jinja2
+pyyaml
+sentry-sdk
+zc.lockfile