diff --git a/README.md b/README.md index c1ec571..a11b4a3 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,15 @@ of Sphinx we're using where:: 3.10 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 3.11 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 2.3.1 ======== ======= ===== ======= ===== ===== ===== ======= ===== ===== + + +## The github hook server + +`build_docs_server.py` is a simple HTTP server handling Github Webhooks +requests to build the doc when needed. It only needs `push` events. + +Its logging can be configured by giving a yaml file path to the +`--logging-config` argument. + +By default the loglevel is `DEBUG` on `stderr`, the default config can +be found in the code so one can bootstrap a different config from it. diff --git a/build_docs.py b/build_docs.py index bef921a..ffa24b0 100755 --- a/build_docs.py +++ b/build_docs.py @@ -40,6 +40,7 @@ from string import Template from textwrap import indent +import zc.lockfile import jinja2 HERE = Path(__file__).resolve().parent @@ -110,9 +111,7 @@ def title(self): Version("3.7", "3.7", "security-fixes", sphinx_version="2.3.1"), Version("3.8", "3.8", "security-fixes", sphinx_version="2.4.4"), Version("3.9", "3.9", "stable", sphinx_version="2.4.4"), - Version( - "3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"] - ), + Version("3.10", "3.10", "pre-release", sphinx_version="3.2.1", sphinxopts=["-j4"]), Version( "3.11", "main", "in development", sphinx_version="3.2.1", sphinxopts=["-j4"] ), @@ -174,6 +173,7 @@ def run(cmd) -> subprocess.CompletedProcess: stdout=subprocess.PIPE, encoding="utf-8", errors="backslashreplace", + check=False, ) if result.returncode: # Log last 20 lines, those are likely the interesting ones. @@ -372,13 +372,13 @@ def setup_switchers(html_root): script = """ \n""".format( "../" * depth ) - with edit(file) as (i, o): - for line in i: + with edit(file) as (ifile, ofile): + for line in ifile: if line == script: continue if line == " \n": - o.write(script) - o.write(line) + ofile.write(script) + ofile.write(line) def build_one( @@ -750,12 +750,75 @@ def setup_logging(log_directory): logging.getLogger().setLevel(logging.DEBUG) +def build_and_publish( + build_root, + www_root, + version, + language, + quick, + group, + log_directory, + skip_cache_invalidation, + theme, +): + """Build and publish a Python doc, for a language, and a version. + + Also ensures that a single process is doing it by using a `.lock` + file per language / version pair. + """ + try: + lock = zc.lockfile.LockFile( + os.path.join( + HERE, + "{version}-{lang}.lock".format(version=version.name, lang=language.tag), + ) + ) + + try: + venv = build_venv(build_root, version, theme) + build_one( + version, + quick, + venv, + build_root, + group, + log_directory, + language, + ) + copy_build_to_webroot( + build_root, + version, + language, + group, + quick, + skip_cache_invalidation, + www_root, + ) + except Exception as err: + logging.exception( + "Exception while building %s version %s", + language.tag, + version.name, + ) + if sentry_sdk: + sentry_sdk.capture_exception(err) + + except zc.lockfile.LockError: + logging.info( + "Skipping build of %s/%s (build already running)", + language.tag, + version.name, + ) + else: + lock.close() + + def main(): args = parse_args() languages_dict = {language.tag: language for language in LANGUAGES} if args.version: version_info() - exit(0) + sys.exit(0) if args.log_directory: args.log_directory = os.path.abspath(args.log_directory) if args.build_root: @@ -782,34 +845,17 @@ def main(): scope.set_tag("version", version.name) scope.set_tag("language", language_tag) language = languages_dict[language_tag] - try: - venv = build_venv(args.build_root, version, args.theme) - build_one( - version, - args.quick, - venv, - args.build_root, - args.group, - args.log_directory, - language, - ) - copy_build_to_webroot( - args.build_root, - version, - language, - args.group, - args.quick, - args.skip_cache_invalidation, - args.www_root, - ) - except Exception as err: - logging.exception( - "Exception while building %s version %s", - language_tag, - version.name, - ) - if sentry_sdk: - sentry_sdk.capture_exception(err) + build_and_publish( + args.build_root, + args.www_root, + version, + language, + args.quick, + args.group, + args.log_directory, + args.skip_cache_invalidation, + args.theme, + ) build_sitemap(args.www_root) build_robots_txt(args.www_root, args.group, args.skip_cache_invalidation) diff --git a/build_docs_server.py b/build_docs_server.py new file mode 100644 index 0000000..1911a1c --- /dev/null +++ b/build_docs_server.py @@ -0,0 +1,169 @@ +"""Github hook server. + +This is a simple HTTP server handling Github Webhooks requests to +build the doc when needed. + +It needs a GH_SECRET environment variable to be able to receive hooks +on `/hook/github`. + +Its logging can be configured by giving a yaml file path to the +`--logging-config` argument. + +By default the loglevel is `DEBUG` on `stderr`, the default config can +be found in the code so one can bootstrap a different config from it. +""" + +from pathlib import Path +import argparse +import asyncio +import logging.config +import os + +from aiohttp import web +from gidgethub import sansio +import yaml + +from build_docs import VERSIONS + + +__version__ = "0.0.1" + +DEFAULT_LOGGING_CONFIG = """ +--- + +version: 1 +disable_existing_loggers: false +formatters: + normal: + format: '%(asctime)s - %(levelname)s - %(message)s' +handlers: + stderr: + class: logging.StreamHandler + stream: ext://sys.stderr + level: DEBUG + formatter: normal +loggers: + build_docs_server: + level: DEBUG + handlers: [stderr] + aiohttp.access: + level: DEBUG + handlers: [stderr] + aiohttp.client: + level: DEBUG + handlers: [stderr] + aiohttp.internal: + level: DEBUG + handlers: [stderr] + aiohttp.server: + level: DEBUG + handlers: [stderr] + aiohttp.web: + level: DEBUG + handlers: [stderr] + aiohttp.websocket: + level: DEBUG + handlers: [stderr] +""" + +logger = logging.getLogger("build_docs_server") + + +async def version(request): + return web.json_response( + { + "name": "docs.python.org Github handler", + "version": __version__, + "source": "https://github.com/python/docsbuild-scripts", + } + ) + + +async def child_waiter(app): + while True: + try: + status = os.waitid(os.P_ALL, 0, os.WNOHANG | os.WEXITED) + logger.debug("Child completed with status %s", str(status)) + except ChildProcessError: + await asyncio.sleep(600) + + +async def start_child_waiter(app): + app["child_waiter"] = asyncio.ensure_future(child_waiter(app)) + + +async def stop_child_waiter(app): + app["child_waiter"].cancel() + + +async def hook(request): + body = await request.read() + event = sansio.Event.from_http( + request.headers, body, secret=os.environ.get("GH_SECRET") + ) + if event.event != "push": + logger.debug("Received a %s event, nothing to do.", event.event) + return web.Response() + touched_files = ( + set(event.data["head_commit"]["added"]) + | set(event.data["head_commit"]["modified"]) + | set(event.data["head_commit"]["removed"]) + ) + if not any("Doc" in touched_file for touched_file in touched_files): + logger.debug("No documentation file modified, ignoring.") + return web.Response() # Nothing to do + branch = event.data["ref"].split("/")[-1] + known_branches = {version.branch for version in VERSION} + if branch not in known_branches: + logger.warning("Ignoring a change in branch %s (unknown branch)", branch) + return web.Response() # Nothing to do + logger.debug("Forking a build for branch %s", branch) + pid = os.fork() + if pid == 0: + os.execl( + "/usr/bin/env", + "/usr/bin/env", + "python", + "build_docs.py", + "--branch", + branch, + ) + else: + return web.Response() + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--path", help="Unix socket to listen for connections.") + parser.add_argument("--port", help="Local port to listen for connections.") + parser.add_argument( + "--logging-config", + help="yml file containing a Python logging dictconfig, see README.md", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + logging.config.dictConfig( + yaml.load( + Path(args.logging_config).read_text() + if args.logging_config + else DEFAULT_LOGGING_CONFIG, + Loader=yaml.SafeLoader, + ) + ) + app = web.Application() + app.on_startup.append(start_child_waiter) + app.on_cleanup.append(stop_child_waiter) + app.add_routes( + [ + web.get("/", version), + web.post("/hooks/github", hook), + ] + ) + web.run_app(app, path=args.path, port=args.port) + + +if __name__ == "__main__": + main() diff --git a/requirements.in b/requirements.in index b8b6a68..e9d104e 100644 --- a/requirements.in +++ b/requirements.in @@ -1,2 +1,6 @@ -sentry-sdk +aiohttp +gidgethub jinja2 +pyyaml +sentry-sdk +zc.lockfile diff --git a/requirements.txt b/requirements.txt index 19d5e46..2e2b837 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,52 @@ # # pip-compile requirements.in # -certifi==2020.6.20 # via sentry-sdk -jinja2==2.11.3 # via -r requirements.in -markupsafe==1.1.1 # via jinja2 -sentry-sdk==0.15.1 # via -r requirements.in -urllib3==1.25.9 # via sentry-sdk +aiohttp==3.7.3 + # via -r requirements.in +async-timeout==3.0.1 + # via aiohttp +attrs==20.3.0 + # via aiohttp +certifi==2020.6.20 + # via sentry-sdk +cffi==1.14.4 + # via cryptography +chardet==3.0.4 + # via aiohttp +cryptography==3.3.1 + # via pyjwt +gidgethub==4.2.0 + # via -r requirements.in +idna==2.10 + # via yarl +jinja2==2.11.2 + # via -r requirements.in +markupsafe==1.1.1 + # via jinja2 +multidict==5.1.0 + # via + # aiohttp + # yarl +pycparser==2.20 + # via cffi +pyjwt[crypto]==1.7.1 + # via gidgethub +pyyaml==5.3.1 + # via -r requirements.in +sentry-sdk==0.15.1 + # via -r requirements.in +six==1.15.0 + # via cryptography +typing-extensions==3.7.4.3 + # via aiohttp +uritemplate==3.0.1 + # via gidgethub +urllib3==1.25.9 + # via sentry-sdk +yarl==1.6.3 + # via aiohttp +zc.lockfile==2.0 + # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools