Sphinx cleanups, bump version requirements, typecheck docs source fil…

…es (#2909)

* add missing type annotations to docs/source/*.py, specify files to check with mypy in pyproject.toml, change sphinx version limits to >5.3, clean up & fix a bunch of references in sphinx, add comment explaining workarounds

check.sh

@@ -57,12 +57,12 @@ echo "::group::Mypy"
 rm -f mypy_annotate.dat
 # Pipefail makes these pipelines fail if mypy does, even if mypy_annotate.py succeeds.
 set -o pipefail
-mypy src/trio --show-error-end --platform linux | python ./src/trio/_tools/mypy_annotate.py --dumpfile mypy_annotate.dat --platform Linux \
+mypy --show-error-end --platform linux | python ./src/trio/_tools/mypy_annotate.py --dumpfile mypy_annotate.dat --platform Linux \
     || { echo "* Mypy (Linux) found type errors." >> "$GITHUB_STEP_SUMMARY"; MYPY=1; }
 # Darwin tests FreeBSD too
-mypy src/trio --show-error-end --platform darwin | python ./src/trio/_tools/mypy_annotate.py --dumpfile mypy_annotate.dat --platform Mac \
+mypy --show-error-end --platform darwin | python ./src/trio/_tools/mypy_annotate.py --dumpfile mypy_annotate.dat --platform Mac \
     || { echo "* Mypy (Mac) found type errors." >> "$GITHUB_STEP_SUMMARY"; MYPY=1; }
-mypy src/trio --show-error-end --platform win32 | python ./src/trio/_tools/mypy_annotate.py --dumpfile mypy_annotate.dat --platform Windows \
+mypy --show-error-end --platform win32 | python ./src/trio/_tools/mypy_annotate.py --dumpfile mypy_annotate.dat --platform Windows \
     || { echo "* Mypy (Windows) found type errors." >> "$GITHUB_STEP_SUMMARY"; MYPY=1; }
 set +o pipefail
 # Re-display errors using Github's syntax, read out of mypy_annotate.dat

docs-requirements.in

@@ -1,5 +1,6 @@
-# RTD is currently installing 1.5.3, which has a bug in :lineno-match:
-sphinx >= 4.0, < 6.2
+# RTD is currently installing 1.5.3, which has a bug in :lineno-match: (??)
+# sphinx 5.3 doesn't work with our _NoValue workaround
+sphinx >= 6.0
 jinja2
 sphinx_rtd_theme
 sphinxcontrib-jquery

docs-requirements.txt

@@ -69,7 +69,7 @@ snowballstemmer==2.2.0
     # via sphinx
 sortedcontainers==2.4.0
     # via -r docs-requirements.in
-sphinx==6.1.3
+sphinx==7.1.2
     # via
     #   -r docs-requirements.in
     #   sphinx-rtd-theme

docs/source/conf.py

@@ -16,8 +16,16 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
+from __future__ import annotations
+
+import collections.abc
 import os
 import sys
+from typing import TYPE_CHECKING, cast
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import Inventory
 
 # For our local_customization module
 sys.path.insert(0, os.path.abspath("."))
@@ -41,6 +49,21 @@
             check=True,
         )
 
+# Sphinx is very finicky, and somewhat buggy, so we have several different
+# methods to help it resolve links.
+# 1. The ones that are not possible to fix are added to `nitpick_ignore`
+# 2. some can be resolved with a simple alias in `autodoc_type_aliases`,
+#    even if that is primarily meant for TypeAliases
+# 3. autodoc_process_signature is hooked up to an event, and we use it for
+#    whole-sale replacing types in signatures where internal details are not
+#    relevant or hard to read.
+# 4. add_intersphinx manually modifies the intersphinx mappings after
+#    objects.inv has been parsed, to resolve bugs and version differences
+#    that causes some objects to be looked up incorrectly.
+# 5. docs/source/typevars.py handles redirecting `typing_extensions` objects to `typing`, and linking `TypeVar`s to `typing.TypeVar` instead of sphinx wanting to link them to their individual definitions.
+# It's possible there's better methods for resolving some of the above
+# problems, but this works for now:tm:
+
 # Warn about all references to unknown targets
 nitpicky = True
 # Except for these ones, which we expect to point to unknown targets:
@@ -51,36 +74,37 @@
     ("py:class", "trio.lowlevel.RunLocal"),
     # trio.abc is documented at random places scattered throughout the docs
     ("py:mod", "trio.abc"),
-    ("py:class", "math.inf"),
     ("py:exc", "Anything else"),
     ("py:class", "async function"),
     ("py:class", "sync function"),
-    # why aren't these found in stdlib?
-    ("py:class", "types.FrameType"),
-    # these are not defined in https://docs.python.org/3/objects.inv
+    # these do not have documentation on python.org
+    # nor entries in objects.inv
     ("py:class", "socket.AddressFamily"),
     ("py:class", "socket.SocketKind"),
-    ("py:class", "Buffer"),  # collections.abc.Buffer, in 3.12
 ]
 autodoc_inherit_docstrings = False
 default_role = "obj"
 
-# These have incorrect __module__ set in stdlib and give the error
-# `py:class reference target not found`
-# Some of the nitpick_ignore's above can probably be fixed with this.
-# See https://github.com/sphinx-doc/sphinx/issues/8315#issuecomment-751335798
+
+# A dictionary for users defined type aliases that maps a type name to the full-qualified object name. It is used to keep type aliases not evaluated in the document.
+# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#confval-autodoc_type_aliases
+# but it can also be used to help resolve various linking problems
 autodoc_type_aliases = {
-    # aliasing doesn't actually fix the warning for types.FrameType, but displaying
-    # "types.FrameType" is more helpful than just "frame"
-    "FrameType": "types.FrameType",
     # SSLListener.accept's return type is seen as trio._ssl.SSLStream
     "SSLStream": "trio.SSLStream",
 }
 
 
+# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#event-autodoc-process-signature
 def autodoc_process_signature(
-    app, what, name, obj, options, signature, return_annotation
-):
+    app: Sphinx,
+    what: object,
+    name: str,
+    obj: object,
+    options: object,
+    signature: str,
+    return_annotation: str,
+) -> tuple[str, str]:
     """Modify found signatures to fix various issues."""
     if signature is not None:
         signature = signature.replace("~_contextvars.Context", "~contextvars.Context")
@@ -103,7 +127,7 @@ def autodoc_process_signature(
 # is shipped (should be in the release after 0.2.4)
 # ...note that this has since grown to contain a bunch of other CSS hacks too
 # though.
-def setup(app):
+def setup(app: Sphinx) -> None:
     app.add_css_file("hackrtd.css")
     app.connect("autodoc-process-signature", autodoc_process_signature)
     # After Intersphinx runs, add additional mappings.
@@ -138,15 +162,49 @@ def setup(app):
 }
 
 
-def add_intersphinx(app) -> None:
-    """Add some specific intersphinx mappings."""
+def add_intersphinx(app: Sphinx) -> None:
+    """Add some specific intersphinx mappings.
+
+    Hooked up to builder-inited. app.builder.env.interpshinx_inventory is not an official API, so this may break on new sphinx versions.
+    """
+
+    def add_mapping(
+        reftype: str,
+        library: str,
+        obj: str,
+        version: str = "3.12",
+        target: str | None = None,
+    ) -> None:
+        """helper function"""
+        url_version = "3" if version == "3.12" else version
+        if target is None:
+            target = f"{library}.{obj}"
+
+        # sphinx doing fancy caching stuff makes this attribute invisible
+        # to type checkers
+        inventory = app.builder.env.intersphinx_inventory  # type: ignore[attr-defined]
+        assert isinstance(inventory, dict)
+        inventory = cast("Inventory", inventory)
+
+        inventory[f"py:{reftype}"][f"{target}"] = (
+            "Python",
+            version,
+            f"https://docs.python.org/{url_version}/library/{library}.html/{obj}",
+            "-",
+        )
+
     # This has been removed in Py3.12, so add a link to the 3.11 version with deprecation warnings.
-    app.builder.env.intersphinx_inventory["py:method"]["pathlib.Path.link_to"] = (
-        "Python",
-        "3.11",
-        "https://docs.python.org/3.11/library/pathlib.html#pathlib.Path.link_to",
-        "-",
-    )
+    add_mapping("method", "pathlib", "Path.link_to", "3.11")
+    # defined in py:data in objects.inv, but sphinx looks for a py:class
+    add_mapping("class", "math", "inf")
+    # `types.FrameType.__module__` is "builtins", so sphinx looks for
+    # builtins.FrameType.
+    # See https://github.com/sphinx-doc/sphinx/issues/11802
+    add_mapping("class", "types", "FrameType")
+    # new in py3.12, and need target because sphinx is unable to look up
+    # the module of the object if compiling on <3.12
+    if not hasattr(collections.abc, "Buffer"):
+        add_mapping("class", "collections.abc", "Buffer", target="Buffer")
 
 
 autodoc_member_order = "bysource"
@@ -193,7 +251,7 @@ def add_intersphinx(app) -> None:
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = []
+exclude_patterns: list[str] = []
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "default"
@@ -252,7 +310,7 @@ def add_intersphinx(app) -> None:
 
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
+latex_elements: dict[str, object] = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',

docs/source/local_customization.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from docutils.parsers.rst import directives as directives
 from sphinx import addnodes
 from sphinx.domains.python import PyClasslike
@@ -8,6 +12,10 @@
     Options as Options,
 )
 
+if TYPE_CHECKING:
+    from sphinx.addnodes import desc_signature
+    from sphinx.application import Sphinx
+
 """
 
 .. interface:: The nursery interface
@@ -18,13 +26,13 @@
 
 
 class Interface(PyClasslike):
-    def handle_signature(self, sig, signode):
+    def handle_signature(self, sig: str, signode: desc_signature) -> tuple[str, str]:
         signode += addnodes.desc_name(sig, sig)
         return sig, ""
 
-    def get_index_text(self, modname, name_cls):
+    def get_index_text(self, modname: str, name_cls: tuple[str, str]) -> str:
         return f"{name_cls[0]} (interface in {modname})"
 
 
-def setup(app):
+def setup(app: Sphinx) -> None:
     app.add_directive_to_domain("py", "interface", Interface)

docs/source/typevars.py

@@ -60,7 +60,7 @@ def lookup_reference(
         new_node["reftarget"] = f"typing.{target[18:]}"
         # This fires off this same event, with our new modified node in order to fetch the right
         # URL to use.
-        return app.emit_firstresult(
+        return app.emit_firstresult(  # type: ignore[no-any-return]
             "missing-reference",
             env,
             new_node,

pyproject.toml

@@ -137,6 +137,7 @@ fixture-parentheses = false
 
 [tool.mypy]
 python_version = "3.8"
+files = ["src/trio/", "docs/source/*.py"]
 
 # Be flexible about dependencies that don't have stubs yet (like pytest)
 ignore_missing_imports = true

test-requirements.in

@@ -22,6 +22,9 @@ codespell
 mypy-extensions; implementation_name == "cpython"
 typing-extensions
 types-cffi; implementation_name == "cpython"
+# annotations in doc files
+types-docutils
+sphinx
 
 # Trio's own dependencies
 cffi; os_name == "nt"