From 3794c04d798c39755f36718b650172c1ae7ff6f9 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 17 Feb 2023 13:48:15 +0100 Subject: [PATCH 01/18] nixos/manual: fix manpage links {manpage} already exapnds to a link but akkoma wants to link to a specific setting. split the mention for clarity. networkd just straight up duplicated what {manpage} generates anyway, so that link can go away completely. --- nixos/modules/services/web-apps/akkoma.md | 4 ++-- nixos/modules/system/boot/networkd.nix | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/web-apps/akkoma.md b/nixos/modules/services/web-apps/akkoma.md index 5419940a68d63..83dd1a8b35f23 100644 --- a/nixos/modules/services/web-apps/akkoma.md +++ b/nixos/modules/services/web-apps/akkoma.md @@ -318,8 +318,8 @@ to make packages available in the chroot. {option}`services.systemd.akkoma.serviceConfig.BindPaths` and {option}`services.systemd.akkoma.serviceConfig.BindReadOnlyPaths` permit access to outside paths through bind mounts. Refer to -[{manpage}`systemd.exec(5)`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#BindPaths=) -for details. +[`BindPaths=`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#BindPaths=) +of {manpage}`systemd.exec(5)` for details. ### Distributed deployment {#modules-services-akkoma-distributed-deployment} diff --git a/nixos/modules/system/boot/networkd.nix b/nixos/modules/system/boot/networkd.nix index 188f2f64dc848..d1ce3d13ee855 100644 --- a/nixos/modules/system/boot/networkd.nix +++ b/nixos/modules/system/boot/networkd.nix @@ -1948,7 +1948,7 @@ in Extra command-line arguments to pass to systemd-networkd-wait-online. These also affect per-interface `systemd-network-wait-online@` services. - See [{manpage}`systemd-networkd-wait-online.service(8)`](https://www.freedesktop.org/software/systemd/man/systemd-networkd-wait-online.service.html) for all available options. + See {manpage}`systemd-networkd-wait-online.service(8)` for all available options. ''; type = with types; listOf str; default = []; From 0236dcb59fb7b58f0ba5fee3ff15cc88ba903d61 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 17 Feb 2023 17:49:08 +0100 Subject: [PATCH 02/18] nixos-render-docs: don't use markdown-it RendererProtocol our renderers carry significantly more state than markdown-it wants to easily cater for, and the html renderer will need even more state still. relying on the markdown-it-provided rendering functions has already proven to be a nuisance, and since parsing and rendering are split well enough we can just replace the rendering part with our own stuff outright. this also frees us from the tyranny of having to set instance variables before calling super().__init__ just to make sure that the renderer creation callback has access to everything it needs. --- .../src/nixos_render_docs/asciidoc.py | 5 +- .../src/nixos_render_docs/commonmark.py | 5 +- .../src/nixos_render_docs/docbook.py | 5 +- .../src/nixos_render_docs/manpage.py | 7 +-- .../src/nixos_render_docs/manual.py | 18 +++--- .../src/nixos_render_docs/md.py | 32 +++++++--- .../src/nixos_render_docs/options.py | 60 +++++++++---------- .../src/tests/test_asciidoc.py | 8 ++- .../src/tests/test_commonmark.py | 8 ++- .../src/tests/test_headings.py | 8 ++- .../nixos-render-docs/src/tests/test_lists.py | 8 ++- .../src/tests/test_manpage.py | 23 +++---- .../src/tests/test_plugins.py | 8 ++- 13 files changed, 101 insertions(+), 94 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py index 637185227e83f..2730dc5e024f0 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py @@ -5,7 +5,6 @@ from .md import Renderer -import markdown_it from markdown_it.token import Token from markdown_it.utils import OptionsDict @@ -59,8 +58,8 @@ class AsciiDocRenderer(Renderer): _list_stack: list[List] _attrspans: list[str] - def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): - super().__init__(manpage_urls, parser) + def __init__(self, manpage_urls: Mapping[str, str]): + super().__init__(manpage_urls) self._parstack = [ Par("\n\n", "====") ] self._list_stack = [] self._attrspans = [] diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py index 4a708b1f92c66..8fe32289b85dd 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py @@ -4,7 +4,6 @@ from .md import md_escape, md_make_code, Renderer -import markdown_it from markdown_it.token import Token from markdown_it.utils import OptionsDict @@ -26,8 +25,8 @@ class CommonMarkRenderer(Renderer): _link_stack: list[str] _list_stack: list[List] - def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): - super().__init__(manpage_urls, parser) + def __init__(self, manpage_urls: Mapping[str, str]): + super().__init__(manpage_urls) self._parstack = [ Par("") ] self._link_stack = [] self._list_stack = [] diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py index e6a761dcf13fd..b279ace06ff1c 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py @@ -32,14 +32,13 @@ class Heading(NamedTuple): partintro_closed: bool = False class DocBookRenderer(Renderer): - __output__ = "docbook" _link_tags: list[str] _deflists: list[Deflist] _headings: list[Heading] _attrspans: list[str] - def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): - super().__init__(manpage_urls, parser) + def __init__(self, manpage_urls: Mapping[str, str]): + super().__init__(manpage_urls) self._link_tags = [] self._deflists = [] self._headings = [] diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py index 1b796d9f04861..a61537b49c54e 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py @@ -75,8 +75,6 @@ class List: # horizontal motion in a line) we do attempt to copy the style of mdoc(7) semantic requests # as appropriate for each markup element. class ManpageRenderer(Renderer): - __output__ = "man" - # whether to emit mdoc .Ql equivalents for inline code or just the contents. this is # mainly used by the options manpage converter to not emit extra quotes in defaults # and examples where it's already clear from context that the following text is code. @@ -90,9 +88,8 @@ class ManpageRenderer(Renderer): _list_stack: list[List] _font_stack: list[str] - def __init__(self, manpage_urls: Mapping[str, str], href_targets: dict[str, str], - parser: Optional[markdown_it.MarkdownIt] = None): - super().__init__(manpage_urls, parser) + def __init__(self, manpage_urls: Mapping[str, str], href_targets: dict[str, str]): + super().__init__(manpage_urls) self._href_targets = href_targets self._link_stack = [] self._do_parbreak_stack = [] diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index efc8b02e8d6b2..dfcdb96cc21f4 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -18,9 +18,8 @@ class ManualDocBookRenderer(DocBookRenderer): _toplevel_tag: str - def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str], - parser: Optional[markdown_it.MarkdownIt] = None): - super().__init__(manpage_urls, parser) + def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str]): + super().__init__(manpage_urls) self._toplevel_tag = toplevel_tag self.rules |= { 'included_sections': lambda *args: self._included_thing("section", *args), @@ -92,7 +91,7 @@ def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: in self._headings[-1] = self._headings[-1]._replace(partintro_closed=True) # must nest properly for structural includes. this requires saving at least # the headings stack, but creating new renderers is cheap and much easier. - r = ManualDocBookRenderer(tag, self._manpage_urls, None) + r = ManualDocBookRenderer(tag, self._manpage_urls) for (included, path) in token.meta['included']: try: result.append(r.render(included, options, env)) @@ -118,16 +117,13 @@ def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsD info = f" language={quoteattr(token.info)}" if token.info != "" else "" return f"\n{escape(token.content)}" -class DocBookConverter(Converter): - def __renderer__(self, manpage_urls: Mapping[str, str], - parser: Optional[markdown_it.MarkdownIt]) -> ManualDocBookRenderer: - return ManualDocBookRenderer('book', manpage_urls, parser) - +class DocBookConverter(Converter[ManualDocBookRenderer]): _base_paths: list[Path] _revision: str def __init__(self, manpage_urls: Mapping[str, str], revision: str): - super().__init__(manpage_urls) + super().__init__() + self._renderer = ManualDocBookRenderer('book', manpage_urls) self._revision = revision def convert(self, file: Path) -> str: @@ -195,7 +191,7 @@ def _parse_options(self, token: Token) -> None: try: conv = options.DocBookConverter( - self._manpage_urls, self._revision, False, 'fragment', varlist_id, id_prefix) + self._renderer._manpage_urls, self._revision, False, 'fragment', varlist_id, id_prefix) with open(self._base_paths[-1].parent / source, 'r') as f: conv.add_options(json.load(f)) token.meta['rendered-options'] = conv.finalize(fragment=True) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py index 96cc8af69bce9..d73a1715f4f9b 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py @@ -1,6 +1,6 @@ from abc import ABC from collections.abc import Mapping, MutableMapping, Sequence -from typing import Any, Callable, cast, get_args, Iterable, Literal, NoReturn, Optional +from typing import Any, Callable, cast, Generic, get_args, Iterable, Literal, NoReturn, Optional, TypeVar import dataclasses import re @@ -44,11 +44,11 @@ def md_make_code(code: str, info: str = "", multiline: Optional[bool] = None) -> AdmonitionKind = Literal["note", "caution", "tip", "important", "warning"] -class Renderer(markdown_it.renderer.RendererProtocol): +class Renderer: _admonitions: dict[AdmonitionKind, tuple[RenderFn, RenderFn]] _admonition_stack: list[AdmonitionKind] - def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): + def __init__(self, manpage_urls: Mapping[str, str]): self._manpage_urls = manpage_urls self.rules = { 'text': self.text, @@ -466,12 +466,26 @@ def block_attr(state: markdown_it.rules_core.StateCore) -> None: md.core.ruler.push("block_attr", block_attr) -class Converter(ABC): - __renderer__: Callable[[Mapping[str, str], markdown_it.MarkdownIt], Renderer] +TR = TypeVar('TR', bound='Renderer') - def __init__(self, manpage_urls: Mapping[str, str]): - self._manpage_urls = manpage_urls +class Converter(ABC, Generic[TR]): + # we explicitly disable markdown-it rendering support and use our own entirely. + # rendering is well separated from parsing and our renderers carry much more state than + # markdown-it easily acknowledges as 'good' (unless we used the untyped env args to + # shuttle that state around, which is very fragile) + class ForbiddenRenderer(markdown_it.renderer.RendererProtocol): + __output__ = "none" + + def __init__(self, parser: Optional[markdown_it.MarkdownIt]): + pass + + def render(self, tokens: Sequence[Token], options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + raise NotImplementedError("do not use Converter._md.renderer. 'tis a silly place") + + _renderer: TR + def __init__(self) -> None: self._md = markdown_it.MarkdownIt( "commonmark", { @@ -479,7 +493,7 @@ def __init__(self, manpage_urls: Mapping[str, str]): 'html': False, # not useful since we target many formats 'typographer': True, # required for smartquotes }, - renderer_cls=lambda parser: self.__renderer__(self._manpage_urls, parser) + renderer_cls=self.ForbiddenRenderer ) self._md.use( container_plugin, @@ -502,4 +516,4 @@ def _parse(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> li def _render(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> str: env = {} if env is None else env tokens = self._parse(src, env) - return self._md.renderer.render(tokens, self._md.options, env) # type: ignore[no-any-return] + return self._renderer.render(tokens, self._md.options, env) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index f29d8fdb89682..8f64bd3ed5381 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -7,12 +7,13 @@ from collections.abc import Mapping, MutableMapping, Sequence from markdown_it.utils import OptionsDict from markdown_it.token import Token -from typing import Any, Optional +from typing import Any, Generic, Optional from urllib.parse import quote from xml.sax.saxutils import escape, quoteattr import markdown_it +from . import md from . import parallel from .asciidoc import AsciiDocRenderer, asciidoc_escape from .commonmark import CommonMarkRenderer @@ -30,15 +31,13 @@ def option_is(option: Option, key: str, typ: str) -> Optional[dict[str, str]]: return None return option[key] # type: ignore[return-value] -class BaseConverter(Converter): +class BaseConverter(Converter[md.TR], Generic[md.TR]): __option_block_separator__: str _options: dict[str, RenderedOption] - def __init__(self, manpage_urls: Mapping[str, str], - revision: str, - markdown_by_default: bool): - super().__init__(manpage_urls) + def __init__(self, revision: str, markdown_by_default: bool): + super().__init__() self._options = {} self._revision = revision self._markdown_by_default = markdown_by_default @@ -153,7 +152,7 @@ def _parallel_render_prepare(self) -> Any: raise NotImplementedError() # since it's good enough so far. @classmethod @abstractmethod - def _parallel_render_init_worker(cls, a: Any) -> BaseConverter: raise NotImplementedError() + def _parallel_render_init_worker(cls, a: Any) -> BaseConverter[md.TR]: raise NotImplementedError() def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption: try: @@ -162,7 +161,7 @@ def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption: raise Exception(f"Failed to render option {name}") from e @classmethod - def _parallel_render_step(cls, s: BaseConverter, a: Any) -> RenderedOption: + def _parallel_render_step(cls, s: BaseConverter[md.TR], a: Any) -> RenderedOption: return s._render_option(*a) def add_options(self, options: dict[str, Any]) -> None: @@ -199,8 +198,7 @@ def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, option token.meta['compact'] = False return super().bullet_list_open(token, tokens, i, options, env) -class DocBookConverter(BaseConverter): - __renderer__ = OptionsDocBookRenderer +class DocBookConverter(BaseConverter[OptionsDocBookRenderer]): __option_block_separator__ = "" def __init__(self, manpage_urls: Mapping[str, str], @@ -209,13 +207,14 @@ def __init__(self, manpage_urls: Mapping[str, str], document_type: str, varlist_id: str, id_prefix: str): - super().__init__(manpage_urls, revision, markdown_by_default) + super().__init__(revision, markdown_by_default) + self._renderer = OptionsDocBookRenderer(manpage_urls) self._document_type = document_type self._varlist_id = varlist_id self._id_prefix = id_prefix def _parallel_render_prepare(self) -> Any: - return (self._manpage_urls, self._revision, self._markdown_by_default, self._document_type, + return (self._renderer._manpage_urls, self._revision, self._markdown_by_default, self._document_type, self._varlist_id, self._id_prefix) @classmethod def _parallel_render_init_worker(cls, a: Any) -> DocBookConverter: @@ -300,11 +299,7 @@ def finalize(self, *, fragment: bool = False) -> str: class OptionsManpageRenderer(OptionDocsRestrictions, ManpageRenderer): pass -class ManpageConverter(BaseConverter): - def __renderer__(self, manpage_urls: Mapping[str, str], - parser: Optional[markdown_it.MarkdownIt] = None) -> OptionsManpageRenderer: - return OptionsManpageRenderer(manpage_urls, self._options_by_id, parser) - +class ManpageConverter(BaseConverter[OptionsManpageRenderer]): __option_block_separator__ = ".sp" _options_by_id: dict[str, str] @@ -314,8 +309,9 @@ def __init__(self, revision: str, markdown_by_default: bool, *, # only for parallel rendering _options_by_id: Optional[dict[str, str]] = None): + super().__init__(revision, markdown_by_default) self._options_by_id = _options_by_id or {} - super().__init__({}, revision, markdown_by_default) + self._renderer = OptionsManpageRenderer({}, self._options_by_id) def _parallel_render_prepare(self) -> Any: return ((self._revision, self._markdown_by_default), { '_options_by_id': self._options_by_id }) @@ -324,10 +320,9 @@ def _parallel_render_init_worker(cls, a: Any) -> ManpageConverter: return cls(*a[0], **a[1]) def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption: - assert isinstance(self._md.renderer, OptionsManpageRenderer) - links = self._md.renderer.link_footnotes = [] + links = self._renderer.link_footnotes = [] result = super()._render_option(name, option) - self._md.renderer.link_footnotes = None + self._renderer.link_footnotes = None return result._replace(links=links) def add_options(self, options: dict[str, Any]) -> None: @@ -339,12 +334,11 @@ def _render_code(self, option: dict[str, Any], key: str) -> list[str]: if lit := option_is(option, key, 'literalDocBook'): raise RuntimeError("can't render manpages in the presence of docbook") else: - assert isinstance(self._md.renderer, OptionsManpageRenderer) try: - self._md.renderer.inline_code_is_quoted = False + self._renderer.inline_code_is_quoted = False return super()._render_code(option, key) finally: - self._md.renderer.inline_code_is_quoted = True + self._renderer.inline_code_is_quoted = True def _render_description(self, desc: str | dict[str, Any]) -> list[str]: if isinstance(desc, str) and not self._markdown_by_default: @@ -428,12 +422,15 @@ def finalize(self) -> str: class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer): pass -class CommonMarkConverter(BaseConverter): - __renderer__ = OptionsCommonMarkRenderer +class CommonMarkConverter(BaseConverter[OptionsCommonMarkRenderer]): __option_block_separator__ = "" + def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool): + super().__init__(revision, markdown_by_default) + self._renderer = OptionsCommonMarkRenderer(manpage_urls) + def _parallel_render_prepare(self) -> Any: - return (self._manpage_urls, self._revision, self._markdown_by_default) + return (self._renderer._manpage_urls, self._revision, self._markdown_by_default) @classmethod def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter: return cls(*a) @@ -481,12 +478,15 @@ def finalize(self) -> str: class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer): pass -class AsciiDocConverter(BaseConverter): - __renderer__ = AsciiDocRenderer +class AsciiDocConverter(BaseConverter[OptionsAsciiDocRenderer]): __option_block_separator__ = "" + def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool): + super().__init__(revision, markdown_by_default) + self._renderer = OptionsAsciiDocRenderer(manpage_urls) + def _parallel_render_prepare(self) -> Any: - return (self._manpage_urls, self._revision, self._markdown_by_default) + return (self._renderer._manpage_urls, self._revision, self._markdown_by_default) @classmethod def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter: return cls(*a) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py index 487506469954b..3cf5b208f3923 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py @@ -1,9 +1,11 @@ -import nixos_render_docs +import nixos_render_docs as nrd from sample_md import sample1 -class Converter(nixos_render_docs.md.Converter): - __renderer__ = nixos_render_docs.asciidoc.AsciiDocRenderer +class Converter(nrd.md.Converter[nrd.asciidoc.AsciiDocRenderer]): + def __init__(self, manpage_urls: dict[str, str]): + super().__init__() + self._renderer = nrd.asciidoc.AsciiDocRenderer(manpage_urls) def test_lists() -> None: c = Converter({}) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py index 5e0d63eb6723d..72700d3dbab38 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py @@ -1,4 +1,4 @@ -import nixos_render_docs +import nixos_render_docs as nrd from sample_md import sample1 @@ -6,8 +6,10 @@ import markdown_it -class Converter(nixos_render_docs.md.Converter): - __renderer__ = nixos_render_docs.commonmark.CommonMarkRenderer +class Converter(nrd.md.Converter[nrd.commonmark.CommonMarkRenderer]): + def __init__(self, manpage_urls: Mapping[str, str]): + super().__init__() + self._renderer = nrd.commonmark.CommonMarkRenderer(manpage_urls) # NOTE: in these tests we represent trailing spaces by ` ` and replace them with real space later, # since a number of editors will strip trailing whitespace on save and that would break the tests. diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_headings.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_headings.py index 0b73cdc8e7c72..8cbf3dabcea24 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_headings.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_headings.py @@ -1,10 +1,12 @@ -import nixos_render_docs +import nixos_render_docs as nrd from markdown_it.token import Token -class Converter(nixos_render_docs.md.Converter): +class Converter(nrd.md.Converter[nrd.docbook.DocBookRenderer]): # actual renderer doesn't matter, we're just parsing. - __renderer__ = nixos_render_docs.docbook.DocBookRenderer + def __init__(self, manpage_urls: dict[str, str]) -> None: + super().__init__() + self._renderer = nrd.docbook.DocBookRenderer(manpage_urls) def test_heading_id_absent() -> None: c = Converter({}) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_lists.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_lists.py index 660c410a85cca..f53442a96d4cf 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_lists.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_lists.py @@ -1,11 +1,13 @@ -import nixos_render_docs +import nixos_render_docs as nrd import pytest from markdown_it.token import Token -class Converter(nixos_render_docs.md.Converter): +class Converter(nrd.md.Converter[nrd.docbook.DocBookRenderer]): # actual renderer doesn't matter, we're just parsing. - __renderer__ = nixos_render_docs.docbook.DocBookRenderer + def __init__(self, manpage_urls: dict[str, str]) -> None: + super().__init__() + self._renderer = nrd.docbook.DocBookRenderer(manpage_urls) @pytest.mark.parametrize("ordered", [True, False]) def test_list_wide(ordered: bool) -> None: diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_manpage.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_manpage.py index fbfd21358a857..9b7e1652f0f66 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_manpage.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_manpage.py @@ -1,4 +1,4 @@ -import nixos_render_docs +import nixos_render_docs as nrd from sample_md import sample1 @@ -6,15 +6,10 @@ import markdown_it -class Converter(nixos_render_docs.md.Converter): - def __renderer__(self, manpage_urls: Mapping[str, str], - parser: Optional[markdown_it.MarkdownIt] = None - ) -> nixos_render_docs.manpage.ManpageRenderer: - return nixos_render_docs.manpage.ManpageRenderer(manpage_urls, self.options_by_id, parser) - +class Converter(nrd.md.Converter[nrd.manpage.ManpageRenderer]): def __init__(self, manpage_urls: Mapping[str, str], options_by_id: dict[str, str] = {}): - self.options_by_id = options_by_id - super().__init__(manpage_urls) + super().__init__() + self._renderer = nrd.manpage.ManpageRenderer(manpage_urls, options_by_id) def test_inline_code() -> None: c = Converter({}) @@ -32,17 +27,15 @@ def test_expand_link_targets() -> None: def test_collect_links() -> None: c = Converter({}, { '#foo': "bar" }) - assert isinstance(c._md.renderer, nixos_render_docs.manpage.ManpageRenderer) - c._md.renderer.link_footnotes = [] + c._renderer.link_footnotes = [] assert c._render("[a](link1) [b](link2)") == "\\fBa\\fR[1]\\fR \\fBb\\fR[2]\\fR" - assert c._md.renderer.link_footnotes == ['link1', 'link2'] + assert c._renderer.link_footnotes == ['link1', 'link2'] def test_dedup_links() -> None: c = Converter({}, { '#foo': "bar" }) - assert isinstance(c._md.renderer, nixos_render_docs.manpage.ManpageRenderer) - c._md.renderer.link_footnotes = [] + c._renderer.link_footnotes = [] assert c._render("[a](link) [b](link)") == "\\fBa\\fR[1]\\fR \\fBb\\fR[1]\\fR" - assert c._md.renderer.link_footnotes == ['link'] + assert c._renderer.link_footnotes == ['link'] def test_full() -> None: c = Converter({ 'man(1)': 'http://example.org' }) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_plugins.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_plugins.py index 1d836a916d964..f94ede6382bf0 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/tests/test_plugins.py +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_plugins.py @@ -1,10 +1,12 @@ -import nixos_render_docs +import nixos_render_docs as nrd from markdown_it.token import Token -class Converter(nixos_render_docs.md.Converter): +class Converter(nrd.md.Converter[nrd.docbook.DocBookRenderer]): # actual renderer doesn't matter, we're just parsing. - __renderer__ = nixos_render_docs.docbook.DocBookRenderer + def __init__(self, manpage_urls: dict[str, str]) -> None: + super().__init__() + self._renderer = nrd.docbook.DocBookRenderer(manpage_urls) def test_attr_span_parsing() -> None: c = Converter({}) From 6f253fc70b5f21fde4a61f873650778478cdcc30 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 17 Feb 2023 21:29:22 +0100 Subject: [PATCH 03/18] nixos-render-docs: drop options, env parameters these weren't used for anything. options never was (and does not contain any information for the renderer that we *want* to honor), and env is not used because typed renderer state is much more useful for all our cases. --- .../src/nixos_render_docs/asciidoc.py | 139 +++++---------- .../src/nixos_render_docs/commonmark.py | 134 +++++--------- .../src/nixos_render_docs/docbook.py | 161 ++++++----------- .../src/nixos_render_docs/manpage.py | 141 +++++---------- .../src/nixos_render_docs/manual.py | 53 +++--- .../src/nixos_render_docs/md.py | 168 ++++++------------ .../src/nixos_render_docs/options.py | 25 +-- .../src/nixos_render_docs/types.py | 5 +- 8 files changed, 288 insertions(+), 538 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py index 2730dc5e024f0..7fc14c1631ef0 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any, cast, Optional from urllib.parse import quote @@ -6,7 +6,6 @@ from .md import Renderer from markdown_it.token import Token -from markdown_it.utils import OptionsDict _asciidoc_escapes = { # escape all dots, just in case one is pasted at SOL @@ -95,142 +94,103 @@ def _list_close(self) -> str: self._list_stack.pop() return "" - def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def text(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return asciidoc_escape(token.content) - def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._break() - def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return " +\n" - def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def softbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f" " - def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_inline(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return f"``{asciidoc_escape(token.content)}``" - def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - return self.fence(token, tokens, i, options, env) - def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return self.fence(token, tokens, i) + def link_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return f"link:{quote(cast(str, token.attrs['href']), safe='/:')}[" - def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "]" - def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._enter_block(True) # allow the next token to be a block or an inline. return f'\n{self._list_stack[-1].head} {{empty}}' - def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return "\n" - def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._list_open(token, '*') - def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._list_close() - def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "__" - def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "__" - def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "**" - def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "**" - def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: attrs = f"[source,{token.info}]\n" if token.info else "" code = token.content if code.endswith('\n'): code = code[:-1] return f"{self._break(True)}{attrs}----\n{code}\n----" - def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: pbreak = self._break(True) self._enter_block(False) return f"{pbreak}[quote]\n{self._parstack[-2].block_delim}\n" - def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return f"\n{self._parstack[-1].block_delim}" - def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("NOTE") - def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("CAUTION") - def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("IMPORTANT") - def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("TIP") - def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("WARNING") - def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"{self._break()}[]" - def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._break() - def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._enter_block(True) return ":: {empty}" - def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return "\n" - def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def myst_role(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True content = asciidoc_escape(token.content) if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)): return f"link:{quote(url, safe='/:')}[{content}]" return f"[.{token.meta['name']}]``{asciidoc_escape(token.content)}``" - def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return f"[[{token.attrs['id']}]]" - def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True (id_part, class_part) = ("", "") if id := token.attrs.get('id'): @@ -240,22 +200,17 @@ def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options class_part = "kbd:[" self._attrspans.append("]") else: - return super().attr_span_begin(token, tokens, i, options, env) + return super().attr_span_begin(token, tokens, i) else: self._attrspans.append("") return id_part + class_part - def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._attrspans.pop() - def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return token.markup.replace("#", "=") + " " - def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "\n" - def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._list_open(token, '.') - def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._list_close() diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py index 8fe32289b85dd..9649eb653d444 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py @@ -1,11 +1,10 @@ -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any, cast, Optional from .md import md_escape, md_make_code, Renderer from markdown_it.token import Token -from markdown_it.utils import OptionsDict @dataclass(kw_only=True) class List: @@ -57,39 +56,29 @@ def _indent_raw(self, s: str) -> str: return s return f"\n{self._parstack[-1].indent}".join(s.splitlines()) - def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def text(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return self._indent_raw(md_escape(token.content)) - def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._maybe_parbreak() - def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f" {self._break()}" - def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def softbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._break() - def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_inline(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return md_make_code(token.content) - def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - return self.fence(token, tokens, i, options, env) - def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return self.fence(token, tokens, i) + def link_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True self._link_stack.append(cast(str, token.attrs['href'])) return "[" - def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"]({md_escape(self._link_stack.pop())})" - def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: lst = self._list_stack[-1] lbreak = "" if not lst.first_item_seen else self._break() * (1 if lst.compact else 2) lst.first_item_seen = True @@ -99,132 +88,99 @@ def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: lst.next_idx += 1 self._enter_block(" " * (len(head) + 1)) return f'{lbreak}{head} ' - def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return "" - def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.append(List(compact=bool(token.meta['compact']))) return self._maybe_parbreak() - def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.pop() return "" - def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "*" - def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "*" - def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "**" - def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "**" - def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: code = token.content if code.endswith('\n'): code = code[:-1] pbreak = self._maybe_parbreak() return pbreak + self._indent_raw(md_make_code(code, info=token.info, multiline=True)) - def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: pbreak = self._maybe_parbreak() self._enter_block("> ") return pbreak + "> " - def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return "" - def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("Note") - def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("Caution") - def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("Important") - def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("Tip") - def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("Warning") - def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.append(List(compact=False)) return "" - def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.pop() return "" - def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: pbreak = self._maybe_parbreak() self._enter_block(" ") # add an opening zero-width non-joiner to separate *our* emphasis from possible # emphasis in the provided term return f'{pbreak} - *{chr(0x200C)}' - def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"{chr(0x200C)}*" - def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True return "" - def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return "" - def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def myst_role(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._parstack[-1].continuing = True content = md_make_code(token.content) if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)): return f"[{content}]({url})" return content # no roles in regular commonmark - def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: # there's no way we can emit attrspans correctly in all cases. we could use inline # html for ids, but that would not round-trip. same holds for classes. since this # renderer is only used for approximate options export and all of these things are # not allowed in options we can ignore them for now. return "" - def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return token.markup + " " - def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "\n" - def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.append( List(next_idx = cast(int, token.attrs.get('start', 1)), compact = bool(token.meta['compact']))) return self._maybe_parbreak() - def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.pop() return "" diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py index b279ace06ff1c..4c90606ff4558 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py @@ -1,9 +1,8 @@ -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Mapping, Sequence from typing import Any, cast, Optional, NamedTuple import markdown_it from markdown_it.token import Token -from markdown_it.utils import OptionsDict from xml.sax.saxutils import escape, quoteattr from .md import Renderer @@ -44,13 +43,11 @@ def __init__(self, manpage_urls: Mapping[str, str]): self._headings = [] self._attrspans = [] - def render(self, tokens: Sequence[Token], options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - result = super().render(tokens, options, env) - result += self._close_headings(None, env) + def render(self, tokens: Sequence[Token]) -> str: + result = super().render(tokens) + result += self._close_headings(None) return result - def renderInline(self, tokens: Sequence[Token], options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def renderInline(self, tokens: Sequence[Token]) -> str: # HACK to support docbook links and xrefs. link handling is only necessary because the docbook # manpage stylesheet converts - in urls to a mathematical minus, which may be somewhat incorrect. for i, token in enumerate(tokens): @@ -64,135 +61,98 @@ def renderInline(self, tokens: Sequence[Token], options: OptionsDict, if tokens[i + 1].type == 'text' and tokens[i + 1].content == token.attrs['href']: tokens[i + 1].content = '' - return super().renderInline(tokens, options, env) + return super().renderInline(tokens) - def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def text(self, token: Token, tokens: Sequence[Token], i: int) -> str: return escape(token.content) - def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "\n" - def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def softbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: # should check options.breaks() and emit hard break if so return "\n" - def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_inline(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"{escape(token.content)}" - def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"{escape(token.content)}" - def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._link_tags.append(token.tag) href = cast(str, token.attrs['href']) (attr, start) = ('linkend', 1) if href[0] == '#' else ('xlink:href', 0) return f"<{token.tag} {attr}={quoteattr(href[start:])}>" - def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"" - def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "\n" # HACK open and close para for docbook change size. remove soon. - def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: spacing = ' spacing="compact"' if token.meta.get('compact', False) else '' return f"\n" - def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "\n" - def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: info = f" language={quoteattr(token.info)}" if token.info != "" else "" return f"{escape(token.content)}" - def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "
" - def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "
" - def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" # markdown-it emits tokens based on the html syntax tree, but docbook is # slightly different. html has
{
{
}}
, # docbook has {} # we have to reject multiple definitions for the same term for time being. - def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._deflists.append(Deflist()) return "" - def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._deflists.pop() return "" - def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._deflists[-1].has_dd = False return "" - def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: if self._deflists[-1].has_dd: raise Exception("multiple definitions per term not supported") self._deflists[-1].has_dd = True return "" - def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def myst_role(self, token: Token, tokens: Sequence[Token], i: int) -> str: if token.meta['name'] == 'command': return f"{escape(token.content)}" if token.meta['name'] == 'file': @@ -215,8 +175,7 @@ def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: Opti else: return ref raise NotImplementedError("md node not supported yet", token) - def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: # we currently support *only* inline anchors and the special .keycap class to produce # docbook elements. (id_part, class_part) = ("", "") @@ -227,31 +186,26 @@ def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options class_part = "" self._attrspans.append("") else: - return super().attr_span_begin(token, tokens, i, options, env) + return super().attr_span_begin(token, tokens, i) else: self._attrspans.append("") return id_part + class_part - def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._attrspans.pop() - def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: start = f' startingnumber="{token.attrs["start"]}"' if 'start' in token.attrs else "" spacing = ' spacing="compact"' if token.meta.get('compact', False) else '' return f"" - def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"" - def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: hlevel = int(token.tag[1:]) - result = self._close_headings(hlevel, env) - (tag, attrs) = self._heading_tag(token, tokens, i, options, env) + result = self._close_headings(hlevel) + (tag, attrs) = self._heading_tag(token, tokens, i) self._headings.append(Heading(tag, hlevel)) attrs_str = "".join([ f" {k}={quoteattr(v)}" for k, v in attrs.items() ]) return result + f'<{tag}{attrs_str}>\n' - def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: heading = self._headings[-1] result = '' if heading.container_tag == 'part': @@ -263,16 +217,14 @@ def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: maybe_id = " xml:id=" + quoteattr(id + "-intro") result += f"" return result - def example_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def example_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: if id := token.attrs.get('id'): return f"" return "" - def example_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def example_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def _close_headings(self, level: Optional[int], env: MutableMapping[str, Any]) -> str: + def _close_headings(self, level: Optional[int]) -> str: # we rely on markdown-it producing h{1..6} tags in token.tag for this to work result = [] while len(self._headings): @@ -285,8 +237,7 @@ def _close_headings(self, level: Optional[int], env: MutableMapping[str, Any]) - break return "\n".join(result) - def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> tuple[str, dict[str, str]]: + def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int) -> tuple[str, dict[str, str]]: attrs = {} if id := token.attrs.get('id'): attrs['xml:id'] = cast(str, id) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py index a61537b49c54e..a01aa1b4634b4 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any, cast, Iterable, Optional @@ -6,7 +6,6 @@ import markdown_it from markdown_it.token import Token -from markdown_it.utils import OptionsDict from .md import Renderer @@ -123,36 +122,27 @@ def _admonition_close(self) -> str: self._leave_block() return ".RE" - def render(self, tokens: Sequence[Token], options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def render(self, tokens: Sequence[Token]) -> str: self._do_parbreak_stack = [ False ] self._font_stack = [ "\\fR" ] - return super().render(tokens, options, env) + return super().render(tokens) - def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def text(self, token: Token, tokens: Sequence[Token], i: int) -> str: return man_escape(token.content) - def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._maybe_parbreak() - def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return ".br" - def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def softbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: return " " - def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_inline(self, token: Token, tokens: Sequence[Token], i: int) -> str: s = _protect_spaces(man_escape(token.content)) return f"\\fR\\(oq{s}\\(cq\\fP" if self.inline_code_is_quoted else s - def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - return self.fence(token, tokens, i, options, env) - def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return self.fence(token, tokens, i) + def link_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: href = cast(str, token.attrs['href']) self._link_stack.append(href) text = "" @@ -161,8 +151,7 @@ def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: Opti text = self._href_targets[href] self._font_stack.append("\\fB") return f"\\fB{text}\0 <" - def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: href = self._link_stack.pop() text = "" if self.link_footnotes is not None: @@ -174,8 +163,7 @@ def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: Opt text = "\\fR" + man_escape(f"[{idx}]") self._font_stack.pop() return f">\0 {text}{self._font_stack[-1]}" - def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._enter_block() lst = self._list_stack[-1] maybe_space = '' if lst.compact or not lst.first_item_seen else '.sp\n' @@ -189,36 +177,28 @@ def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: f'.RS {lst.width}\n' f"\\h'-{len(head) + 1}'\\fB{man_escape(head)}\\fP\\h'1'\\c" ) - def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return ".RE" - def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.append(List(width=4, compact=bool(token.meta['compact']))) return self._maybe_parbreak() - def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.pop() return "" - def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._font_stack.append("\\fI") return "\\fI" - def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._font_stack.pop() return self._font_stack[-1] - def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._font_stack.append("\\fB") return "\\fB" - def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._font_stack.pop() return self._font_stack[-1] - def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: s = man_escape(token.content).rstrip('\n') return ( '.sp\n' @@ -228,8 +208,7 @@ def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsD '.fi\n' '.RE' ) - def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: maybe_par = self._maybe_parbreak("\n") self._enter_block() return ( @@ -237,62 +216,44 @@ def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options ".RS 4\n" f"\\h'-3'\\fI\\(lq\\(rq\\fP\\h'1'\\c" ) - def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return ".RE" - def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open("Note") - def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open( "Caution") - def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open( "Important") - def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open( "Tip") - def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_open( "Warning") - def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return self._admonition_close() - def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return ".RS 4" - def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return ".RE" - def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: return ".PP" - def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._enter_block() return ".RS 4" - def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._leave_block() return ".RE" - def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def myst_role(self, token: Token, tokens: Sequence[Token], i: int) -> str: if token.meta['name'] in [ 'command', 'env', 'option' ]: return f'\\fB{man_escape(token.content)}\\fP' elif token.meta['name'] in [ 'file', 'var' ]: @@ -303,23 +264,18 @@ def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: Opti return f'\\fB{man_escape(page)}\\fP\\fR({man_escape(section)})\\fP' else: raise NotImplementedError("md node not supported yet", token) - def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: # mdoc knows no anchors so we can drop those, but classes must be rejected. if 'class' in token.attrs: - return super().attr_span_begin(token, tokens, i, options, env) + return super().attr_span_begin(token, tokens, i) return "" - def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "" - def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported in manpages", token) - def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported in manpages", token) - def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: # max item head width for a number, a dot, and one leading space and one trailing space width = 3 + len(str(cast(int, token.meta['end']))) self._list_stack.append( @@ -327,7 +283,6 @@ def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, optio next_idx = cast(int, token.attrs.get('start', 1)), compact = bool(token.meta['compact']))) return self._maybe_parbreak() - def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: self._list_stack.pop() return "" diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index dfcdb96cc21f4..7ac82958a5da3 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -2,14 +2,13 @@ import json from abc import abstractmethod -from collections.abc import Mapping, MutableMapping, Sequence +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any, cast, NamedTuple, Optional, Union from xml.sax.saxutils import escape, quoteattr import markdown_it from markdown_it.token import Token -from markdown_it.utils import OptionsDict from . import options from .docbook import DocBookRenderer, Heading @@ -30,8 +29,7 @@ def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str]): 'included_options': self.included_options, } - def render(self, tokens: Sequence[Token], options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def render(self, tokens: Sequence[Token]) -> str: wanted = { 'h1': 'title' } wanted |= { 'h2': 'subtitle' } if self._toplevel_tag == 'book' else {} for (i, (tag, kind)) in enumerate(wanted.items()): @@ -62,16 +60,15 @@ def render(self, tokens: Sequence[Token], options: OptionsDict, return (f'' - f' {self.renderInline(tokens[1].children, options, env)}' - f' {self.renderInline(tokens[4].children, options, env)}' - f' {super().render(tokens[6:], options, env)}' + f' {self.renderInline(tokens[1].children)}' + f' {self.renderInline(tokens[4].children)}' + f' {super().render(tokens[6:])}' f'') - return super().render(tokens, options, env) + return super().render(tokens) - def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> tuple[str, dict[str, str]]: - (tag, attrs) = super()._heading_tag(token, tokens, i, options, env) + def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int) -> tuple[str, dict[str, str]]: + (tag, attrs) = super()._heading_tag(token, tokens, i) # render() has already verified that we don't have supernumerary headings and since the # book tag is handled specially we can leave the check this simple if token.tag != 'h1': @@ -81,8 +78,7 @@ def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int, options: O 'xmlns:xlink': "http://www.w3.org/1999/xlink", }) - def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: int, - options: OptionsDict, env: MutableMapping[str, Any]) -> str: + def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: int) -> str: result = [] # close existing partintro. the generic render doesn't really need this because # it doesn't have a concept of structure in the way the manual does. @@ -94,26 +90,21 @@ def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: in r = ManualDocBookRenderer(tag, self._manpage_urls) for (included, path) in token.meta['included']: try: - result.append(r.render(included, options, env)) + result.append(r.render(included)) except Exception as e: raise RuntimeError(f"rendering {path}") from e return "".join(result) - def included_options(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str: return cast(str, token.meta['rendered-options']) # TODO minimize docbook diffs with existing conversions. remove soon. - def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - return super().paragraph_open(token, tokens, i, options, env) + "\n " - def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - return "\n" + super().paragraph_close(token, tokens, i, options, env) - def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return super().paragraph_open(token, tokens, i) + "\n " + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "\n" + super().paragraph_close(token, tokens, i) + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"\n{escape(token.content)}" - def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: info = f" language={quoteattr(token.info)}" if token.info != "" else "" return f"\n{escape(token.content)}" @@ -134,8 +125,8 @@ def convert(self, file: Path) -> str: except Exception as e: raise RuntimeError(f"failed to render manual {file}") from e - def _parse(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> list[Token]: - tokens = super()._parse(src, env) + def _parse(self, src: str) -> list[Token]: + tokens = super()._parse(src) for token in tokens: if token.type != "fence" or not token.info.startswith("{=include=} "): continue @@ -145,12 +136,12 @@ def _parse(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> li self._parse_options(token) elif typ in [ 'sections', 'chapters', 'preface', 'parts', 'appendix' ]: token.type = 'included_' + typ - self._parse_included_blocks(token, env) + self._parse_included_blocks(token) else: raise RuntimeError(f"unsupported structural include type '{typ}'") return tokens - def _parse_included_blocks(self, token: Token, env: Optional[MutableMapping[str, Any]]) -> None: + def _parse_included_blocks(self, token: Token) -> None: assert token.map included = token.meta['included'] = [] for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): @@ -161,7 +152,7 @@ def _parse_included_blocks(self, token: Token, env: Optional[MutableMapping[str, try: self._base_paths.append(path) with open(path, 'r') as f: - tokens = self._parse(f.read(), env) + tokens = self._parse(f.read()) included.append((tokens, path)) self._base_paths.pop() except Exception as e: diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py index d73a1715f4f9b..e8fee1b713282 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py @@ -104,169 +104,120 @@ def _join_block(self, ls: Iterable[str]) -> str: def _join_inline(self, ls: Iterable[str]) -> str: return "".join(ls) - def admonition_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def admonition_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: tag = token.meta['kind'] self._admonition_stack.append(tag) - return self._admonitions[tag][0](token, tokens, i, options, env) - def admonition_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: - return self._admonitions[self._admonition_stack.pop()][1](token, tokens, i, options, env) + return self._admonitions[tag][0](token, tokens, i) + def admonition_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return self._admonitions[self._admonition_stack.pop()][1](token, tokens, i) - def render(self, tokens: Sequence[Token], options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def render(self, tokens: Sequence[Token]) -> str: def do_one(i: int, token: Token) -> str: if token.type == "inline": assert token.children is not None - return self.renderInline(token.children, options, env) + return self.renderInline(token.children) elif token.type in self.rules: - return self.rules[token.type](tokens[i], tokens, i, options, env) + return self.rules[token.type](tokens[i], tokens, i) else: raise NotImplementedError("md token not supported yet", token) return self._join_block(map(lambda arg: do_one(*arg), enumerate(tokens))) - def renderInline(self, tokens: Sequence[Token], options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def renderInline(self, tokens: Sequence[Token]) -> str: def do_one(i: int, token: Token) -> str: if token.type in self.rules: - return self.rules[token.type](tokens[i], tokens, i, options, env) + return self.rules[token.type](tokens[i], tokens, i) else: raise NotImplementedError("md token not supported yet", token) return self._join_inline(map(lambda arg: do_one(*arg), enumerate(tokens))) - def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def text(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def softbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_inline(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def link_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def em_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def strong_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def note_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def caution_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def important_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def tip_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def warning_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dl_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dt_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def dd_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def myst_role(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def example_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def example_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) - def example_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def example_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported", token) def _is_escaped(src: str, pos: int) -> bool: @@ -510,10 +461,9 @@ def __init__(self) -> None: self._md.use(_block_attr) self._md.enable(["smartquotes", "replacements"]) - def _parse(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> list[Token]: - return self._md.parse(src, env if env is not None else {}) + def _parse(self, src: str) -> list[Token]: + return self._md.parse(src, {}) - def _render(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> str: - env = {} if env is None else env - tokens = self._parse(src, env) - return self._renderer.render(tokens, self._md.options, env) + def _render(self, src: str) -> str: + tokens = self._parse(src) + return self._renderer.render(tokens) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 8f64bd3ed5381..88c6d74433189 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -4,8 +4,7 @@ import json from abc import abstractmethod -from collections.abc import Mapping, MutableMapping, Sequence -from markdown_it.utils import OptionsDict +from collections.abc import Mapping, Sequence from markdown_it.token import Token from typing import Any, Generic, Optional from urllib.parse import quote @@ -174,29 +173,23 @@ def add_options(self, options: dict[str, Any]) -> None: def finalize(self) -> str: raise NotImplementedError() class OptionDocsRestrictions: - def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported in options doc", token) - def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported in options doc", token) - def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported in options doc", token) - def example_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def example_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: raise RuntimeError("md token not supported in options doc", token) class OptionsDocBookRenderer(OptionDocsRestrictions, DocBookRenderer): # TODO keep optionsDocBook diff small. remove soon if rendering is still good. - def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: token.meta['compact'] = False - return super().ordered_list_open(token, tokens, i, options, env) - def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, - env: MutableMapping[str, Any]) -> str: + return super().ordered_list_open(token, tokens, i) + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: token.meta['compact'] = False - return super().bullet_list_open(token, tokens, i, options, env) + return super().bullet_list_open(token, tokens, i) class DocBookConverter(BaseConverter[OptionsDocBookRenderer]): __option_block_separator__ = "" diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py index d20e056aacdc7..c6146429ea02d 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py @@ -1,8 +1,7 @@ -from collections.abc import Sequence, MutableMapping +from collections.abc import Sequence from typing import Any, Callable, Optional, Tuple, NamedTuple from markdown_it.token import Token -from markdown_it.utils import OptionsDict OptionLoc = str | dict[str, str] Option = dict[str, str | dict[str, str] | list[OptionLoc]] @@ -12,4 +11,4 @@ class RenderedOption(NamedTuple): lines: list[str] links: Optional[list[str]] = None -RenderFn = Callable[[Token, Sequence[Token], int, OptionsDict, MutableMapping[str, Any]], str] +RenderFn = Callable[[Token, Sequence[Token], int], str] From 068916ae8fccebf137ffe68b511bff26f1069ef8 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 19 Feb 2023 15:43:11 +0100 Subject: [PATCH 04/18] nixos-render-docs: keep revision in renderer, not converter ultimately it's the renderer that needs it, for the options rendering that will be simplified in a bit. --- .../src/nixos_render_docs/manual.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 7ac82958a5da3..78bf7659451bd 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -16,10 +16,12 @@ class ManualDocBookRenderer(DocBookRenderer): _toplevel_tag: str + _revision: str - def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str]): + def __init__(self, toplevel_tag: str, revision: str, manpage_urls: Mapping[str, str]): super().__init__(manpage_urls) self._toplevel_tag = toplevel_tag + self._revision = revision self.rules |= { 'included_sections': lambda *args: self._included_thing("section", *args), 'included_chapters': lambda *args: self._included_thing("chapter", *args), @@ -87,7 +89,7 @@ def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: in self._headings[-1] = self._headings[-1]._replace(partintro_closed=True) # must nest properly for structural includes. this requires saving at least # the headings stack, but creating new renderers is cheap and much easier. - r = ManualDocBookRenderer(tag, self._manpage_urls) + r = ManualDocBookRenderer(tag, self._revision, self._manpage_urls) for (included, path) in token.meta['included']: try: result.append(r.render(included)) @@ -110,12 +112,10 @@ def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: class DocBookConverter(Converter[ManualDocBookRenderer]): _base_paths: list[Path] - _revision: str def __init__(self, manpage_urls: Mapping[str, str], revision: str): super().__init__() - self._renderer = ManualDocBookRenderer('book', manpage_urls) - self._revision = revision + self._renderer = ManualDocBookRenderer('book', revision, manpage_urls) def convert(self, file: Path) -> str: self._base_paths = [ file ] @@ -182,7 +182,7 @@ def _parse_options(self, token: Token) -> None: try: conv = options.DocBookConverter( - self._renderer._manpage_urls, self._revision, False, 'fragment', varlist_id, id_prefix) + self._renderer._manpage_urls, self._renderer._revision, False, 'fragment', varlist_id, id_prefix) with open(self._base_paths[-1].parent / source, 'r') as f: conv.add_options(json.load(f)) token.meta['rendered-options'] = conv.finalize(fragment=True) From 5b8be28e66a31ba4683d0fc337f67a46d5db8f9a Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 19 Feb 2023 15:56:52 +0100 Subject: [PATCH 05/18] nixos-render-docs: don't render options during manual parsing we should really be rendering options at *rendering* time, not at parse time. currently this is just an academic exercise, but the html renderer will have to inspect the options.json data after the entire document has been parsed, but before anything gets rendered. --- .../src/nixos_render_docs/manual.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 78bf7659451bd..780a5f38c32ac 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -97,7 +97,10 @@ def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: in raise RuntimeError(f"rendering {path}") from e return "".join(result) def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str: - return cast(str, token.meta['rendered-options']) + conv = options.DocBookConverter(self._manpage_urls, self._revision, False, 'fragment', + token.meta['list-id'], token.meta['id-prefix']) + conv.add_options(token.meta['source']) + return conv.finalize(fragment=True) # TODO minimize docbook diffs with existing conversions. remove soon. def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: @@ -181,11 +184,10 @@ def _parse_options(self, token: Token) -> None: " ".join(items.keys())) try: - conv = options.DocBookConverter( - self._renderer._manpage_urls, self._renderer._revision, False, 'fragment', varlist_id, id_prefix) with open(self._base_paths[-1].parent / source, 'r') as f: - conv.add_options(json.load(f)) - token.meta['rendered-options'] = conv.finalize(fragment=True) + token.meta['id-prefix'] = id_prefix + token.meta['list-id'] = varlist_id + token.meta['source'] = json.load(f) except Exception as e: raise RuntimeError(f"processing options block in line {token.map[0] + 1}") from e From 2ab8e742a541659baa0470e3be8fc7e01ff51175 Mon Sep 17 00:00:00 2001 From: pennae Date: Wed, 15 Feb 2023 12:57:32 +0100 Subject: [PATCH 06/18] nixos-render-docs: move recursive manual parsing to base class the html renderer will need all of these functions as well. some extensions will be needed, but we'll add those as they become necessary. --- .../src/nixos_render_docs/manual.py | 153 +++++++++--------- 1 file changed, 77 insertions(+), 76 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 780a5f38c32ac..2e83ac90b5b6f 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -4,16 +4,90 @@ from abc import abstractmethod from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, cast, NamedTuple, Optional, Union +from typing import Any, cast, Generic, NamedTuple, Optional, Union from xml.sax.saxutils import escape, quoteattr import markdown_it from markdown_it.token import Token -from . import options +from . import md, options from .docbook import DocBookRenderer, Heading from .md import Converter +class BaseConverter(Converter[md.TR], Generic[md.TR]): + _base_paths: list[Path] + + def convert(self, file: Path) -> str: + self._base_paths = [ file ] + try: + with open(file, 'r') as f: + return self._render(f.read()) + except Exception as e: + raise RuntimeError(f"failed to render manual {file}") from e + + def _parse(self, src: str) -> list[Token]: + tokens = super()._parse(src) + for token in tokens: + if token.type != "fence" or not token.info.startswith("{=include=} "): + continue + typ = token.info[12:].strip() + if typ == 'options': + token.type = 'included_options' + self._parse_options(token) + elif typ in [ 'sections', 'chapters', 'preface', 'parts', 'appendix' ]: + token.type = 'included_' + typ + self._parse_included_blocks(token) + else: + raise RuntimeError(f"unsupported structural include type '{typ}'") + return tokens + + def _parse_included_blocks(self, token: Token) -> None: + assert token.map + included = token.meta['included'] = [] + for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): + line = line.strip() + path = self._base_paths[-1].parent / line + if path in self._base_paths: + raise RuntimeError(f"circular include found in line {lnum}") + try: + self._base_paths.append(path) + with open(path, 'r') as f: + tokens = self._parse(f.read()) + included.append((tokens, path)) + self._base_paths.pop() + except Exception as e: + raise RuntimeError(f"processing included file {path} from line {lnum}") from e + + def _parse_options(self, token: Token) -> None: + assert token.map + + items = {} + for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): + if len(args := line.split(":", 1)) != 2: + raise RuntimeError(f"options directive with no argument in line {lnum}") + (k, v) = (args[0].strip(), args[1].strip()) + if k in items: + raise RuntimeError(f"duplicate options directive {k} in line {lnum}") + items[k] = v + try: + id_prefix = items.pop('id-prefix') + varlist_id = items.pop('list-id') + source = items.pop('source') + except KeyError as e: + raise RuntimeError(f"options directive {e} missing in block at line {token.map[0] + 1}") + if items.keys(): + raise RuntimeError( + f"unsupported options directives in block at line {token.map[0] + 1}", + " ".join(items.keys())) + + try: + with open(self._base_paths[-1].parent / source, 'r') as f: + token.meta['id-prefix'] = id_prefix + token.meta['list-id'] = varlist_id + token.meta['source'] = json.load(f) + except Exception as e: + raise RuntimeError(f"processing options block in line {token.map[0] + 1}") from e + class ManualDocBookRenderer(DocBookRenderer): _toplevel_tag: str _revision: str @@ -113,84 +187,11 @@ def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: info = f" language={quoteattr(token.info)}" if token.info != "" else "" return f"\n{escape(token.content)}" -class DocBookConverter(Converter[ManualDocBookRenderer]): - _base_paths: list[Path] - +class DocBookConverter(BaseConverter[ManualDocBookRenderer]): def __init__(self, manpage_urls: Mapping[str, str], revision: str): super().__init__() self._renderer = ManualDocBookRenderer('book', revision, manpage_urls) - def convert(self, file: Path) -> str: - self._base_paths = [ file ] - try: - with open(file, 'r') as f: - return self._render(f.read()) - except Exception as e: - raise RuntimeError(f"failed to render manual {file}") from e - - def _parse(self, src: str) -> list[Token]: - tokens = super()._parse(src) - for token in tokens: - if token.type != "fence" or not token.info.startswith("{=include=} "): - continue - typ = token.info[12:].strip() - if typ == 'options': - token.type = 'included_options' - self._parse_options(token) - elif typ in [ 'sections', 'chapters', 'preface', 'parts', 'appendix' ]: - token.type = 'included_' + typ - self._parse_included_blocks(token) - else: - raise RuntimeError(f"unsupported structural include type '{typ}'") - return tokens - - def _parse_included_blocks(self, token: Token) -> None: - assert token.map - included = token.meta['included'] = [] - for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): - line = line.strip() - path = self._base_paths[-1].parent / line - if path in self._base_paths: - raise RuntimeError(f"circular include found in line {lnum}") - try: - self._base_paths.append(path) - with open(path, 'r') as f: - tokens = self._parse(f.read()) - included.append((tokens, path)) - self._base_paths.pop() - except Exception as e: - raise RuntimeError(f"processing included file {path} from line {lnum}") from e - - def _parse_options(self, token: Token) -> None: - assert token.map - - items = {} - for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): - if len(args := line.split(":", 1)) != 2: - raise RuntimeError(f"options directive with no argument in line {lnum}") - (k, v) = (args[0].strip(), args[1].strip()) - if k in items: - raise RuntimeError(f"duplicate options directive {k} in line {lnum}") - items[k] = v - try: - id_prefix = items.pop('id-prefix') - varlist_id = items.pop('list-id') - source = items.pop('source') - except KeyError as e: - raise RuntimeError(f"options directive {e} missing in block at line {token.map[0] + 1}") - if items.keys(): - raise RuntimeError( - f"unsupported options directives in block at line {token.map[0] + 1}", - " ".join(items.keys())) - - try: - with open(self._base_paths[-1].parent / source, 'r') as f: - token.meta['id-prefix'] = id_prefix - token.meta['list-id'] = varlist_id - token.meta['source'] = json.load(f) - except Exception as e: - raise RuntimeError(f"processing options block in line {token.map[0] + 1}") from e - def _build_cli_db(p: argparse.ArgumentParser) -> None: From a7c25bb01f2373eb1f113272731e1659bc91de95 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 19 Feb 2023 01:33:36 +0100 Subject: [PATCH 07/18] nixos-render-docs: add Freezable class for most of our data classes we can use dataclasses.dataclass with frozen=True or even plain named tuples. the TOC structure we'll need to generate proper navigation links is most easily represented and used as a cyclic structure though, and for that we can use neither. if we want to make the TOC structures immutable (which seems like a good idea) we'll need a hack of *some* kind, and this hack seems like the least intrusive. --- .../src/nixos_render_docs/utils.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/utils.py diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/utils.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/utils.py new file mode 100644 index 0000000000000..3377d1fa4fe18 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/utils.py @@ -0,0 +1,21 @@ +from typing import Any + +_frozen_classes: dict[type, type] = {} + +# make a derived class freezable (ie, disallow modifications). +# we do this by changing the class of an instance at runtime when freeze() +# is called, providing a derived class that is exactly the same except +# for a __setattr__ that raises an error when called. this beats having +# a field for frozenness and an unconditional __setattr__ that checks this +# field because it does not insert anything into the class dict. +class Freezeable: + def freeze(self) -> None: + cls = type(self) + if not (frozen := _frozen_classes.get(cls)): + def __setattr__(instance: Any, n: str, v: Any) -> None: + raise TypeError(f'{cls.__name__} is frozen') + frozen = type(cls.__name__, (cls,), { + '__setattr__': __setattr__, + }) + _frozen_classes[cls] = frozen + self.__class__ = frozen From 7b0824c0031dd65e1abd6d88ca537a90d4dfbe23 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Feb 2023 20:34:31 +0100 Subject: [PATCH 08/18] nixos-render-docs: check heading presence during parsing check that all required headings are present during parsing, not during rendering. building a correct TOC will need this since every TOC entry needs a heading to set its title, and every included substructure needs a title. also improve the error message on repeated title headings slightly, giving the end line turns out to not be very useful. --- .../src/nixos_render_docs/manual.py | 34 ++++++------------- .../src/nixos_render_docs/manual_structure.py | 29 ++++++++++++++++ 2 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 2e83ac90b5b6f..7e1923f35ec44 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -4,7 +4,7 @@ from abc import abstractmethod from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, cast, Generic, NamedTuple, Optional, Union +from typing import Any, cast, Generic, get_args, NamedTuple, Optional, Union from xml.sax.saxutils import escape, quoteattr import markdown_it @@ -12,13 +12,16 @@ from . import md, options from .docbook import DocBookRenderer, Heading +from .manual_structure import check_titles, FragmentType, TocEntryType from .md import Converter class BaseConverter(Converter[md.TR], Generic[md.TR]): _base_paths: list[Path] + _current_type: list[TocEntryType] def convert(self, file: Path) -> str: self._base_paths = [ file ] + self._current_type = ['book'] try: with open(file, 'r') as f: return self._render(f.read()) @@ -27,6 +30,7 @@ def convert(self, file: Path) -> str: def _parse(self, src: str) -> list[Token]: tokens = super()._parse(src) + check_titles(self._current_type[-1], tokens) for token in tokens: if token.type != "fence" or not token.info.startswith("{=include=} "): continue @@ -34,11 +38,14 @@ def _parse(self, src: str) -> list[Token]: if typ == 'options': token.type = 'included_options' self._parse_options(token) - elif typ in [ 'sections', 'chapters', 'preface', 'parts', 'appendix' ]: + else: + fragment_type = typ.removesuffix('s') + if fragment_type not in get_args(FragmentType): + raise RuntimeError(f"unsupported structural include type '{typ}'") + self._current_type.append(cast(FragmentType, fragment_type)) token.type = 'included_' + typ self._parse_included_blocks(token) - else: - raise RuntimeError(f"unsupported structural include type '{typ}'") + self._current_type.pop() return tokens def _parse_included_blocks(self, token: Token) -> None: @@ -106,25 +113,6 @@ def __init__(self, toplevel_tag: str, revision: str, manpage_urls: Mapping[str, } def render(self, tokens: Sequence[Token]) -> str: - wanted = { 'h1': 'title' } - wanted |= { 'h2': 'subtitle' } if self._toplevel_tag == 'book' else {} - for (i, (tag, kind)) in enumerate(wanted.items()): - if len(tokens) < 3 * (i + 1): - raise RuntimeError(f"missing {kind} ({tag}) heading") - token = tokens[3 * i] - if token.type != 'heading_open' or token.tag != tag: - assert token.map - raise RuntimeError(f"expected {kind} ({tag}) heading in line {token.map[0] + 1}", token) - for t in tokens[3 * len(wanted):]: - if t.type != 'heading_open' or (info := wanted.get(t.tag)) is None: - continue - assert t.map - raise RuntimeError( - f"only one {info[0]} heading ({t.markup} [text...]) allowed per " - f"{self._toplevel_tag}, but found a second in lines [{t.map[0] + 1}..{t.map[1]}]. " - "please remove all such headings except the first or demote the subsequent headings.", - t) - # books get special handling because they have *two* title tags. doing this with # generic code is more complicated than it's worth. the checks above have verified # that both titles actually exist. diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py new file mode 100644 index 0000000000000..d7cf449a417b3 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py @@ -0,0 +1,29 @@ +from typing import Literal, Sequence + +from markdown_it.token import Token + +# FragmentType is used to restrict structural include blocks. +FragmentType = Literal['preface', 'part', 'chapter', 'section', 'appendix'] + +# in the TOC all fragments are allowed, plus the all-encompassing book. +TocEntryType = Literal['book', 'preface', 'part', 'chapter', 'section', 'appendix'] + +def check_titles(kind: TocEntryType, tokens: Sequence[Token]) -> None: + wanted = { 'h1': 'title' } + wanted |= { 'h2': 'subtitle' } if kind == 'book' else {} + for (i, (tag, role)) in enumerate(wanted.items()): + if len(tokens) < 3 * (i + 1): + raise RuntimeError(f"missing {role} ({tag}) heading") + token = tokens[3 * i] + if token.type != 'heading_open' or token.tag != tag: + assert token.map + raise RuntimeError(f"expected {role} ({tag}) heading in line {token.map[0] + 1}", token) + for t in tokens[3 * len(wanted):]: + if t.type != 'heading_open' or not (role := wanted.get(t.tag, '')): + continue + assert t.map + raise RuntimeError( + f"only one {role} heading ({t.markup} [text...]) allowed per " + f"{kind}, but found a second in line {t.map[0] + 1}. " + "please remove all such headings except the first or demote the subsequent headings.", + t) From ba201144605ed4ba83d165a37c3660ef2b49b193 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Feb 2023 20:36:29 +0100 Subject: [PATCH 09/18] nixos-render-docs: check heading continuity while not technically necessary for correct rendering of *contents* we do need to disallow heading levels being skipped to build a correct TOC. treating headings that have skipped a number of levels to actually be headings that many levels up only gets confusing, and inserting artifical intermediate headings suffers from problems, such as which ids to use and what to call them. --- .../src/nixos_render_docs/manual_structure.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py index d7cf449a417b3..c6842db606f19 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py @@ -27,3 +27,14 @@ def check_titles(kind: TocEntryType, tokens: Sequence[Token]) -> None: f"{kind}, but found a second in line {t.map[0] + 1}. " "please remove all such headings except the first or demote the subsequent headings.", t) + + last_heading_level = 0 + for token in tokens: + if token.type != 'heading_open': + continue + level = int(token.tag[1:]) # because tag = h1..h6 + if level > last_heading_level + 1: + assert token.map + raise RuntimeError(f"heading in line {token.map[0] + 1} skips one or more heading levels, " + "which is currently not allowed") + last_heading_level = level From 163b667352e19411473fdf8603f0883c1b106d58 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Feb 2023 20:41:34 +0100 Subject: [PATCH 10/18] nixos-render-docs: require headings to have ids without this we cannot build a TOC to arbitrary depth without generating ids for headings, but generated ids are fragile and liable to either break or point to different things if the manual changes shape. we already have the convention that all headings should have an id, this formalizes it. --- .../src/nixos_render_docs/manual_structure.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py index c6842db606f19..32b6287b34ad4 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py @@ -32,6 +32,14 @@ def check_titles(kind: TocEntryType, tokens: Sequence[Token]) -> None: for token in tokens: if token.type != 'heading_open': continue + + # book subtitle headings do not need an id, only book title headings do. + # every other headings needs one too. we need this to build a TOC and to + # provide stable links if the manual changes shape. + if 'id' not in token.attrs and (kind != 'book' or token.tag != 'h2'): + assert token.map + raise RuntimeError(f"heading in line {token.map[0] + 1} does not have an id") + level = int(token.tag[1:]) # because tag = h1..h6 if level > last_heading_level + 1: assert token.map From 768794d6c11b5e37f954405a3f03d63ef45897f6 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Feb 2023 20:48:12 +0100 Subject: [PATCH 11/18] nixos-render-docs: check book structure text content in the toplevel file of a book will not render properly. the first proper element will be a preface, part, or chapter anyway, and those require includes to produce. parts do not currently allow headings in the part file itself, but that's mainly a renderer limitation. we can add support for headings in part intros when we need them in all other cases includes must be followed by either another include, a heading, or end of file. text content could not be properly linked to from a TOC without a preceding heading. --- .../src/nixos_render_docs/manual.py | 6 +-- .../src/nixos_render_docs/manual_structure.py | 43 ++++++++++++++++++- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 7e1923f35ec44..858ecad9c11ae 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -12,7 +12,7 @@ from . import md, options from .docbook import DocBookRenderer, Heading -from .manual_structure import check_titles, FragmentType, TocEntryType +from .manual_structure import check_structure, FragmentType, is_include, TocEntryType from .md import Converter class BaseConverter(Converter[md.TR], Generic[md.TR]): @@ -30,9 +30,9 @@ def convert(self, file: Path) -> str: def _parse(self, src: str) -> list[Token]: tokens = super()._parse(src) - check_titles(self._current_type[-1], tokens) + check_structure(self._current_type[-1], tokens) for token in tokens: - if token.type != "fence" or not token.info.startswith("{=include=} "): + if not is_include(token): continue typ = token.info[12:].strip() if typ == 'options': diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py index 32b6287b34ad4..93a8ecc3f9350 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py @@ -8,7 +8,41 @@ # in the TOC all fragments are allowed, plus the all-encompassing book. TocEntryType = Literal['book', 'preface', 'part', 'chapter', 'section', 'appendix'] -def check_titles(kind: TocEntryType, tokens: Sequence[Token]) -> None: +def is_include(token: Token) -> bool: + return token.type == "fence" and token.info.startswith("{=include=} ") + +# toplevel file must contain only the title headings and includes, anything else +# would cause strange rendering. +def _check_book_structure(tokens: Sequence[Token]) -> None: + for token in tokens[6:]: + if not is_include(token): + assert token.map + raise RuntimeError(f"unexpected content in line {token.map[0] + 1}, " + "expected structural include") + +# much like books, parts may not contain headings other than their title heading. +# this is a limitation of the current renderers that do not handle this case well +# even though it is supported in docbook (and probably supportable anywhere else). +def _check_part_structure(tokens: Sequence[Token]) -> None: + _check_fragment_structure(tokens) + for token in tokens[3:]: + if token.type == 'heading_open': + assert token.map + raise RuntimeError(f"unexpected heading in line {token.map[0] + 1}") + +# two include blocks must either be adjacent or separated by a heading, otherwise +# we cannot generate a correct TOC (since there'd be nothing to link to between +# the two includes). +def _check_fragment_structure(tokens: Sequence[Token]) -> None: + for i, token in enumerate(tokens): + if is_include(token) \ + and i + 1 < len(tokens) \ + and not (is_include(tokens[i + 1]) or tokens[i + 1].type == 'heading_open'): + assert token.map + raise RuntimeError(f"unexpected content in line {token.map[0] + 1}, " + "expected heading or structural include") + +def check_structure(kind: TocEntryType, tokens: Sequence[Token]) -> None: wanted = { 'h1': 'title' } wanted |= { 'h2': 'subtitle' } if kind == 'book' else {} for (i, (tag, role)) in enumerate(wanted.items()): @@ -46,3 +80,10 @@ def check_titles(kind: TocEntryType, tokens: Sequence[Token]) -> None: raise RuntimeError(f"heading in line {token.map[0] + 1} skips one or more heading levels, " "which is currently not allowed") last_heading_level = level + + if kind == 'book': + _check_book_structure(tokens) + elif kind == 'part': + _check_part_structure(tokens) + else: + _check_fragment_structure(tokens) From 23dc31a9755cf34a2040fe574798c09d1c857df0 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 18 Feb 2023 21:23:48 +0100 Subject: [PATCH 12/18] nixos-render-docs: allow for options in include blocks while docbook relies on external chunk-toc info to do chunking of the rendered manual we have nothing of the sort for html. there it seems easiest to add annotations to blocks to create new chunks. such annotations could be extended to docbook to create the chunk-toc instead of passing it in externally, but with docbook on the way out that seems like a waste of effort. --- .../src/nixos_render_docs/manual.py | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 858ecad9c11ae..8d7bf4a102f37 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -4,7 +4,7 @@ from abc import abstractmethod from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, cast, Generic, get_args, NamedTuple, Optional, Union +from typing import Any, cast, ClassVar, Generic, get_args, NamedTuple, Optional, Union from xml.sax.saxutils import escape, quoteattr import markdown_it @@ -16,6 +16,14 @@ from .md import Converter class BaseConverter(Converter[md.TR], Generic[md.TR]): + # per-converter configuration for ns:arg=value arguments to include blocks, following + # the include type. html converters need something like this to support chunking, or + # another external method like the chunktocs docbook uses (but block options seem like + # a much nicer of doing this). + INCLUDE_ARGS_NS: ClassVar[str] + INCLUDE_FRAGMENT_ALLOWED_ARGS: ClassVar[set[str]] = set() + INCLUDE_OPTIONS_ALLOWED_ARGS: ClassVar[set[str]] = set() + _base_paths: list[Path] _current_type: list[TocEntryType] @@ -34,21 +42,35 @@ def _parse(self, src: str) -> list[Token]: for token in tokens: if not is_include(token): continue - typ = token.info[12:].strip() + directive = token.info[12:].split() + if not directive: + continue + args = { k: v for k, _sep, v in map(lambda s: s.partition('='), directive[1:]) } + typ = directive[0] if typ == 'options': token.type = 'included_options' - self._parse_options(token) + self._process_include_args(token, args, self.INCLUDE_OPTIONS_ALLOWED_ARGS) + self._parse_options(token, args) else: fragment_type = typ.removesuffix('s') if fragment_type not in get_args(FragmentType): raise RuntimeError(f"unsupported structural include type '{typ}'") self._current_type.append(cast(FragmentType, fragment_type)) token.type = 'included_' + typ - self._parse_included_blocks(token) + self._process_include_args(token, args, self.INCLUDE_FRAGMENT_ALLOWED_ARGS) + self._parse_included_blocks(token, args) self._current_type.pop() return tokens - def _parse_included_blocks(self, token: Token) -> None: + def _process_include_args(self, token: Token, args: dict[str, str], allowed: set[str]) -> None: + ns = self.INCLUDE_ARGS_NS + ":" + args = { k[len(ns):]: v for k, v in args.items() if k.startswith(ns) } + if unknown := set(args.keys()) - allowed: + assert token.map + raise RuntimeError(f"unrecognized include argument in line {token.map[0] + 1}", unknown) + token.meta['include-args'] = args + + def _parse_included_blocks(self, token: Token, block_args: dict[str, str]) -> None: assert token.map included = token.meta['included'] = [] for (lnum, line) in enumerate(token.content.splitlines(), token.map[0] + 2): @@ -65,7 +87,7 @@ def _parse_included_blocks(self, token: Token) -> None: except Exception as e: raise RuntimeError(f"processing included file {path} from line {lnum}") from e - def _parse_options(self, token: Token) -> None: + def _parse_options(self, token: Token, block_args: dict[str, str]) -> None: assert token.map items = {} @@ -176,6 +198,8 @@ def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: return f"\n{escape(token.content)}" class DocBookConverter(BaseConverter[ManualDocBookRenderer]): + INCLUDE_ARGS_NS = "docbook" + def __init__(self, manpage_urls: Mapping[str, str], revision: str): super().__init__() self._renderer = ManualDocBookRenderer('book', revision, manpage_urls) From 7a74ce51a1643aa6e83c912203bb6c3a987376b9 Mon Sep 17 00:00:00 2001 From: pennae Date: Sun, 19 Feb 2023 19:19:13 +0100 Subject: [PATCH 13/18] nixos-render-docs: add toc generator the docbook toolchain uses docbook-xsl to generate its TOC, our html renderer will have to do this on its own. this generator uses a very straight-forward algorithm of only inspecting headings, but anything else could be inspected as well. (examples come to mind, but those do not have titles and would thus make for bad toc entries) we also use path information (that will be taken from include block args in the html renderer) to produce navigation information. the algorithm we use mirrors what docbook does, linking to the next/previous files in depth-first toc order. toc entries are linked to the tokens they refer to for easy use later. --- .../src/nixos_render_docs/manual_structure.py | 103 +++++++++++++++++- 1 file changed, 100 insertions(+), 3 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py index 93a8ecc3f9350..c271ca3c5aa5f 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual_structure.py @@ -1,7 +1,15 @@ -from typing import Literal, Sequence +from __future__ import annotations + +import dataclasses as dc +import html +import itertools + +from typing import cast, get_args, Iterable, Literal, Sequence from markdown_it.token import Token +from .utils import Freezeable + # FragmentType is used to restrict structural include blocks. FragmentType = Literal['preface', 'part', 'chapter', 'section', 'appendix'] @@ -21,8 +29,9 @@ def _check_book_structure(tokens: Sequence[Token]) -> None: "expected structural include") # much like books, parts may not contain headings other than their title heading. -# this is a limitation of the current renderers that do not handle this case well -# even though it is supported in docbook (and probably supportable anywhere else). +# this is a limitation of the current renderers and TOC generators that do not handle +# this case well even though it is supported in docbook (and probably supportable +# anywhere else). def _check_part_structure(tokens: Sequence[Token]) -> None: _check_fragment_structure(tokens) for token in tokens[3:]: @@ -87,3 +96,91 @@ def check_structure(kind: TocEntryType, tokens: Sequence[Token]) -> None: _check_part_structure(tokens) else: _check_fragment_structure(tokens) + +@dc.dataclass(frozen=True) +class XrefTarget: + id: str + """link label for `[](#local-references)`""" + title_html: str + """toc label""" + toc_html: str | None + """text for `` tags and `title="..."` attributes""" + title: str | None + """path to file that contains the anchor""" + path: str + """whether to drop the `#anchor` from links when expanding xrefs""" + drop_fragment: bool = False + + def href(self) -> str: + path = html.escape(self.path, True) + return path if self.drop_fragment else f"{path}#{html.escape(self.id, True)}" + +@dc.dataclass +class TocEntry(Freezeable): + kind: TocEntryType + target: XrefTarget + parent: TocEntry | None = None + prev: TocEntry | None = None + next: TocEntry | None = None + children: list[TocEntry] = dc.field(default_factory=list) + starts_new_chunk: bool = False + + @property + def root(self) -> TocEntry: + return self.parent.root if self.parent else self + + @classmethod + def of(cls, token: Token) -> TocEntry: + entry = token.meta.get('TocEntry') + if not isinstance(entry, TocEntry): + raise RuntimeError('requested toc entry, none found', token) + return entry + + @classmethod + def collect_and_link(cls, xrefs: dict[str, XrefTarget], tokens: Sequence[Token]) -> TocEntry: + result = cls._collect_entries(xrefs, tokens, 'book') + + def flatten_with_parent(this: TocEntry, parent: TocEntry | None) -> Iterable[TocEntry]: + this.parent = parent + return itertools.chain([this], *[ flatten_with_parent(c, this) for c in this.children ]) + + flat = list(flatten_with_parent(result, None)) + prev = flat[0] + prev.starts_new_chunk = True + paths_seen = set([prev.target.path]) + for c in flat[1:]: + if prev.target.path != c.target.path and c.target.path not in paths_seen: + c.starts_new_chunk = True + c.prev, prev.next = prev, c + prev = c + paths_seen.add(c.target.path) + + for c in flat: + c.freeze() + + return result + + @classmethod + def _collect_entries(cls, xrefs: dict[str, XrefTarget], tokens: Sequence[Token], + kind: TocEntryType) -> TocEntry: + # we assume that check_structure has been run recursively over the entire input. + # list contains (tag, entry) pairs that will collapse to a single entry for + # the full sequence. + entries: list[tuple[str, TocEntry]] = [] + for token in tokens: + if token.type.startswith('included_') and (included := token.meta.get('included')): + fragment_type_str = token.type[9:].removesuffix('s') + assert fragment_type_str in get_args(TocEntryType) + fragment_type = cast(TocEntryType, fragment_type_str) + for fragment, _path in included: + entries[-1][1].children.append(cls._collect_entries(xrefs, fragment, fragment_type)) + elif token.type == 'heading_open' and (id := cast(str, token.attrs.get('id', ''))): + while len(entries) > 1 and entries[-1][0] >= token.tag: + entries[-2][1].children.append(entries.pop()[1]) + entries.append((token.tag, + TocEntry(kind if token.tag == 'h1' else 'section', xrefs[id]))) + token.meta['TocEntry'] = entries[-1][1] + + while len(entries) > 1: + entries[-2][1].children.append(entries.pop()[1]) + return entries[0][1] From 82e62614e9171981a3972940b8e96e02c7e55f83 Mon Sep 17 00:00:00 2001 From: pennae <github@quasiparticle.net> Date: Sun, 19 Feb 2023 22:53:21 +0100 Subject: [PATCH 14/18] nixos-render-docs: add html renderer the basic html renderer. it doesn't have all the docbook compatibility codes embedded into it, but there is a good amount. this renderer is unaware of manual structure and does not traverse structural include tokens (if it finds any it'll just fail), that task falls to derived classes. once we have more uses for structural includes than just the manual we may revisit this decision. --- .../src/nixos_render_docs/__init__.py | 1 + .../src/nixos_render_docs/html.py | 245 ++++++++++++++++++ .../nixos-render-docs/src/tests/test_html.py | 179 +++++++++++++ 3 files changed, 425 insertions(+) create mode 100644 pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/html.py create mode 100644 pkgs/tools/nix/nixos-render-docs/src/tests/test_html.py diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py index 1c58accb41662..7f7463e5c837c 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py @@ -8,6 +8,7 @@ from typing import Any, Dict from .md import Converter +from . import html from . import manual from . import options from . import parallel diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/html.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/html.py new file mode 100644 index 0000000000000..39d2da6adf8c0 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/html.py @@ -0,0 +1,245 @@ +from collections.abc import Mapping, Sequence +from typing import cast, Optional, NamedTuple + +from html import escape +from markdown_it.token import Token + +from .manual_structure import XrefTarget +from .md import Renderer + +class UnresolvedXrefError(Exception): + pass + +class Heading(NamedTuple): + container_tag: str + level: int + html_tag: str + # special handling for part content: whether partinfo div was already closed from + # elsewhere or still needs closing. + partintro_closed: bool + # tocs are generated when the heading opens, but have to be emitted into the file + # after the heading titlepage (and maybe partinfo) has been closed. + toc_fragment: str + +_bullet_list_styles = [ 'disc', 'circle', 'square' ] +_ordered_list_styles = [ '1', 'a', 'i', 'A', 'I' ] + +class HTMLRenderer(Renderer): + _xref_targets: Mapping[str, XrefTarget] + + _headings: list[Heading] + _attrspans: list[str] + _hlevel_offset: int = 0 + _bullet_list_nesting: int = 0 + _ordered_list_nesting: int = 0 + + def __init__(self, manpage_urls: Mapping[str, str], xref_targets: Mapping[str, XrefTarget]): + super().__init__(manpage_urls) + self._headings = [] + self._attrspans = [] + self._xref_targets = xref_targets + + def render(self, tokens: Sequence[Token]) -> str: + result = super().render(tokens) + result += self._close_headings(None) + return result + + def text(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return escape(token.content) + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "<p>" + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</p>" + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "<br />" + def softbreak(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "\n" + def code_inline(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return f'<code class="literal">{escape(token.content)}</code>' + def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return self.fence(token, tokens, i) + def link_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + href = escape(cast(str, token.attrs['href']), True) + tag, title, target, text = "link", "", 'target="_top"', "" + if href.startswith('#'): + if not (xref := self._xref_targets.get(href[1:])): + raise UnresolvedXrefError(f"bad local reference, id {href} not known") + if tokens[i + 1].type == 'link_close': + tag, text = "xref", xref.title_html + if xref.title: + title = f'title="{escape(xref.title, True)}"' + target, href = "", xref.href() + return f'<a class="{tag}" href="{href}" {title} {target}>{text}' + def link_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</a>" + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<li class="listitem">' + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</li>" + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + extra = 'compact' if token.meta.get('compact', False) else '' + style = _bullet_list_styles[self._bullet_list_nesting % len(_bullet_list_styles)] + self._bullet_list_nesting += 1 + return f'<div class="itemizedlist"><ul class="itemizedlist {extra}" style="list-style-type: {style};">' + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + self._bullet_list_nesting -= 1 + return "</ul></div>" + def em_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<span class="emphasis"><em>' + def em_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</em></span>" + def strong_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<span class="strong"><strong>' + def strong_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</strong></span>" + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: + # TODO use token.info. docbook doesn't so we can't yet. + return f'<pre class="programlisting">\n{escape(token.content)}</pre>' + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="blockquote"><blockquote class="blockquote">' + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</blockquote></div>" + def note_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="note"><h3 class="title">Note</h3>' + def note_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</div>" + def caution_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="caution"><h3 class="title">Caution</h3>' + def caution_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</div>" + def important_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="important"><h3 class="title">Important</h3>' + def important_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</div>" + def tip_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="tip"><h3 class="title">Tip</h3>' + def tip_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</div>" + def warning_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="warning"><h3 class="title">Warning</h3>' + def warning_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</div>" + def dl_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<div class="variablelist"><dl class="variablelist">' + def dl_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</dl></div>" + def dt_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return '<dt><span class="term">' + def dt_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</span></dt>" + def dd_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "<dd>" + def dd_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "</dd>" + def myst_role(self, token: Token, tokens: Sequence[Token], i: int) -> str: + if token.meta['name'] == 'command': + return f'<span class="command"><strong>{escape(token.content)}</strong></span>' + if token.meta['name'] == 'file': + return f'<code class="filename">{escape(token.content)}</code>' + if token.meta['name'] == 'var': + return f'<code class="varname">{escape(token.content)}</code>' + if token.meta['name'] == 'env': + return f'<code class="envar">{escape(token.content)}</code>' + if token.meta['name'] == 'option': + return f'<code class="option">{escape(token.content)}</code>' + if token.meta['name'] == 'manpage': + [page, section] = [ s.strip() for s in token.content.rsplit('(', 1) ] + section = section[:-1] + man = f"{page}({section})" + title = f'<span class="refentrytitle">{escape(page)}</span>' + vol = f"({escape(section)})" + ref = f'<span class="citerefentry">{title}{vol}</span>' + if man in self._manpage_urls: + return f'<a class="link" href="{escape(self._manpage_urls[man], True)}" target="_top">{ref}</a>' + else: + return ref + return super().myst_role(token, tokens, i) + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int) -> str: + # we currently support *only* inline anchors and the special .keycap class to produce + # keycap-styled spans. + (id_part, class_part) = ("", "") + if s := token.attrs.get('id'): + id_part = f'<a id="{escape(cast(str, s), True)}" />' + if s := token.attrs.get('class'): + if s == 'keycap': + class_part = '<span class="keycap"><strong>' + self._attrspans.append("</strong></span>") + else: + return super().attr_span_begin(token, tokens, i) + else: + self._attrspans.append("") + return id_part + class_part + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return self._attrspans.pop() + def heading_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + hlevel = int(token.tag[1:]) + htag, hstyle = self._make_hN(hlevel) + if hstyle: + hstyle = f'style="{escape(hstyle, True)}"' + if anchor := cast(str, token.attrs.get('id', '')): + anchor = f'<a id="{escape(anchor, True)}"></a>' + result = self._close_headings(hlevel) + tag = self._heading_tag(token, tokens, i) + toc_fragment = self._build_toc(tokens, i) + self._headings.append(Heading(tag, hlevel, htag, tag != 'part', toc_fragment)) + return ( + f'{result}' + f'<div class="{tag}">' + f' <div class="titlepage">' + f' <div>' + f' <div>' + f' <{htag} class="title" {hstyle}>' + f' {anchor}' + ) + def heading_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + heading = self._headings[-1] + result = ( + f' </{heading.html_tag}>' + f' </div>' + f' </div>' + f'</div>' + ) + if heading.container_tag == 'part': + result += '<div class="partintro">' + else: + result += heading.toc_fragment + return result + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + extra = 'compact' if token.meta.get('compact', False) else '' + start = f'start="{token.attrs["start"]}"' if 'start' in token.attrs else "" + style = _ordered_list_styles[self._ordered_list_nesting % len(_ordered_list_styles)] + self._ordered_list_nesting += 1 + return f'<div class="orderedlist"><ol class="orderedlist {extra}" {start} type="{style}">' + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + self._ordered_list_nesting -= 1; + return "</ol></div>" + def example_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + if id := token.attrs.get('id'): + return f'<a id="{escape(cast(str, id), True)}" />' + return "" + def example_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "" + + def _make_hN(self, level: int) -> tuple[str, str]: + return f"h{min(6, max(1, level + self._hlevel_offset))}", "" + + def _maybe_close_partintro(self) -> str: + if self._headings: + heading = self._headings[-1] + if heading.container_tag == 'part' and not heading.partintro_closed: + self._headings[-1] = heading._replace(partintro_closed=True) + return heading.toc_fragment + "</div>" + return "" + + def _close_headings(self, level: Optional[int]) -> str: + result = [] + while len(self._headings) and (level is None or self._headings[-1].level >= level): + result.append(self._maybe_close_partintro()) + result.append("</div>") + self._headings.pop() + return "\n".join(result) + + def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int) -> str: + return "section" + def _build_toc(self, tokens: Sequence[Token], i: int) -> str: + return "" diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_html.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_html.py new file mode 100644 index 0000000000000..df366a8babd7e --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_html.py @@ -0,0 +1,179 @@ +import nixos_render_docs as nrd +import pytest + +from sample_md import sample1 + +class Converter(nrd.md.Converter[nrd.html.HTMLRenderer]): + def __init__(self, manpage_urls: dict[str, str], xrefs: dict[str, nrd.manual_structure.XrefTarget]): + super().__init__() + self._renderer = nrd.html.HTMLRenderer(manpage_urls, xrefs) + +def unpretty(s: str) -> str: + return "".join(map(str.strip, s.splitlines())).replace('␣', ' ').replace('↵', '\n') + +def test_lists_styles() -> None: + # nested lists rotate through a number of list style + c = Converter({}, {}) + assert c._render("- - - - foo") == unpretty(""" + <div class="itemizedlist"><ul class="itemizedlist compact" style="list-style-type: disc;"> + <li class="listitem"> + <div class="itemizedlist"><ul class="itemizedlist compact" style="list-style-type: circle;"> + <li class="listitem"> + <div class="itemizedlist"><ul class="itemizedlist compact" style="list-style-type: square;"> + <li class="listitem"> + <div class="itemizedlist"><ul class="itemizedlist compact" style="list-style-type: disc;"> + <li class="listitem"><p>foo</p></li> + </ul></div> + </li> + </ul></div> + </li> + </ul></div> + </li> + </ul></div> + """) + assert c._render("1. 1. 1. 1. 1. 1. foo") == unpretty(""" + <div class="orderedlist"><ol class="orderedlist compact" type="1"> + <li class="listitem"> + <div class="orderedlist"><ol class="orderedlist compact" type="a"> + <li class="listitem"> + <div class="orderedlist"><ol class="orderedlist compact" type="i"> + <li class="listitem"> + <div class="orderedlist"><ol class="orderedlist compact" type="A"> + <li class="listitem"> + <div class="orderedlist"><ol class="orderedlist compact" type="I"> + <li class="listitem"> + <div class="orderedlist"><ol class="orderedlist compact" type="1"> + <li class="listitem"><p>foo</p></li> + </ol></div> + </li> + </ol></div> + </li> + </ol></div> + </li> + </ol></div> + </li> + </ol></div> + </li> + </ol></div> + """) + +def test_xrefs() -> None: + # nested lists rotate through a number of list style + c = Converter({}, { + 'foo': nrd.manual_structure.XrefTarget('foo', '<hr/>', 'toc1', 'title1', 'index.html'), + 'bar': nrd.manual_structure.XrefTarget('bar', '<br/>', 'toc2', 'title2', 'index.html', True), + }) + assert c._render("[](#foo)") == '<p><a class="xref" href="index.html#foo" title="title1" ><hr/></a></p>' + assert c._render("[](#bar)") == '<p><a class="xref" href="index.html" title="title2" ><br/></a></p>' + with pytest.raises(nrd.html.UnresolvedXrefError) as exc: + c._render("[](#baz)") + assert exc.value.args[0] == 'bad local reference, id #baz not known' + +def test_full() -> None: + c = Converter({ 'man(1)': 'http://example.org' }, {}) + assert c._render(sample1) == unpretty(""" + <div class="warning"> + <h3 class="title">Warning</h3> + <p>foo</p> + <div class="note"> + <h3 class="title">Note</h3> + <p>nested</p> + </div> + </div> + <p> + <a class="link" href="link" target="_top">↵ + multiline↵ + </a> + </p> + <p> + <a class="link" href="http://example.org" target="_top"> + <span class="citerefentry"><span class="refentrytitle">man</span>(1)</span> + </a> reference + </p> + <p><a id="b" />some <a id="a" />nested anchors</p> + <p> + <span class="emphasis"><em>emph</em></span>␣ + <span class="strong"><strong>strong</strong></span>␣ + <span class="emphasis"><em>nesting emph <span class="strong"><strong>and strong</strong></span>␣ + and <code class="literal">code</code></em></span> + </p> + <div class="itemizedlist"> + <ul class="itemizedlist " style="list-style-type: disc;"> + <li class="listitem"><p>wide bullet</p></li> + <li class="listitem"><p>list</p></li> + </ul> + </div> + <div class="orderedlist"> + <ol class="orderedlist " type="1"> + <li class="listitem"><p>wide ordered</p></li> + <li class="listitem"><p>list</p></li> + </ol> + </div> + <div class="itemizedlist"> + <ul class="itemizedlist compact" style="list-style-type: disc;"> + <li class="listitem"><p>narrow bullet</p></li> + <li class="listitem"><p>list</p></li> + </ul> + </div> + <div class="orderedlist"> + <ol class="orderedlist compact" type="1"> + <li class="listitem"><p>narrow ordered</p></li> + <li class="listitem"><p>list</p></li> + </ol> + </div> + <div class="blockquote"> + <blockquote class="blockquote"> + <p>quotes</p> + <div class="blockquote"> + <blockquote class="blockquote"> + <p>with <span class="emphasis"><em>nesting</em></span></p> + <pre class="programlisting">↵ + nested code block↵ + </pre> + </blockquote> + </div> + <div class="itemizedlist"> + <ul class="itemizedlist compact" style="list-style-type: disc;"> + <li class="listitem"><p>and lists</p></li> + <li class="listitem"> + <pre class="programlisting">↵ + containing code↵ + </pre> + </li> + </ul> + </div> + <p>and more quote</p> + </blockquote> + </div> + <div class="orderedlist"> + <ol class="orderedlist compact" start="100" type="1"> + <li class="listitem"><p>list starting at 100</p></li> + <li class="listitem"><p>goes on</p></li> + </ol> + </div> + <div class="variablelist"> + <dl class="variablelist"> + <dt><span class="term">deflist</span></dt> + <dd> + <div class="blockquote"> + <blockquote class="blockquote"> + <p> + with a quote↵ + and stuff + </p> + </blockquote> + </div> + <pre class="programlisting">↵ + code block↵ + </pre> + <pre class="programlisting">↵ + fenced block↵ + </pre> + <p>text</p> + </dd> + <dt><span class="term">more stuff in same deflist</span></dt> + <dd> + <p>foo</p> + </dd> + </dl> + </div>""") From d520d55dee017fff8b3d0682260e12a27035378e Mon Sep 17 00:00:00 2001 From: pennae <github@quasiparticle.net> Date: Sun, 19 Feb 2023 23:03:32 +0100 Subject: [PATCH 15/18] nixos-render-docs: add options html renderer it's not hooked up to anything yet, but that will come soon. there's a bit of docbook compat here that must be interoperable with the actual docbook exporter, but luckily it's not all that much. --- .../src/nixos_render_docs/options.py | 116 +++++++++++++++++- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 88c6d74433189..06e5f97112168 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -1,14 +1,15 @@ from __future__ import annotations import argparse +import html import json +import xml.sax.saxutils as xml from abc import abstractmethod from collections.abc import Mapping, Sequence from markdown_it.token import Token from typing import Any, Generic, Optional from urllib.parse import quote -from xml.sax.saxutils import escape, quoteattr import markdown_it @@ -17,7 +18,9 @@ from .asciidoc import AsciiDocRenderer, asciidoc_escape from .commonmark import CommonMarkRenderer from .docbook import DocBookRenderer, make_xml_id +from .html import HTMLRenderer from .manpage import ManpageRenderer, man_escape +from .manual_structure import XrefTarget from .md import Converter, md_escape, md_make_code from .types import OptionLoc, Option, RenderedOption @@ -240,10 +243,10 @@ def _decl_def_header(self, header: str) -> list[str]: def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: if href is not None: - href = " xlink:href=" + quoteattr(href) + href = " xlink:href=" + xml.quoteattr(href) return [ f"<member><filename{href}>", - escape(name), + xml.escape(name), "</filename></member>" ] @@ -273,8 +276,8 @@ def finalize(self, *, fragment: bool = False) -> str: result += [ "<varlistentry>", # NOTE adding extra spaces here introduces spaces into xref link expansions - (f"<term xlink:href={quoteattr('#' + id)} xml:id={quoteattr(id)}>" + - f"<option>{escape(name)}</option></term>"), + (f"<term xlink:href={xml.quoteattr('#' + id)} xml:id={xml.quoteattr(id)}>" + + f"<option>{xml.escape(name)}</option></term>"), "<listitem>" ] result += opt.lines @@ -524,6 +527,109 @@ def finalize(self) -> str: return "\n".join(result) +class OptionsHTMLRenderer(OptionDocsRestrictions, HTMLRenderer): + # TODO docbook compat. must be removed together with the matching docbook handlers. + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + token.meta['compact'] = False + return super().ordered_list_open(token, tokens, i) + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: + token.meta['compact'] = False + return super().bullet_list_open(token, tokens, i) + def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: + # TODO use token.info. docbook doesn't so we can't yet. + return f'<pre class="programlisting">{html.escape(token.content)}</pre>' + +class HTMLConverter(BaseConverter[OptionsHTMLRenderer]): + __option_block_separator__ = "" + + def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool, + varlist_id: str, id_prefix: str, xref_targets: Mapping[str, XrefTarget]): + super().__init__(revision, markdown_by_default) + self._xref_targets = xref_targets + self._varlist_id = varlist_id + self._id_prefix = id_prefix + self._renderer = OptionsHTMLRenderer(manpage_urls, self._xref_targets) + + def _parallel_render_prepare(self) -> Any: + return (self._renderer._manpage_urls, self._revision, self._markdown_by_default, + self._varlist_id, self._id_prefix, self._xref_targets) + @classmethod + def _parallel_render_init_worker(cls, a: Any) -> HTMLConverter: + return cls(*a) + + def _render_code(self, option: dict[str, Any], key: str) -> list[str]: + if lit := option_is(option, key, 'literalDocBook'): + raise RuntimeError("can't render html in the presence of docbook") + else: + return super()._render_code(option, key) + + def _render_description(self, desc: str | dict[str, Any]) -> list[str]: + if isinstance(desc, str) and not self._markdown_by_default: + raise RuntimeError("can't render html in the presence of docbook") + else: + return super()._render_description(desc) + + def _related_packages_header(self) -> list[str]: + return [ + '<p><span class="emphasis"><em>Related packages:</em></span></p>', + ] + + def _decl_def_header(self, header: str) -> list[str]: + return [ + f'<p><span class="emphasis"><em>{header}:</em></span></p>', + '<table border="0" summary="Simple list" class="simplelist">' + ] + + def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: + if href is not None: + href = f' href="{html.escape(href, True)}"' + return [ + "<tr><td>", + f'<code class="filename"><a class="filename" {href} target="_top">', + f'{html.escape(name)}', + '</a></code>', + "</td></tr>" + ] + + def _decl_def_footer(self) -> list[str]: + return [ "</table>" ] + + def finalize(self) -> str: + result = [] + + result += [ + '<div class="variablelist">', + f'<a id="{html.escape(self._varlist_id, True)}"></a>', + ' <dl class="variablelist">', + ] + + for (name, opt) in self._sorted_options(): + id = make_xml_id(self._id_prefix + name) + target = self._xref_targets[id] + result += [ + '<dt>', + ' <span class="term">', + # docbook compat, these could be one tag + f' <a id="{html.escape(id, True)}"></a><a class="term" href="{target.href()}">' + # no spaces here (and string merging) for docbook output compat + f'<code class="option">{html.escape(name)}</code>', + ' </a>', + ' </span>', + '</dt>', + '<dd>', + ] + result += opt.lines + result += [ + "</dd>", + ] + + result += [ + " </dl>", + "</div>" + ] + + return "\n".join(result) + def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) From feaa97e5dcbb889404442ebe952357ef927d17ca Mon Sep 17 00:00:00 2001 From: pennae <github@quasiparticle.net> Date: Sat, 18 Feb 2023 21:10:42 +0100 Subject: [PATCH 16/18] nixos-render-docs: render directly from file to file this will be necessary for html since there we have to do chunking into multiple files ourselves. writing one file from the caller of the converter and all others from within the converter is unnecessarily spread out, and returning a dict of file names and their contents is not quite as meaningful for docbook (which has only one file to begin with). --- .../src/nixos_render_docs/manual.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 8d7bf4a102f37..27850c0bbd17a 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -27,14 +27,14 @@ class BaseConverter(Converter[md.TR], Generic[md.TR]): _base_paths: list[Path] _current_type: list[TocEntryType] - def convert(self, file: Path) -> str: - self._base_paths = [ file ] + def convert(self, infile: Path, outfile: Path) -> None: + self._base_paths = [ infile ] self._current_type = ['book'] try: - with open(file, 'r') as f: - return self._render(f.read()) + converted = self._render(infile.read_text()) + outfile.write_text(converted) except Exception as e: - raise RuntimeError(f"failed to render manual {file}") from e + raise RuntimeError(f"failed to render manual {infile}") from e def _parse(self, src: str) -> list[Token]: tokens = super()._parse(src) @@ -215,8 +215,7 @@ def _build_cli_db(p: argparse.ArgumentParser) -> None: def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: md = DocBookConverter(json.load(manpage_urls), args.revision) - converted = md.convert(args.infile) - args.outfile.write_text(converted) + md.convert(args.infile, args.outfile) def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) From 36f04733ddc40beb54659f290aaf369d380f312b Mon Sep 17 00:00:00 2001 From: pennae <github@quasiparticle.net> Date: Wed, 15 Feb 2023 16:28:52 +0100 Subject: [PATCH 17/18] nixos-render-docs: add manual html converter this converter is currently supposed to be able to reproduce the docbook-generated html DOMs exactly, though not necessarily the html *files*. it mirrors many docbook behaviours that seem rather odd, such as top-level sections in chapters using the same heading depth as understood by html as their parent chapters do. over time we can hopefully remove all special casing needed to reproduce docbook rendering, but for now at least it doesn't hurt *too* much. --- .../src/nixos_render_docs/__init__.py | 1 - .../src/nixos_render_docs/manual.py | 455 +++++++++++++++++- 2 files changed, 433 insertions(+), 23 deletions(-) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py index 7f7463e5c837c..1c58accb41662 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py @@ -8,7 +8,6 @@ from typing import Any, Dict from .md import Converter -from . import html from . import manual from . import options from . import parallel diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py index 27850c0bbd17a..40dea3c7d1d85 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py @@ -1,19 +1,23 @@ import argparse +import html import json +import re +import xml.sax.saxutils as xml from abc import abstractmethod from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any, cast, ClassVar, Generic, get_args, NamedTuple, Optional, Union -from xml.sax.saxutils import escape, quoteattr import markdown_it from markdown_it.token import Token from . import md, options -from .docbook import DocBookRenderer, Heading -from .manual_structure import check_structure, FragmentType, is_include, TocEntryType -from .md import Converter +from .docbook import DocBookRenderer, Heading, make_xml_id +from .html import HTMLRenderer, UnresolvedXrefError +from .manual_structure import check_structure, FragmentType, is_include, TocEntry, TocEntryType, XrefTarget +from .md import Converter, Renderer +from .utils import Freezeable class BaseConverter(Converter[md.TR], Generic[md.TR]): # per-converter configuration for ns:arg=value arguments to include blocks, following @@ -31,11 +35,16 @@ def convert(self, infile: Path, outfile: Path) -> None: self._base_paths = [ infile ] self._current_type = ['book'] try: - converted = self._render(infile.read_text()) + tokens = self._parse(infile.read_text()) + self._postprocess(infile, outfile, tokens) + converted = self._renderer.render(tokens) outfile.write_text(converted) except Exception as e: raise RuntimeError(f"failed to render manual {infile}") from e + def _postprocess(self, infile: Path, outfile: Path, tokens: Sequence[Token]) -> None: + pass + def _parse(self, src: str) -> list[Token]: tokens = super()._parse(src) check_structure(self._current_type[-1], tokens) @@ -117,12 +126,12 @@ def _parse_options(self, token: Token, block_args: dict[str, str]) -> None: except Exception as e: raise RuntimeError(f"processing options block in line {token.map[0] + 1}") from e -class ManualDocBookRenderer(DocBookRenderer): +class RendererMixin(Renderer): _toplevel_tag: str _revision: str - def __init__(self, toplevel_tag: str, revision: str, manpage_urls: Mapping[str, str]): - super().__init__(manpage_urls) + def __init__(self, toplevel_tag: str, revision: str, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) self._toplevel_tag = toplevel_tag self._revision = revision self.rules |= { @@ -139,20 +148,39 @@ def render(self, tokens: Sequence[Token]) -> str: # generic code is more complicated than it's worth. the checks above have verified # that both titles actually exist. if self._toplevel_tag == 'book': - assert tokens[1].children - assert tokens[4].children - if (maybe_id := cast(str, tokens[0].attrs.get('id', ""))): - maybe_id = "xml:id=" + quoteattr(maybe_id) - return (f'<book xmlns="http://docbook.org/ns/docbook"' - f' xmlns:xlink="http://www.w3.org/1999/xlink"' - f' {maybe_id} version="5.0">' - f' <title>{self.renderInline(tokens[1].children)}' - f' {self.renderInline(tokens[4].children)}' - f' {super().render(tokens[6:])}' - f'') + return self._render_book(tokens) return super().render(tokens) + @abstractmethod + def _render_book(self, tokens: Sequence[Token]) -> str: + raise NotImplementedError() + + @abstractmethod + def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: int) -> str: + raise NotImplementedError() + + @abstractmethod + def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str: + raise NotImplementedError() + +class ManualDocBookRenderer(RendererMixin, DocBookRenderer): + def __init__(self, toplevel_tag: str, revision: str, manpage_urls: Mapping[str, str]): + super().__init__(toplevel_tag, revision, manpage_urls) + + def _render_book(self, tokens: Sequence[Token]) -> str: + assert tokens[1].children + assert tokens[4].children + if (maybe_id := cast(str, tokens[0].attrs.get('id', ""))): + maybe_id = "xml:id=" + xml.quoteattr(maybe_id) + return (f'' + f' {self.renderInline(tokens[1].children)}' + f' {self.renderInline(tokens[4].children)}' + f' {super(DocBookRenderer, self).render(tokens[6:])}' + f'') + def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int) -> tuple[str, dict[str, str]]: (tag, attrs) = super()._heading_tag(token, tokens, i) # render() has already verified that we don't have supernumerary headings and since the @@ -192,10 +220,10 @@ def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int) -> str: def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int) -> str: return "\n" + super().paragraph_close(token, tokens, i) def code_block(self, token: Token, tokens: Sequence[Token], i: int) -> str: - return f"\n{escape(token.content)}" + return f"\n{xml.escape(token.content)}" def fence(self, token: Token, tokens: Sequence[Token], i: int) -> str: - info = f" language={quoteattr(token.info)}" if token.info != "" else "" - return f"\n{escape(token.content)}" + info = f" language={xml.quoteattr(token.info)}" if token.info != "" else "" + return f"\n{xml.escape(token.content)}" class DocBookConverter(BaseConverter[ManualDocBookRenderer]): INCLUDE_ARGS_NS = "docbook" @@ -205,6 +233,366 @@ def __init__(self, manpage_urls: Mapping[str, str], revision: str): self._renderer = ManualDocBookRenderer('book', revision, manpage_urls) +class HTMLParameters(NamedTuple): + generator: str + stylesheets: Sequence[str] + scripts: Sequence[str] + toc_depth: int + chunk_toc_depth: int + +class ManualHTMLRenderer(RendererMixin, HTMLRenderer): + _base_path: Path + _html_params: HTMLParameters + + def __init__(self, toplevel_tag: str, revision: str, html_params: HTMLParameters, + manpage_urls: Mapping[str, str], xref_targets: dict[str, XrefTarget], + base_path: Path): + super().__init__(toplevel_tag, revision, manpage_urls, xref_targets) + self._base_path, self._html_params = base_path, html_params + + def _push(self, tag: str, hlevel_offset: int) -> Any: + result = (self._toplevel_tag, self._headings, self._attrspans, self._hlevel_offset) + self._hlevel_offset += hlevel_offset + self._toplevel_tag, self._headings, self._attrspans = tag, [], [] + return result + + def _pop(self, state: Any) -> None: + (self._toplevel_tag, self._headings, self._attrspans, self._hlevel_offset) = state + + def _render_book(self, tokens: Sequence[Token]) -> str: + assert tokens[4].children + title_id = cast(str, tokens[0].attrs.get('id', "")) + title = self._xref_targets[title_id].title + # subtitles don't have IDs, so we can't use xrefs to get them + subtitle = self.renderInline(tokens[4].children) + + toc = TocEntry.of(tokens[0]) + return "\n".join([ + self._file_header(toc), + '
', + '
', + '
', + f'

{title}

', + f'

{subtitle}

', + '
', + "
", + '
', + self._build_toc(tokens, 0), + super(HTMLRenderer, self).render(tokens[6:]), + '
', + self._file_footer(toc), + ]) + + def _file_header(self, toc: TocEntry) -> str: + prev_link, up_link, next_link = "", "", "" + prev_a, next_a, parent_title = "", "", " " + home = toc.root + if toc.prev: + prev_link = f'' + prev_a = f'Prev' + if toc.parent: + up_link = ( + f'' + ) + if (part := toc.parent) and part.kind != 'book': + assert part.target.title + parent_title = part.target.title + if toc.next: + next_link = f'' + next_a = f'Next' + return "\n".join([ + '', + '', + '', + ' ', + f' {toc.target.title}', + "".join((f'' + for style in self._html_params.stylesheets)), + "".join((f'' + for script in self._html_params.scripts)), + f' ', + f' ', + f' {up_link}{prev_link}{next_link}', + ' ', + ' ', + '
', + ' ', + ' ', + f' ', + ' ', + ' ', + f' ', + f' ', + f' ', + ' ', + '
{toc.target.title}
{prev_a} {parent_title} {next_a}
', + '
', + '
', + ]) + + def _file_footer(self, toc: TocEntry) -> str: + # prev, next = self._get_prev_and_next() + prev_a, up_a, home_a, next_a = "", " ", " ", "" + prev_text, up_text, next_text = "", "", "" + home = toc.root + if toc.prev: + prev_a = f'Prev' + assert toc.prev.target.title + prev_text = toc.prev.target.title + if toc.parent: + home_a = f'Home' + if toc.parent != home: + up_a = f'Up' + if toc.next: + next_a = f'Next' + assert toc.next.target.title + next_text = toc.next.target.title + return "\n".join([ + '
', + '
', + ' ', + ' ', + f' ', + f' ', + f' ', + ' ', + ' ', + f' ', + f' ', + f' ', + ' ', + '
{prev_a} {up_a} {next_a}
{prev_text} {home_a} {next_text}
', + '
', + ' ', + '', + ]) + + def _heading_tag(self, token: Token, tokens: Sequence[Token], i: int) -> str: + if token.tag == 'h1': + return self._toplevel_tag + return super()._heading_tag(token, tokens, i) + def _build_toc(self, tokens: Sequence[Token], i: int) -> str: + toc = TocEntry.of(tokens[i]) + if toc.kind == 'section': + return "" + def walk_and_emit(toc: TocEntry, depth: int) -> list[str]: + if depth <= 0: + return [] + result = [] + for child in toc.children: + result.append( + f'
' + f' ' + f' {child.target.toc_html}' + f' ' + f'
' + ) + # we want to look straight through parts because docbook-xsl does too, but it + # also makes for more uesful top-level tocs. + next_level = walk_and_emit(child, depth - (0 if child.kind == 'part' else 1)) + if next_level: + result.append(f'
{"".join(next_level)}
') + return result + toc_depth = ( + self._html_params.chunk_toc_depth + if toc.starts_new_chunk and toc.kind != 'book' + else self._html_params.toc_depth + ) + if not (items := walk_and_emit(toc, toc_depth)): + return "" + return ( + f'
' + f'

Table of Contents

' + f'
' + f' {"".join(items)}' + f'
' + f'
' + ) + + def _make_hN(self, level: int) -> tuple[str, str]: + # for some reason chapters don't increase the hN nesting count in docbook xslts. duplicate + # this for consistency. + if self._toplevel_tag == 'chapter': + level -= 1 + # TODO docbook compat. these are never useful for us, but not having them breaks manual + # compare workflows while docbook is still allowed. + style = "" + if level + self._hlevel_offset < 3 \ + and (self._toplevel_tag == 'section' or (self._toplevel_tag == 'chapter' and level > 0)): + style = "clear: both" + tag, hstyle = super()._make_hN(max(1, level)) + return tag, style + + def _included_thing(self, tag: str, token: Token, tokens: Sequence[Token], i: int) -> str: + outer, inner = [], [] + # since books have no non-include content the toplevel book wrapper will not count + # towards nesting depth. other types will have at least a title+id heading which + # *does* count towards the nesting depth. chapters give a -1 to included sections + # mirroring the special handing in _make_hN. sigh. + hoffset = ( + 0 if not self._headings + else self._headings[-1].level - 1 if self._toplevel_tag == 'chapter' + else self._headings[-1].level + ) + outer.append(self._maybe_close_partintro()) + into = token.meta['include-args'].get('into-file') + fragments = token.meta['included'] + state = self._push(tag, hoffset) + if into: + toc = TocEntry.of(fragments[0][0][0]) + inner.append(self._file_header(toc)) + # we do not set _hlevel_offset=0 because docbook doesn't either. + else: + inner = outer + for included, path in fragments: + try: + inner.append(self.render(included)) + except Exception as e: + raise RuntimeError(f"rendering {path}") from e + if into: + inner.append(self._file_footer(toc)) + (self._base_path / into).write_text("".join(inner)) + self._pop(state) + return "".join(outer) + + def included_options(self, token: Token, tokens: Sequence[Token], i: int) -> str: + conv = options.HTMLConverter(self._manpage_urls, self._revision, False, + token.meta['list-id'], token.meta['id-prefix'], + self._xref_targets) + conv.add_options(token.meta['source']) + return conv.finalize() + +def _to_base26(n: int) -> str: + return (_to_base26(n // 26) if n > 26 else "") + chr(ord("A") + n % 26) + +class HTMLConverter(BaseConverter[ManualHTMLRenderer]): + INCLUDE_ARGS_NS = "html" + INCLUDE_FRAGMENT_ALLOWED_ARGS = { 'into-file' } + + _revision: str + _html_params: HTMLParameters + _manpage_urls: Mapping[str, str] + _xref_targets: dict[str, XrefTarget] + _redirection_targets: set[str] + _appendix_count: int = 0 + + def _next_appendix_id(self) -> str: + self._appendix_count += 1 + return _to_base26(self._appendix_count - 1) + + def __init__(self, revision: str, html_params: HTMLParameters, manpage_urls: Mapping[str, str]): + super().__init__() + self._revision, self._html_params, self._manpage_urls = revision, html_params, manpage_urls + self._xref_targets = {} + self._redirection_targets = set() + # renderer not set on purpose since it has a dependency on the output path! + + def convert(self, infile: Path, outfile: Path) -> None: + self._renderer = ManualHTMLRenderer('book', self._revision, self._html_params, + self._manpage_urls, self._xref_targets, outfile.parent) + super().convert(infile, outfile) + + def _parse(self, src: str) -> list[Token]: + tokens = super()._parse(src) + for token in tokens: + if not token.type.startswith('included_') \ + or not (into := token.meta['include-args'].get('into-file')): + continue + assert token.map + if len(token.meta['included']) == 0: + raise RuntimeError(f"redirection target {into} in line {token.map[0] + 1} is empty!") + # we use blender-style //path to denote paths relative to the origin file + # (usually index.html). this makes everything a lot easier and clearer. + if not into.startswith("//") or '/' in into[2:]: + raise RuntimeError(f"html:into-file must be a relative-to-origin //filename", into) + into = token.meta['include-args']['into-file'] = into[2:] + if into in self._redirection_targets: + raise RuntimeError(f"redirection target {into} in line {token.map[0] + 1} is already in use") + self._redirection_targets.add(into) + return tokens + + # xref | (id, type, heading inlines, file, starts new file) + def _collect_ids(self, tokens: Sequence[Token], target_file: str, typ: str, file_changed: bool + ) -> list[XrefTarget | tuple[str, str, Token, str, bool]]: + result: list[XrefTarget | tuple[str, str, Token, str, bool]] = [] + # collect all IDs and their xref substitutions. headings are deferred until everything + # has been parsed so we can resolve links in headings. if that's even used anywhere. + for (i, bt) in enumerate(tokens): + if bt.type == 'heading_open' and (id := cast(str, bt.attrs.get('id', ''))): + result.append((id, typ if bt.tag == 'h1' else 'section', tokens[i + 1], target_file, + i == 0 and file_changed)) + elif bt.type == 'included_options': + id_prefix = bt.meta['id-prefix'] + for opt in bt.meta['source'].keys(): + id = make_xml_id(f"{id_prefix}{opt}") + name = html.escape(opt) + result.append(XrefTarget(id, f'{name}', name, None, target_file)) + elif bt.type.startswith('included_'): + sub_file = bt.meta['include-args'].get('into-file', target_file) + subtyp = bt.type.removeprefix('included_').removesuffix('s') + for si, (sub, _path) in enumerate(bt.meta['included']): + result += self._collect_ids(sub, sub_file, subtyp, si == 0 and sub_file != target_file) + elif bt.type == 'inline': + assert bt.children + result += self._collect_ids(bt.children, target_file, typ, False) + elif id := cast(str, bt.attrs.get('id', '')): + # anchors and examples have no titles we could use, but we'll have to put + # *something* here to communicate that there's no title. + result.append(XrefTarget(id, "???", None, None, target_file)) + return result + + def _render_xref(self, id: str, typ: str, inlines: Token, path: str, drop_fragment: bool) -> XrefTarget: + assert inlines.children + title_html = self._renderer.renderInline(inlines.children) + if typ == 'appendix': + # NOTE the docbook compat is strong here + n = self._next_appendix_id() + prefix = f"Appendix\u00A0{n}.\u00A0" + # HACK for docbook compat: prefix the title inlines with appendix id if + # necessary. the alternative is to mess with titlepage rendering in headings, + # which seems just a lot worse than this + prefix_tokens = [Token(type='text', tag='', nesting=0, content=prefix)] + inlines.children = prefix_tokens + list(inlines.children) + title = prefix + title_html + toc_html = f"{n}. {title_html}" + title_html = f"Appendix {n}" + else: + toc_html, title = title_html, title_html + title_html = ( + f"{title_html}" + if typ == 'chapter' + else title_html if typ in [ 'book', 'part' ] + else f'the section called “{title_html}”' + ) + return XrefTarget(id, title_html, toc_html, re.sub('<.*?>', '', title), path, drop_fragment) + + def _postprocess(self, infile: Path, outfile: Path, tokens: Sequence[Token]) -> None: + xref_queue = self._collect_ids(tokens, outfile.name, 'book', True) + + failed = False + deferred = [] + while xref_queue: + for item in xref_queue: + try: + target = item if isinstance(item, XrefTarget) else self._render_xref(*item) + except UnresolvedXrefError as e: + if failed: + raise + deferred.append(item) + continue + + if target.id in self._xref_targets: + raise RuntimeError(f"found duplicate id #{target.id}") + self._xref_targets[target.id] = target + if len(deferred) == len(xref_queue): + failed = True # do another round and report the first error + xref_queue = deferred + + TocEntry.collect_and_link(self._xref_targets, tokens) + + def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) @@ -212,17 +600,40 @@ def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('infile', type=Path) p.add_argument('outfile', type=Path) +def _build_cli_html(p: argparse.ArgumentParser) -> None: + p.add_argument('--manpage-urls', required=True) + p.add_argument('--revision', required=True) + p.add_argument('--generator', default='nixos-render-docs') + p.add_argument('--stylesheet', default=[], action='append') + p.add_argument('--script', default=[], action='append') + p.add_argument('--toc-depth', default=1, type=int) + p.add_argument('--chunk-toc-depth', default=1, type=int) + p.add_argument('infile', type=Path) + p.add_argument('outfile', type=Path) + def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: md = DocBookConverter(json.load(manpage_urls), args.revision) md.convert(args.infile, args.outfile) +def _run_cli_html(args: argparse.Namespace) -> None: + with open(args.manpage_urls, 'r') as manpage_urls: + md = HTMLConverter( + args.revision, + HTMLParameters(args.generator, args.stylesheet, args.script, args.toc_depth, + args.chunk_toc_depth), + json.load(manpage_urls)) + md.convert(args.infile, args.outfile) + def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) _build_cli_db(formats.add_parser('docbook')) + _build_cli_html(formats.add_parser('html')) def run_cli(args: argparse.Namespace) -> None: if args.format == 'docbook': _run_cli_db(args) + elif args.format == 'html': + _run_cli_html(args) else: raise RuntimeError('format not hooked up', args) From 54f4992e80df3b8007dd8235c9f8d6ec77eef16f Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 20 Feb 2023 18:55:15 +0100 Subject: [PATCH 18/18] nixos/manual: render html with nixos-render-doc if !allowDocBook this reproduces the docbook-generated html manual exactly enough to appease the compare workflows while we still support both toolchains. it's also a lot faster than the docbook toolchain, rendering the entire html manual in about two seconds on this machine (while docbook needs about 20). --- nixos/doc/manual/default.nix | 90 ++++++++++++++++++++++++------------ nixos/doc/manual/manual.md | 5 +- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 714b3efca20aa..2e07edd61c2a7 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -135,28 +135,32 @@ let } ''; + prepareManualFromMD = '' + cp -r --no-preserve=all $inputs/* . + + substituteInPlace ./manual.md \ + --replace '@NIXOS_VERSION@' "${version}" + substituteInPlace ./configuration/configuration.md \ + --replace \ + '@MODULE_CHAPTERS@' \ + ${lib.escapeShellArg (lib.concatMapStringsSep "\n" (p: "${p.value}") config.meta.doc)} + substituteInPlace ./nixos-options.md \ + --replace \ + '@NIXOS_OPTIONS_JSON@' \ + ${optionsDoc.optionsJSON}/share/doc/nixos/options.json + substituteInPlace ./development/writing-nixos-tests.section.md \ + --replace \ + '@NIXOS_TEST_OPTIONS_JSON@' \ + ${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json + ''; + manual-combined = runCommand "nixos-manual-combined" { inputs = lib.sourceFilesBySuffices ./. [ ".xml" ".md" ]; nativeBuildInputs = [ pkgs.nixos-render-docs pkgs.libxml2.bin pkgs.libxslt.bin ]; meta.description = "The NixOS manual as plain docbook XML"; } '' - cp -r --no-preserve=all $inputs/* . - - substituteInPlace ./manual.md \ - --replace '@NIXOS_VERSION@' "${version}" - substituteInPlace ./configuration/configuration.md \ - --replace \ - '@MODULE_CHAPTERS@' \ - ${lib.escapeShellArg (lib.concatMapStringsSep "\n" (p: "${p.value}") config.meta.doc)} - substituteInPlace ./nixos-options.md \ - --replace \ - '@NIXOS_OPTIONS_JSON@' \ - ${optionsDoc.optionsJSON}/share/doc/nixos/options.json - substituteInPlace ./development/writing-nixos-tests.section.md \ - --replace \ - '@NIXOS_TEST_OPTIONS_JSON@' \ - ${testOptionsDoc.optionsJSON}/share/doc/nixos/options.json + ${prepareManualFromMD} nixos-render-docs -j $NIX_BUILD_CORES manual docbook \ --manpage-urls ${manpageUrls} \ @@ -193,7 +197,14 @@ in rec { # Generate the NixOS manual. manualHTML = runCommand "nixos-manual-html" - { nativeBuildInputs = [ buildPackages.libxml2.bin buildPackages.libxslt.bin ]; + { nativeBuildInputs = + if allowDocBook then [ + buildPackages.libxml2.bin + buildPackages.libxslt.bin + ] else [ + buildPackages.nixos-render-docs + ]; + inputs = lib.optionals (! allowDocBook) (lib.sourceFilesBySuffices ./. [ ".md" ]); meta.description = "The NixOS manual in HTML format"; allowedReferences = ["out"]; } @@ -201,23 +212,44 @@ in rec { # Generate the HTML manual. dst=$out/share/doc/nixos mkdir -p $dst - xsltproc \ - ${manualXsltprocOptions} \ - --stringparam id.warnings "1" \ - --nonet --output $dst/ \ - ${docbook_xsl_ns}/xml/xsl/docbook/xhtml/chunktoc.xsl \ - ${manual-combined}/manual-combined.xml \ - |& tee xsltproc.out - grep "^ID recommended on" xsltproc.out &>/dev/null && echo "error: some IDs are missing" && false - rm xsltproc.out - - mkdir -p $dst/images/callouts - cp ${docbook_xsl_ns}/xml/xsl/docbook/images/callouts/*.svg $dst/images/callouts/ cp ${../../../doc/style.css} $dst/style.css cp ${../../../doc/overrides.css} $dst/overrides.css cp -r ${pkgs.documentation-highlighter} $dst/highlightjs + ${if allowDocBook then '' + xsltproc \ + ${manualXsltprocOptions} \ + --stringparam id.warnings "1" \ + --nonet --output $dst/ \ + ${docbook_xsl_ns}/xml/xsl/docbook/xhtml/chunktoc.xsl \ + ${manual-combined}/manual-combined.xml \ + |& tee xsltproc.out + grep "^ID recommended on" xsltproc.out &>/dev/null && echo "error: some IDs are missing" && false + rm xsltproc.out + + mkdir -p $dst/images/callouts + cp ${docbook_xsl_ns}/xml/xsl/docbook/images/callouts/*.svg $dst/images/callouts/ + '' else '' + ${prepareManualFromMD} + + # TODO generator is set like this because the docbook/md manual compare workflow will + # trigger if it's different + nixos-render-docs -j $NIX_BUILD_CORES manual html \ + --manpage-urls ${manpageUrls} \ + --revision ${lib.escapeShellArg revision} \ + --generator "DocBook XSL Stylesheets V${docbook_xsl_ns.version}" \ + --stylesheet style.css \ + --stylesheet overrides.css \ + --stylesheet highlightjs/mono-blue.css \ + --script ./highlightjs/highlight.pack.js \ + --script ./highlightjs/loader.js \ + --toc-depth 1 \ + --chunk-toc-depth 1 \ + ./manual.md \ + $dst/index.html + ''} + mkdir -p $out/nix-support echo "nix-build out $out" >> $out/nix-support/hydra-build-products echo "doc manual $dst" >> $out/nix-support/hydra-build-products diff --git a/nixos/doc/manual/manual.md b/nixos/doc/manual/manual.md index 1972eaeda8728..8cb766eeccf64 100644 --- a/nixos/doc/manual/manual.md +++ b/nixos/doc/manual/manual.md @@ -47,7 +47,10 @@ development/development.md contributing-to-this-manual.chapter.md ``` -```{=include=} appendix +```{=include=} appendix html:into-file=//options.html nixos-options.md +``` + +```{=include=} appendix html:into-file=//release-notes.html release-notes/release-notes.md ```