diff --git a/MANIFEST.in b/MANIFEST.in index f7ce279..304cc60 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,9 @@ include adi_doctools/miscellaneous/*.js include adi_doctools/miscellaneous/*.svg include adi_doctools/miscellaneous/*.css +include adi_doctools/miscellaneous/sphinx-template/conf.py +include adi_doctools/miscellaneous/sphinx-template/Makefile +include adi_doctools/miscellaneous/sphinx-template/index.rst include adi_doctools/theme/cosmic/static/*.umd.js include adi_doctools/theme/cosmic/static/*.js.map include adi_doctools/theme/cosmic/static/*.min.css diff --git a/adi_doctools/__init__.py b/adi_doctools/__init__.py index 8934333..f2bafe6 100644 --- a/adi_doctools/__init__.py +++ b/adi_doctools/__init__.py @@ -1,14 +1,16 @@ import os -from .theme import navigation_tree, get_pygments_theme, write_pygments_css, wrap_elements +from .theme import (navigation_tree, get_pygments_theme, + write_pygments_css, wrap_elements) from .theme import setup as theme_setup, names as theme_names from .directive import setup as directive_setup from .role import setup as role_setup -__version__ = "0.3.2" +__version__ = "0.3.3" dft_is_system_top = False + def get_navigation_tree(context, repo): # The navigation tree, generated from the sphinx-provided ToC tree. if "toctree" in context: @@ -24,8 +26,11 @@ def get_navigation_tree(context, repo): return navigation_tree(toctree_html, context['content_root'], repo) + def html_page_context(app, pagename, templatename, context, doctree): - context["sidebar_tree"], context["subdomain_tree"] = get_navigation_tree(context, app.env.config.repository); + ret = get_navigation_tree(context, app.env.config.repository) + context["sidebar_tree"], context["subdomain_tree"] = ret + def builder_inited(app): if app.builder.format == 'html': @@ -35,16 +40,17 @@ def builder_inited(app): # Add bundled JavaScript if current theme is from this extension. if app.env.config.html_theme in theme_names: app.add_js_file("app.umd.js", priority=500, defer="") - app.config.values["html_permalinks_icon"] = ("#", *app.config.values["html_permalinks_icon"][1:]) - builder = app.builder + conf_ = ("#", *app.config.values["html_permalinks_icon"][1:]) + app.config.values["html_permalinks_icon"] = conf_ get_pygments_theme(app) else: app.add_css_file("third-party.css", priority=500, defer="") + def build_finished(app, exc): """ Injects assets. - It's a last resort solution, prefer adding the files to the theme static folder, + It's the last resort, prefer adding the files to the theme static folder, since they are auto copied to _assets by sphinx. It is used for: * Add the Author Mode's pooling mode's JavaScript. @@ -54,7 +60,8 @@ def build_finished(app, exc): def copy_asset(app, uri): from sphinx.util.fileutil import copy_asset_file - src_uri = os.path.join(os.path.dirname(__file__), f"miscellaneous/{uri}") + src_uri = os.path.join(os.path.dirname(__file__), + f"miscellaneous/{uri}") build_uri = os.path.join(app.builder.outdir, '_static', uri) copy_asset_file(src_uri, build_uri) @@ -69,6 +76,7 @@ def copy_asset(app, uri): copy_asset(app, "esd-warning.svg") + def setup(app): for setup in theme_setup: setup(app) diff --git a/adi_doctools/cli/__init__.py b/adi_doctools/cli/__init__.py index 9c7cfab..93f80da 100644 --- a/adi_doctools/cli/__init__.py +++ b/adi_doctools/cli/__init__.py @@ -2,6 +2,7 @@ from .author_mode import author_mode from .hdl_render import hdl_render +from .aggregate import aggregate @click.group() def entry_point(): @@ -12,7 +13,8 @@ def entry_point(): commands = [ author_mode, - hdl_render + hdl_render, + aggregate ] for cmd in commands: diff --git a/adi_doctools/cli/aggregate.py b/adi_doctools/cli/aggregate.py new file mode 100644 index 0000000..cd857df --- /dev/null +++ b/adi_doctools/cli/aggregate.py @@ -0,0 +1,393 @@ +import os +import click +import subprocess +import re + +remote = "git@github.com:analogdevicesinc/{}.git" + +lut = { + 'hdl': { + 'doc_folder': 'docs', + 'extra': ( + # cwd # cmd # no_parallel + "library", ["make", "all"], False + ), + 'name': 'HDL', + 'branch': 'mv-doctools' + }, + 'no-os': { + 'doc_folder': 'doc/sphinx', + 'name': 'no-OS', + 'branch': 'main' + }, + 'documentation': { + 'doc_folder': 'docs', + 'name': 'System Level', + 'branch': 'main' + }, + 'doctools': { + 'doc_folder': 'docs', + 'name': 'Doc Tools', + 'branch': 'main' + } +} + + +def patch_index(name, sourcedir, indexfile, dry_run): + file = os.path.join(sourcedir, 'index.rst') + toctree = [] + + with open(file, "r") as f: + data = f.readlines() + if ".. toctree::\n" not in data: + return + in_toc = False + for i in range(0, len(data)): + if in_toc: + if data[i][0:12] == ' :caption:': + data[i] = "" + continue + + if data[i][0:3] == ' ' and data[i][0:4] != ' :': + pos = data[i].find('<') + if pos == -1: + data[i] = f" {name}/{data[i][3:]}" + else: + data[i] = f"{data[i][:pos+1]}{name}/{data[i][pos+1:]}" + + if data[i][0:3] != ' ' and data[i] != '\n': + toctree[-1] = [toctree[-1][0], i - 1] + if data[i] == ".. toctree::\n": + toctree.append([i, i]) + else: + in_toc = False + continue + else: + if data[i] == ".. toctree::\n": + toctree.append([i, i]) + in_toc = True + + if dry_run: + return + + with open(indexfile, "r") as f: + data_ = f.readlines() + # Find end of toctree + if ".. toctree::\n" in data_: + i = data_.index(".. toctree::\n") + for i in range(i + 1, len(data_)): + if data_[i][0:3] != ' ' and data_[i] != '\n': + break + else: + i = len(data_) + + header = data_[:i] + body = data_[i:] + + if len(toctree) > 1: + click.echo(click.style(f"Repo {name} containes multiple toctrees!", + fg='red')) + for tc in toctree: + header.append(".. toctree::\n") + header.append(f" :caption: {lut[name]['name']}\n") + header.extend(data[tc[0]+1:tc[1]]) + header.append('\n') + + header.extend(body) + + with open(indexfile, "w") as f: + for line in header: + f.write(line) + + +def get_sphinx_dirs(cwd) -> tuple[bool, str, str]: + mk = os.path.join(cwd, 'Makefile') + if not os.path.isfile(mk): + click.echo(click.style(f"{mk} does not exist, skipped!", fg='red')) + return (True, '', '') + + with open(mk, 'r') as f: + data = f.read() + builddir_ = re.search(r'^BUILDDIR\s*=\s*(.*)$', data, re.MULTILINE) + sourcedir_ = re.search(r'^SOURCEDIR\s*=\s*(.*)$', data, re.MULTILINE) + builddir_ = builddir_.group(1).strip() if builddir_ else None + sourcedir_ = sourcedir_.group(1).strip() if sourcedir_ else None + if builddir_ is None or sourcedir_ is None: + click.echo(click.style(f"Failed to parse {mk}, skipped!", fg='red')) + return (True, '', '') + builddir = os.path.join(cwd, f"{builddir_}/html") + sourcedir = os.path.join(cwd, sourcedir_) + if not os.path.isdir(sourcedir): + click.echo(click.style(f"Parsed {sourcedir} does not exist, skipped!", + fg='red')) + return (True, '', '') + + return [False, builddir, sourcedir] + + +def do_extra_steps(repo_dir, no_parallel, dry_run): + for l_ in lut: + if 'extra' in lut[l_]: + cwd, cmd, no_p = lut[l_]['extra'] + cwd = os.path.join(repo_dir, f"{l_}/{cwd}") + nproc = 1 if no_parallel or no_p else 4 + if not dry_run: + if cmd[0] == 'make': + subprocess.call(f"cd {cwd}; {' '.join(cmd)} -j{nproc}", + shell=True) + else: + # Unknown cmd, do not append nproc + subprocess.call(f"cd {cwd}; {' '.join(cmd)}", + shell=True) + else: + if cmd[0] == 'make': + click.echo(f"cd {cwd}; {' '.join(cmd)} -j{nproc}") + else: + click.echo(f"cd {cwd}; {' '.join(cmd)}") + + +def gen_symbolic_doc(repo_dir, no_parallel, dry_run): + p = [] + mk = [] + for r in lut: + sphinx_cmd = ["make", "html"] + cwd = os.path.join(repo_dir, f"{r}/{lut[r]['doc_folder']}") + mk.append(get_sphinx_dirs(cwd)) + if mk[-1][0]: + continue + if not dry_run: + p__ = subprocess.Popen(sphinx_cmd, cwd=cwd) + p__.wait() if no_parallel else p.append(p__) + else: + click.echo(f"cd {cwd}; {' '.join(sphinx_cmd)}") + for p_ in p: + p_.wait() + + d_ = os.path.abspath(os.path.join(repo_dir, os.pardir)) + out = os.path.join(d_, 'html') + if not dry_run: + os.mkdir(out) + else: + click.echo(f"mkdir {out}") + for r, m in zip(lut, mk): + if m[0]: + continue + d_ = os.path.join(out, r) + cp_cmd = ["cp", "-r", m[1], d_] + if not dry_run: + p__ = subprocess.Popen(cp_cmd) + p__.wait() + else: + click.echo(' '.join(cp_cmd)) + + +def gen_monolithic_doc(repo_dir, no_parallel, dry_run): + d_ = os.path.abspath(os.path.join(repo_dir, os.pardir)) + docs_dir = os.path.join(d_, 'docs') + indexfile = os.path.join(docs_dir, 'index.rst') + if os.path.isdir(docs_dir): + cmd = f"rm -r {docs_dir}" + if not dry_run: + subprocess.call(cmd, shell=True) + else: + click.echo(cmd) + # Copy template + src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir)) + sphinx_template = os.path.join(src_dir, 'miscellaneous/sphinx-template') + cp_cmd = f"cp -r {sphinx_template} {docs_dir}" + if not dry_run: + subprocess.run(cp_cmd, shell=True) + else: + click.echo(cp_cmd) + + mk = [] + for r in lut: + cwd = os.path.join(repo_dir, f"{r}/{lut[r]['doc_folder']}") + mk.append(get_sphinx_dirs(cwd)) + if mk[-1][0]: + continue + if not dry_run: + os.mkdir(os.path.join(docs_dir, r)) + else: + click.echo(f"mkdir {os.path.join(docs_dir, r)}") + cp_cmd = f"""\ + for dir in */; do + dir="${{dir%/}}" + if [ "$dir" != "_build" ] && [ "$dir" != "extensions" ]; then + cp -r $dir {d_}/docs/{r} + fi + done + for file in *.rst; do + cp $file {d_}/docs/{r} + done\ + """ + cwd = mk[-1][2] + if not dry_run: + subprocess.run(cp_cmd, shell=True, cwd=cwd) + else: + click.echo(f"cd {cwd}; {cp_cmd}") + + # Prefixes references with repo name, expect already external + # references :ref:`repo:str` + cwd = f"{d_}/docs/{r}" + patch_cmd = """\ + # Patch :ref:`str` into :ref:`{r} str` + find . -type f -exec sed -i -E \ + "s/(:ref:\\`)([^<>:]+)(\\`)/\\1{r} \\2\\3/g" {{}} \\; + # Patch:ref:`Title ` into :ref:`Title <{r} str>` + find . -type f -exec sed -i -E \ + "s/(:ref:\\`)([^<]+)( <)([^:>]+)(>)/\\1\\2\\3{r} \\4\\5/g" {{}} \\; + # Patch ^.. _str:$ into .. _{r} str: + find . -type f -exec sed -i -E \ + "s/^(.. _)([^:]+)(:)\\$/\\1{r} \\2\\3/g" {{}} \\;\ + """.format(r=r) + if not dry_run: + subprocess.run(patch_cmd, shell=True, cwd=cwd) + else: + click.echo(f"cd {cwd}; {patch_cmd}") + + # Patch includes outside the docs source, + # e.g. no-OS include README.rst + depth = '../' * (3 + lut[r]['doc_folder'].count('/')) + include_cmd = """ + find . -type f -exec sed -i -E \ + "s|^(.. include:: )({depth})(.*)|\\1../../../repos/{r}/\\3|g" {{}} \\;\ + """.format(r=r, depth=depth) + if not dry_run: + subprocess.run(include_cmd, shell=True, cwd=cwd) + else: + click.echo(f"cd {cwd}; {include_cmd}") + + patch_index(r, mk[-1][2], indexfile, dry_run) + + # Convert external references into local prefixed + cwd = docs_dir + for r in lut: + ref_cmd = """\ + find . -type f -exec sed -i "s|ref:\\`{r}:|ref:\\`{r} |g" {{}} \\;\ + """.format(r=r) + if not dry_run: + subprocess.run(ref_cmd, shell=True, cwd=cwd) + else: + click.echo(f"cd {cwd}; {ref_cmd}") + ref_cmd = """\ + find . -type f -exec sed -i "s|<|<|g" {} \\; + """ + if not dry_run: + subprocess.run(ref_cmd, shell=True, cwd=cwd) + else: + click.echo(f"cd {cwd}; {ref_cmd}") + + sphinx_cmd = ["make", "html"] + cwd = docs_dir + if not dry_run: + subprocess.run(sphinx_cmd, cwd=cwd) + else: + click.echo(f"cd {cwd}; {' '.join(sphinx_cmd)}") + + cp_cmd = ["cp", "-r", '_build/html', '../html_mono'] + if not dry_run: + p__ = subprocess.Popen(cp_cmd, cwd=cwd) + p__.wait() + else: + click.echo(f"cd {cwd}; {' '.join(cp_cmd)}") + + +@click.command() +@click.option( + '--directory', + '-d', + is_flag=False, + type=click.Path(exists=False), + default=None, + required=True, + help="Path to create aggregated output." +) +@click.option( + '--symbolic', + '-s', + is_flag=True, + default=False, + help="Keep each repo doc independent." +) +@click.option( + '--extra', + '-t', + is_flag=True, + default=False, + help="Compile extra features." +) +@click.option( + '--no-parallel', + '-t', + is_flag=True, + default=False, + help="Run all steps in sequence." +) +@click.option( + '--dry-run', + '-n', + is_flag=True, + default=False, + help="Don't actually run; just print them." +) +def aggregate(directory, symbolic, extra, no_parallel, dry_run): + """ + Creates an aggregated documentation out of every repo documentation, + by default, will conjoin/patch each into a single Sphinx build. + """ + directory = os.path.abspath(directory) + + if not extra: + click.echo("Extra features disabled, use --extra to enable.") + + repos_dir = os.path.join(directory, 'repos') + if not dry_run: + if not os.path.isdir(directory): + os.mkdir(directory) + if not os.path.isdir(repos_dir): + os.mkdir(repos_dir) + + d = 'html' if symbolic else 'html_mono' + d__ = os.path.join(directory, d) + if os.path.isdir(d__): + cmd = f"rm -r {d__}" + if not dry_run: + subprocess.call(cmd, shell=True) + else: + click.echo(cmd) + + p = [] + for r in lut: + r_ = remote.format(r) + cwd = os.path.join(repos_dir, r) + if not os.path.isdir(cwd): + git_cmd = ["git", "clone", r_, "--depth=1", "-b", + lut[r]['branch'], '--', cwd] + if not dry_run: + p__ = subprocess.Popen(git_cmd) + p__.wait() if no_parallel else p.append(p__) + else: + click.echo(' '.join(git_cmd)) + else: + git_cmd = ["git", "pull"] + if not dry_run: + p__ = subprocess.Popen(git_cmd, cwd=cwd) + p__.wait() if no_parallel else p.append(p__) + else: + click.echo(f"cd {cwd}; {' '.join(git_cmd)}") + for p_ in p: + p_.wait() + + if extra: + do_extra_steps(repos_dir, no_parallel, dry_run) + + if symbolic: + gen_symbolic_doc(repos_dir, no_parallel, dry_run) + else: + gen_monolithic_doc(repos_dir, no_parallel, dry_run) + + type_ = "symbolic" if symbolic else "monolithic" + out_ = "html" if symbolic else "html_mono" + click.echo(f"Done, {type_} documentation written to {directory}/{out_}") diff --git a/adi_doctools/cli/author_mode.py b/adi_doctools/cli/author_mode.py index d731fbf..41ccb48 100644 --- a/adi_doctools/cli/author_mode.py +++ b/adi_doctools/cli/author_mode.py @@ -4,12 +4,12 @@ import importlib import datetime -error_msg = { - 'no_mk': 'f"File Makefile not found, is {directory} a docs folder?"', - 'inv_mk': 'f"Failed parse Makefile, is {directory} a docs folder?"', - 'inv_f': 'f"Could not find {f}, check rollup output."', - 'inv_bdir': 'f"Could not find BUILDDIR {builddir}."', - 'inv_srcdir': 'f"Could not find SOURCEDIR {sourcedir}."' +string = { + 'no_mk': "File Makefile not found, is {} a docs folder?", + 'inv_mk': "Failed parse Makefile, is {} a docs folder?", + 'inv_f': "Could not find {}, check rollup output.", + 'inv_bdir': "Could not find BUILDDIR {}.", + 'inv_srcdir': "Could not find SOURCEDIR {}." } # Hall of shame of poorly managed artifacts @@ -23,6 +23,7 @@ is_flag=False, type=click.Path(exists=True), default=None, + required=True, help="Path to the docs folder with the Makefile." ) @click.option( @@ -77,21 +78,21 @@ def author_mode(directory, port, dev, no_selenium): def symbolic_assert(file, msg): if not os.path.isfile(file): - click.echo(msg) + click.echo(msg.format(file)) return True else: return False def dir_assert(file, msg): if not os.path.isdir(file): - click.echo(msg) + click.echo(msg.format(file)) return True else: return False directory = os.path.abspath(directory) makefile = os.path.join(directory, 'Makefile') - if symbolic_assert(makefile, eval(error_msg['no_mk'])): + if symbolic_assert(makefile, string['no_mk']): return # Get builddir and sourcedir, to ensure working with any doc @@ -103,11 +104,11 @@ def dir_assert(file, msg): builddir_ = builddir_.group(1).strip() if builddir_ else None sourcedir_ = sourcedir_.group(1).strip() if sourcedir_ else None if builddir_ is None or sourcedir_ is None: - click.echo(eval(error_msg['inv_mk'])) + click.echo(string['inv_mk'].format(directory)) return builddir = os.path.join(directory, f"{builddir_}/html") sourcedir = os.path.join(directory, sourcedir_) - if dir_assert(sourcedir, eval(error_msg['inv_srcdir'])): + if dir_assert(sourcedir, string['inv_srcdir']): return devpool_js = "ADOC_DEVPOOL= " if not with_selenium else "" @@ -145,7 +146,7 @@ def dir_assert(file, msg): subprocess.call(f"{rollup_node_file} -c {rollup_ci_file}", shell=True, cwd=par_dir) for f in w_files: - if symbolic_assert(f, eval(error_msg['inv_f'])): + if symbolic_assert(f, string['inv_f']): return # Build doc the first time diff --git a/adi_doctools/cli/hdl_render.py b/adi_doctools/cli/hdl_render.py index 861066f..d08a2e8 100644 --- a/adi_doctools/cli/hdl_render.py +++ b/adi_doctools/cli/hdl_render.py @@ -1,8 +1,7 @@ import os import click -from lxml import etree -from ..tool.hdl_parser import parse_hdl_component, parse_hdl_regmap +from ..tool.hdl_parser import parse_hdl_component from ..tool.hdl_render import hdl_component @click.command() diff --git a/adi_doctools/miscellaneous/sphinx-template/Makefile b/adi_doctools/miscellaneous/sphinx-template/Makefile new file mode 100755 index 0000000..955dbcd --- /dev/null +++ b/adi_doctools/miscellaneous/sphinx-template/Makefile @@ -0,0 +1,13 @@ +SHELL = /bin/bash +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/adi_doctools/miscellaneous/sphinx-template/conf.py b/adi_doctools/miscellaneous/sphinx-template/conf.py new file mode 100755 index 0000000..83d8881 --- /dev/null +++ b/adi_doctools/miscellaneous/sphinx-template/conf.py @@ -0,0 +1,30 @@ +# -- Project information ----------------------------------------------------- + +project = 'ADI Documentation' +copyright = '2024, Analog Devices Inc.' +author = 'Analog Devices Inc.' + +# -- General configuration --------------------------------------------------- + +extensions = [ + "adi_doctools", +] + +needs_extensions = { + 'adi_doctools':'0.3' +} + +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +source_suffix = '.rst' + +# -- Custom extensions configuration ------------------------------------------- + +is_system_top = True + +# -- Options for HTML output -------------------------------------------------- + +html_theme = 'cosmic' + +html_theme_options = { + "no_index": True +} diff --git a/adi_doctools/miscellaneous/sphinx-template/index.rst b/adi_doctools/miscellaneous/sphinx-template/index.rst new file mode 100755 index 0000000..3f36509 --- /dev/null +++ b/adi_doctools/miscellaneous/sphinx-template/index.rst @@ -0,0 +1,4 @@ +:hide-toc: + +Documentation +=============================================================================== diff --git a/adi_doctools/theme/cosmic/style/style.scss b/adi_doctools/theme/cosmic/style/style.scss index 9188bf7..adbf046 100644 --- a/adi_doctools/theme/cosmic/style/style.scss +++ b/adi_doctools/theme/cosmic/style/style.scss @@ -98,7 +98,7 @@ body { } } -@media (max-width: $width-narrowest) { +@media (max-width: $width-narrow) { .documentwrapper { margin-top: 3.5rem; }