Documentation as template in monolithic (0.3.8)

Use Documentation repository sphinx files (conf.py, index.rst) as the
template for the monolithic output.
The pseudo_subdomain dic is used to keep track of Documentation's
multiple doctrees; the values at this dict must match the toctree
caption.

Signed-off-by: Jorge Marques <jorge.marques@analog.com>

MANIFEST.in

@@ -1,9 +1,6 @@
 include adi_doctools/miscellaneous/*.js
 include adi_doctools/miscellaneous/*.svg
 include adi_doctools/miscellaneous/*.css
-include adi_doctools/miscellaneous/sphinx-template/conf.py
-include adi_doctools/miscellaneous/sphinx-template/Makefile
-include adi_doctools/miscellaneous/sphinx-template/index.rst
 include adi_doctools/theme/cosmic/static/*.umd.js
 include adi_doctools/theme/cosmic/static/*.js.map
 include adi_doctools/theme/cosmic/static/*.min.css

adi_doctools/__init__.py

@@ -6,12 +6,10 @@
 from .directive import setup as directive_setup
 from .role import setup as role_setup
 
-__version__ = "0.3.7"
+__version__ = "0.3.8"
 
-dft_monolithic = False
 
-
-def get_navigation_tree(context, repo, monolithic, pagename):
+def get_navigation_tree(app, context, pagename):
     # The navigation tree, generated from the sphinx-provided ToC tree.
     if "toctree" in context:
         toctree = context["toctree"]
@@ -24,14 +22,12 @@ def get_navigation_tree(context, repo, monolithic, pagename):
     else:
         toctree_html = ""
 
-    return navigation_tree(toctree_html, context['content_root'], repo,
-                           monolithic, pagename)
+    return navigation_tree(app, toctree_html, context['content_root'],
+                           pagename)
 
 
 def html_page_context(app, pagename, templatename, context, doctree):
-    repository = app.env.config.repository
-    monolithic = app.env.config.monolithic
-    ret = get_navigation_tree(context, repository, monolithic, pagename)
+    ret = get_navigation_tree(app, context, pagename)
     context["sidebar_tree"], context["subdomain_tree"] = ret
 
 
@@ -88,8 +84,6 @@ def setup(app):
     for setup in role_setup:
         setup(app)
 
-    app.add_config_value('monolithic', dft_monolithic, 'env')
-
     app.add_post_transform(wrap_elements)
 
     app.connect("html-page-context", html_page_context)

adi_doctools/cli/aggregate.py

@@ -6,6 +6,11 @@
 remote = "git@github.com:analogdevicesinc/{}.git"
 
 lut = {
+    'documentation': {
+        'doc_folder': 'docs',
+        'name': 'System Level',
+        'branch': 'main'
+    },
     'hdl': {
         'doc_folder': 'docs',
         'extra': (
@@ -20,21 +25,16 @@
         'name': 'no-OS',
         'branch': 'sphinx-mk'
     },
-    'documentation': {
-        'doc_folder': 'docs',
-        'name': 'System Level',
+    'pyadi-iio': {
+        'doc_folder': 'doc',
+        'name': 'pyadi-iio',
         'branch': 'main'
     },
     'doctools': {
         'doc_folder': 'docs',
         'name': 'Doctools',
         'branch': 'main'
     },
-    'pyadi-iio': {
-        'doc_folder': 'doc',
-        'name': 'pyadi-iio',
-        'branch': 'main'
-    },
 }
 
 dry_run = True
@@ -99,9 +99,10 @@ def patch_index(name, docsdir, indexfile):
                 if data[i][0:3] == '   ' and data[i][0:4] != '   :':
                     pos = data[i].find('<')
                     if pos == -1:
-                        data[i] = f"   {name}/{data[i][3:]}"
+                        str_ = f"   {name}/{data[i][3:]}"
                     else:
-                        data[i] = f"{data[i][:pos+1]}{name}/{data[i][pos+1:]}"
+                        str_ = f"   {data[i][:pos+1]}{name}/{data[i][pos+1:]}"
+                    data[i] = str_
 
                 if data[i][0:3] != '   ' and data[i] != '\n':
                     toctree[-1] = [toctree[-1][0], i - 1]
@@ -110,6 +111,10 @@ def patch_index(name, docsdir, indexfile):
                     else:
                         in_toc = False
                     continue
+                elif i == len(data) - 1 and in_toc:
+                    toctree[-1] = [toctree[-1][0], i + 1]
+                    in_toc = False
+                    break
             else:
                 if data[i] == ".. toctree::\n":
                     toctree.append([i, i])
@@ -126,17 +131,19 @@ def patch_index(name, docsdir, indexfile):
 
     with open(indexfile, "r") as f:
         data_ = f.readlines()
-        # Find end of toctree
+        # Find end of last toctree
         if ".. toctree::\n" in data_:
-            i = data_.index(".. toctree::\n")
+            i = len(data_) - 1 - data_[::-1].index(".. toctree::\n")
             for i in range(i + 1, len(data_)):
                 if data_[i][0:3] != '   ' and data_[i] != '\n':
                     break
         else:
             i = len(data_)
 
-        header = data_[:i]
-        body = data_[i:]
+        header = data_[:i+1]
+        if i == len(data_)-1:
+            header.append('\n')
+        body = data_[i+1:]
 
         if len(toctree) > 1:
             click.echo(click.style(f"Repo {name} containes multiple toctrees!",
@@ -231,17 +238,13 @@ def add_pyadi_iio_to_path():
     indexfile = os.path.join(docs_dir, 'index.rst')
     if os.path.isdir(docs_dir):
         pr.run(f"rm -r {docs_dir}")
-    # Copy template
-    src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                              os.pardir))
-    sphinx_template = os.path.join(src_dir, 'miscellaneous/sphinx-template')
-    pr.run(f"cp -r {sphinx_template} {docs_dir}")
+    pr.mkdir(docs_dir)
 
-    mk = []
+    mk = {}
     for r in lut:
         cwd = os.path.join(repo_dir, f"{r}/{lut[r]['doc_folder']}")
-        mk.append(get_sphinx_dirs(cwd))
-        if mk[-1][0]:
+        mk[r] = get_sphinx_dirs(cwd)
+        if mk[r][0]:
             continue
         pr.mkdir(os.path.join(docs_dir, r))
         cp_cmd = f"""\
@@ -255,7 +258,7 @@ def add_pyadi_iio_to_path():
             cp $file {d_}/docs/{r}
         done\
         """
-        cwd = mk[-1][2]
+        cwd = mk[r][2]
         pr.run(cp_cmd, cwd)
 
         # Prefixes references with repo name, expect already external
@@ -283,7 +286,15 @@ def add_pyadi_iio_to_path():
         """.format(r=r, depth=depth)
         pr.run(include_cmd, cwd)
 
-        patch_index(r, docs_dir, indexfile)
+    # Convert documentation into top-level
+    cwd = f"{docs_dir}/documentation"
+    pr.run(f"mv {cwd}/* {docs_dir} ; rmdir {cwd}")
+    pr.run(f"cp -r {mk['documentation'][2]}/conf.py {docs_dir}")
+    pr.run(f"echo monolithic = True >> {docs_dir}/conf.py")
+
+    for r in lut:
+        if r != 'documentation':
+            patch_index(r, docs_dir, indexfile)
 
     # Convert external references into local prefixed
     cwd = docs_dir
@@ -297,7 +308,7 @@ def add_pyadi_iio_to_path():
     """
     pr.run(ref_cmd, cwd)
 
-    pr.run("make html", docs_dir)
+    pr.run("sphinx-build . _build", docs_dir)
 
     cwd = d_
     pr.run("ln -sf docs/_build/html html_mono", cwd)
@@ -377,7 +388,8 @@ def aggregate(directory, symbolic, extra, no_parallel_, dry_run_, open_):
         pr.run(f"rm -r {d__}")
 
     for r in lut:
-        r_ = remote.format(r)
+        r__ = remote if 'remote' not in lut[r] else lut[r]['remote']
+        r_ = r__.format(r)
         cwd = os.path.join(repos_dir, r)
         if not os.path.isdir(cwd):
             git_cmd = ["git", "clone", r_, "--depth=1", "-b",

adi_doctools/theme/__init__.py

@@ -20,6 +20,8 @@
 
 def theme_config_setup(app):
     app.add_config_value('repository', None, 'env')
+    app.add_config_value('pseudo_subdomains', {}, 'env')
+    app.add_config_value('monolithic', False, 'env')
 
 
 setup = [
@@ -30,7 +32,7 @@ def theme_config_setup(app):
 names = ['cosmic']
 
 
-def subdomain_tree(content_root, repo, monolithic, pagename):
+def subdomain_tree(content_root, conf_vars, pagename):
     """
     Create the subdomain tree linking to other repos documentations.
     From the 'repository' config value, a 'current' class is added to
@@ -41,29 +43,36 @@ def subdomain_tree(content_root, repo, monolithic, pagename):
     While something with 0 depth is improper:
     hdl-docs.example.com -> ../no-os -XXX-> hdl-docs.example.com/no-os
     """
+    repo, monolithic, pseudo_subdomains = conf_vars
     root = etree.Element("root")
     home = "index.html"
     if monolithic:
         depth = ''
     else:
         depth = '../'
 
-    for sd in repository:
+    if monolithic:
+        subdomains = {**pseudo_subdomains, **repository}
+        del subdomains['documentation']
+    else:
+        subdomains = repository
+    for sd in subdomains:
+        href = f"{content_root}{depth}{sd}/{home}"
+        attrib = {}
         if sd == repo:
-            link = etree.Element("a", attrib={
-                'href': f"{content_root}{home}",
-                'class': 'current'
-            })
-        else:
-            link = etree.Element("a", attrib={
-                'href': f"{content_root}{depth}{sd}/{home}",
-            })
-        link.text = repository[sd]
+            if not monolithic:
+                href = f"{content_root}{home}"
+            attrib = {'class': 'current'}
+        link = etree.Element("a", attrib={
+            'href': href,
+            **attrib
+        })
+        link.text = subdomains[sd]
         root.append(link)
     return etree.tostring(root, pretty_print=True, encoding='unicode')
 
 
-def navigation_tree(toctree_html, content_root, repo, monolithic, pagename):
+def navigation_tree(app, toctree_html, content_root, pagename):
     """
     Add collapsible sections to the navigation tree.
     Adapted from
@@ -79,19 +88,28 @@ def navigation_tree(toctree_html, content_root, repo, monolithic, pagename):
     parser = etree.HTMLParser()
     root = etree.fromstring(toctree_html, parser)
 
+    conf_vars = (
+        app.env.config.repository.lower(),
+        app.env.config.monolithic,
+        app.env.config.pseudo_subdomains
+    )
+    _, monolithic, _ = conf_vars
+
     lvl = [0]
 
     def get_repo(pagename):
         i = pagename.find('/')
         return pagename[0:i] if i != -1 else ''
 
-    def monolithic_tree(root, repo, monolithic, pagename):
+    def monolithic_tree(root, conf_vars, pagename):
+        repo, _, pseudo_subdomains = conf_vars
         # Keep unchanged for standalone pages (e.g. /index.html, /search.html)
-        if repo not in repository:
+        subdomains = {**pseudo_subdomains, **repository}
+        if repo not in subdomains:
             return
 
         # Pop toctrees that are not from the current repo
-        title = repository[repo]
+        title = subdomains[repo]
         body = root.find('./body')
         txt = ''
         for e in body.getchildren():
@@ -142,15 +160,15 @@ def iterate(elem):
             lvl.pop()
 
     if monolithic:
-        repo = get_repo(pagename)
-        monolithic_tree(root, repo, monolithic, pagename)
+        conf_vars = (get_repo(pagename), *conf_vars[1:])
+        monolithic_tree(root, conf_vars, pagename)
 
     for ul in root.findall('./body/ul'):
         for li in ul.findall('./li[@class]'):
             iterate(li)
 
     _sidebar_tree = etree.tostring(root, pretty_print=True, encoding='unicode')
-    _subdomain_tree = subdomain_tree(content_root, repo, monolithic, pagename)
+    _subdomain_tree = subdomain_tree(content_root, conf_vars, pagename)
     return (_sidebar_tree, _subdomain_tree)
 
 

adi_doctools/theme/cosmic/layout.html

@@ -34,7 +34,7 @@
         <label id="show-sidebar" for="input-showsidebar" title="Show/hide index"></label>
       </div>
       <div id="right">
-        <div id="logo"></div>
+        <a id="logo" href="{{ content_root }}index.html"></a>
         <span>
           <label id="show-onthispage" for="input-showonthispage" title="Show/hide contents"></label>
         </span>