WIP Reference docstrings for markdown documentations

.gitignore

@@ -26,3 +26,8 @@ src/cozy-realtime
 
 # docs/ is generated, no need to have it in the repo
 docs/
+
+*.pyc
+*.egg-info
+.DS_Store
+*.egg

generate_config.py

@@ -3,6 +3,7 @@
 - Generate mkdocs.yml from mkdocs.yml.tpl
 """
 
+import argparse
 import yaml
 import json
 import os
@@ -12,6 +13,8 @@
 from collections import OrderedDict, namedtuple
 import fnmatch
 import sh
+from markdown_sphinxjs import gather_doclets_from_dir
+
 
 def simple_glob(directory, glob_pattern):
     matches = []
@@ -73,6 +76,7 @@ def find_entry(tree, name):
     except IndexError:
         return None
 
+
 def walk_dict(d, path=None):
     if path is None:
         path = []
@@ -114,16 +118,26 @@ def fetch_external_doc(repository, destination):
             sh.git('clone', repository, '--depth', '1', '.')
 
 
+TMP_DIR = '/tmp/cozy_docs'
+def ensure_root_tmp_dir():
+    sh.mkdir('-p', TMP_DIR)
+
+
+def get_doc_tmp_dir(doc_name):
+    return osp.join(TMP_DIR, doc_name)
+
+
 def fetch_all_external_docs_from_file(filename):
+    ensure_root_tmp_dir()
     with open(filename) as f:
         external_docs = [parse_external_doc_line(l) for l in f]
-    for name, repository, doc_directory in external_docs:
-        tmpdir = osp.join('/tmp', name)
-        print('Fetching %s...' % name)
+    for docname, repository, doc_directory in external_docs:
+        print('Fetching %s...' % docname)
+        tmpdir = get_doc_tmp_dir(docname)
         fetch_external_doc(repository, tmpdir)
-        src_dir = osp.join('src', name)
+        src_dir = osp.join('src', docname)
         sh.rm('-f', src_dir)
-        print('Linking %s...' % name)
+        print('Linking %s...' % docname)
         sh.ln('-s', osp.join(tmpdir, doc_directory), src_dir)
 
 
@@ -184,12 +198,24 @@ def replace_toc_placeholders(nav, named_tocs):
             cur[single_key] = flatten_entry_if_single(toc)
 
 
-def main(argv):
+def ensure_jsdoc_cache_exists(src_dir, force=False):
+    cache_file = '/tmp/jsdoc_data_cache.json'
+    gather_doclets_from_dir(src_dir, jsdoc_cache=cache_file, force=force)
+
+
+def main():
     OUTSIDE_DOCS = 'OUTSIDE_DOCS'
 
-    if '--fetch' in argv:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--fetch', action='store_true', help='Fetch all external repositories')
+    parser.add_argument('--force-jsdoc', action='store_true', help='Force re-analysis of jsdocs')
+    args = parser.parse_args()
+
+    if args.fetch:
         fetch_all_external_docs_from_file(OUTSIDE_DOCS)
 
+    ensure_jsdoc_cache_exists(TMP_DIR, force=args.force_jsdoc)
+
     with open('./mkdocs.yml.tpl') as f:
         data = ordered_load(f, yaml.SafeLoader)
 
@@ -219,4 +245,4 @@ def main(argv):
         warning = '#\n# THIS FILE IS AUTOMATICALLY GENERATED, PLEASE EDIT `mkdocs.yml.tpl` AND LAUNCH `python generate_config.py`\n#\n'
         f.write(warning + content)
 
-main(sys.argv)
+main()

markdown_sphinxjs/README.md

@@ -0,0 +1,8 @@
+Markdown SphinxJS
+=================
+
+Auto import JSDoc in your Markdown documentation.
+
+This is based on the wonderful [SphinxJS] that does the same for the Sphinx documentation generator.
+
+[SphinxJS]: https://github.com/mozilla/sphinx-js/

markdown_sphinxjs/markdown_sphinxjs/__init__.py

@@ -0,0 +1,185 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+import re
+import os
+import json
+import tempfile
+
+from markdown.extensions import Extension
+from markdown.blockprocessors import BlockProcessor
+from markdown.util import etree
+
+from sphinx_js import gather_doclets
+
+
+class Config:
+    def __init__(self, **attrs):
+        for key, val in attrs.items():
+            setattr(self, key, val)
+
+
+class App:
+    """Make alike for Sphinx app for SphinxJS to work"""
+
+    def __init__(self, config):
+        self.config = Config(**config)
+        self._sphinxjs_doclets_by_class = {}
+        self.confdir = "/tmp"
+
+
+DEFAULT_JSDOC_CONFIG = {
+  "opts": {
+    "recurse": True
+  },
+  "source": {
+    "includePattern": ".+\\.js(doc)?x?$",
+    "excludePattern": "((^|\\/|\\\\)_)|(min)|(dist)",
+    "exclude": [
+      "node_modules",
+      "plugins"
+    ]
+  }
+}
+
+
+def gather_doclets_from_dir(src_dir, jsdoc_cache=None, force=False):
+    if force and os.path.isfile(jsdoc_cache):
+        os.unlink(jsdoc_cache)
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as configfile:
+        configfile.write(json.dumps(DEFAULT_JSDOC_CONFIG, indent=2))
+        configfile.seek(0)
+        app = App(
+            {
+                "js_source_path": src_dir,
+                "js_language": "javascript",
+                "root_for_relative_js_paths": src_dir,
+                "jsdoc_config_path": configfile.name,
+                "jsdoc_cache": jsdoc_cache,
+                "sphinx_js_lax": True
+            }
+        )
+        gather_doclets(app)
+    return {
+        "by_class": app._sphinxjs_doclets_by_class,
+        "by_path": app._sphinxjs_doclets_by_path,
+    }
+
+
+def make_definition_node(ancestor, definition, path):
+    div = etree.SubElement(ancestor, "div")
+    div.attrib["class"] = "markdown-sphinxjs-description"
+
+    name = etree.SubElement(div, "h4")
+    name.text = "%s.%s(%s) => %s" % (
+        definition["memberof"],
+        definition["name"],
+        ", ".join(definition["meta"]["code"]["paramnames"]),
+        definition["returns"][0]["type"]["names"][0]
+    )
+    p = etree.SubElement(div, "p")
+    p.text = definition["description"]
+    param_table = etree.SubElement(div, "table")
+    param_head = etree.SubElement(param_table, "thead")
+    head_row = etree.SubElement(param_table, "tr")
+    name = etree.SubElement(head_row, "th")
+    name.text = 'Parameter'
+    type = etree.SubElement(head_row, "th")
+    type.text = 'Type'
+    desc = etree.SubElement(head_row, "th")
+    desc.text = 'Description'
+
+    # data = etree.SubElement(div, "pre")
+    # data.text = json.dumps(definition, indent=2)
+
+    params = etree.SubElement(param_table, "tbody")
+    for param in definition["params"]:
+        row = etree.SubElement(params, "tr")
+        name = etree.SubElement(row, "td")
+        name.text = param["name"]
+        type = etree.SubElement(row, "td")
+        type.text = ", ".join(param["type"]["names"])
+        desc = etree.SubElement(row, "td")
+        desc.text = param["description"]
+
+    for example in definition["examples"]:
+        example_node = etree.SubElement(div, "pre")
+        example_node.text = """%s""" % example
+
+    return div
+
+
+class MarkdownJSExtension(Extension):
+    doclets = None
+    def __init__(self, directory, jsdoc_cache, **kwargs):
+        super(MarkdownJSExtension, self).__init__(**kwargs)
+        self.config = {"directory": directory}
+        self.index = {}
+        if not MarkdownJSExtension.doclets:
+            # Markdown extensions are instantiated for each file processed but doclets
+            # gathering is costly and we do not want to do it every time, this is why
+            # we store the generated doclets in the class. 
+            # This is a hacky way to have doclets computation be done only once.
+            MarkdownJSExtension.doclets = gather_doclets_from_dir(directory, jsdoc_cache)
+        self.doclets = MarkdownJSExtension.doclets
+
+    def extendMarkdown(self, md, **kwargs):
+        md.registerExtension(self)
+
+        md.parser.blockprocessors.register(
+            MarkdownJSProcessor(md.parser, self.doclets), "markdown-sphinxjs", 105
+        )
+
+
+class MarkdownJSProcessor(BlockProcessor):
+    """
+    Understands blocks beginining by --->
+    The arrow must be followed by an identifier for a function.
+
+    Finds the function referenced by identifier and outputs a div with the description
+    and parameters of the function.
+
+    Mostly copied from admonition block processor"""
+
+    RE = re.compile(r'^---> ?([\w\-/#]+(?: +[\w\-#]+)*)(?:\n|$)')
+    RE_SPACES = re.compile("  +")
+
+    def __init__(self, parser, doclets):
+        super(MarkdownJSProcessor, self).__init__(parser)
+        self.doclets = doclets
+
+    def test(self, parent, block):
+        sibling = self.lastChild(parent)
+        return self.RE.search(block)
+
+    def build(self, ancestor, match):
+        path_tokens = match.group(1).split(' ')
+        definition, path = self.doclets["by_path"].get_with_path(path_tokens)
+        return make_definition_node(ancestor, definition, path)
+
+    def run(self, parent, blocks):
+        sibling = self.lastChild(parent)
+        block = blocks.pop(0)
+        m = self.RE.search(block)
+
+        if m:
+            block = block[m.end() :]  # removes the first line
+
+        block, theRest = self.detab(block)
+
+        if m:
+            div = self.build(parent, m)
+        else:
+            div = sibling
+
+        self.parser.parseChunk(div, block)
+
+        if theRest:
+            # This block contained unindented line(s) after the first indented
+            # line. Insert these lines as the first block of the master blocks
+            # list for future processing.
+            blocks.insert(0, theRest)
+
+
+def makeExtension(**kwargs):  # pragma: no cover
+    return MarkdownJSExtension(**kwargs)

markdown_sphinxjs/requirements.txt

@@ -0,0 +1 @@
+markdown==3.1.1

markdown_sphinxjs/setup.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Note: To use the 'upload' functionality of this file, you must:
+#   $ pip install twine
+
+import io
+import os
+import sys
+from shutil import rmtree
+
+from setuptools import find_packages, setup, Command
+
+# Package meta-data.
+NAME = 'markdownsphinxjs'
+DESCRIPTION = 'SphinxJS for markdown'
+URL = 'https://github.com/cozy/markdown_sphinxjs'
+EMAIL = 'contact@cozycloud.cc'
+AUTHOR = 'ptbrowne'
+REQUIRES_PYTHON = '>=3.4.3'
+VERSION = '1.0.0'
+
+# What packages are required for this module to be executed?
+REQUIRED = ['markdown', 'sphinx_js']
+
+# The rest you shouldn't have to touch too much :)
+# ------------------------------------------------
+# Except, perhaps the License and Trove Classifiers!
+# If you do change the License, remember to change the Trove Classifier for that!
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+# Import the README and use it as the long-description.
+# Note: this will only work if 'README.md' is present in your MANIFEST.in file!
+with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = '\n' + f.read()
+
+# Load the package's __version__.py module as a dictionary.
+about = {}
+if not VERSION:
+    with open(os.path.join(here, NAME, '__version__.py')) as f:
+        exec(f.read(), about)
+else:
+    about['__version__'] = VERSION
+
+
+class UploadCommand(Command):
+    """Support setup.py upload."""
+
+    description = 'Build and publish the package.'
+    user_options = []
+
+    @staticmethod
+    def status(s):
+        """Prints things in bold."""
+        print('\033[1m{0}\033[0m'.format(s))
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        try:
+            self.status('Removing previous builds…')
+            rmtree(os.path.join(here, 'dist'))
+        except OSError:
+            pass
+
+        self.status('Building Source and Wheel (universal) distribution…')
+        os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable))
+
+        self.status('Uploading the package to PyPi via Twine…')
+        os.system('twine upload dist/*')
+
+        self.status('Pushing git tags…')
+        os.system('git tag v{0}'.format(about['__version__']))
+        os.system('git push --tags')
+
+        sys.exit()
+
+print(find_packages(exclude=('tests',)))
+# Where the magic happens:
+setup(
+    name=NAME,
+    version=about['__version__'],
+    description=DESCRIPTION,
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    author=AUTHOR,
+    author_email=EMAIL,
+    python_requires=REQUIRES_PYTHON,
+    url=URL,
+    packages=find_packages(exclude=('tests',)),
+    # If your package is a single module, use this instead of 'packages':
+    #py_modules=['markdownsphinxjs'],
+    #entry_points={'console_scripts': ['mint=mint.cli:main']},
+    install_requires=REQUIRED,
+    include_package_data=True,
+    license='MIT',
+    classifiers=[
+        # Trove classifiers
+        # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
+    ],
+    # $ setup.py publish support.
+    cmdclass={'upload': UploadCommand},
+)

markdown_sphinxjs/test.py

@@ -0,0 +1,20 @@
+import markdown
+from jsdoc_reference import JSDocReferenceExtension
+
+ext = JSDocReferenceExtension(directory='/Users/cozy/code/cozy/konnector-libs/packages/cozy-konnector-libs/src')
+m = markdown.Markdown(extensions=[ext])
+
+html = m.convert(
+    """
+## Hello
+
+How are you ?
+
+--->findDuplicates
+
+
+!!!!note
+    salut
+"""
+)
+print(html)

mkdocs.yml.tpl

@@ -78,6 +78,9 @@ markdown_extensions:
 - smarty
 - toc:
     permalink: true
+- markdown_sphinxjs:
+    directory: /tmp/cozy_docs
+    jsdoc_cache: /tmp/jsdoc_data_cache.json
 extra:
   search:
     tokenizer: "[^a-z\u0430-\u044F\u04510-9\\-\\.]"

requirements.txt

@@ -29,3 +29,4 @@ Unidecode==1.0.22
 urllib3==1.23
 mkdocs==1.0.4
 mkdocs-material==4.4.0
+git+git://github.com/ptbrowne/sphinx-js@master#egg=sphinxjs

src/css/extra.css

@@ -39,4 +39,12 @@
 /* stack pattern */
 .home-action a > * + * {
   margin-bottom: 1rem
-} 
+}
+
+/* From markdown-sphinxjs */
+.markdown-sphinxjs-description {
+  border-top-left-radius: 2px;
+  border-bottom-left-radius: 2px;
+  border-left: 0.25rem #ccc solid;
+  padding-left: 1rem;
+}

src/tutorials/konnector/getting-started.md

@@ -7,6 +7,8 @@
 
 The easiest way to create a new connector is to use [cozy-konnector-template](https://github.com/konnectors/cozy-konnector-template).
 
+---> BaseKonnector# waitForTwoFaCode
+
 First of all, [download](https://github.com/konnectors/cozy-konnector-template/archive/master.zip) or clone the repository:
 
 ```sh