docs(Reference): Use dgeni to generate docs from code

Use dgeni to generate json files from code. Scrape these json files for ingestion into the
neo-one-website.

re neo-one-suite#702

packages/neo-one-website/package.json

@@ -26,6 +26,8 @@
     "@types/styled-components": "^4.1.0",
     "@types/webpack": "^4.4.18",
     "clipboard": "^2.0.3",
+    "dgeni": "^0.4.10",
+    "dgeni-packages": "^0.26.12",
     "fs-extra": "^7.0.1",
     "gray-matter": "^4.0.1",
     "html-to-react": "^1.3.3",

packages/neo-one-website/src/utils/dgeniSetup.ts

@@ -0,0 +1,98 @@
+// tslint:disable only-arrow-functions no-object-mutation no-submodule-imports no-any no-invalid-template-strings
+import { Package } from 'dgeni';
+// @ts-ignore
+import nunjucks from 'dgeni-packages/nunjucks';
+import typescript from 'dgeni-packages/typescript';
+import * as path from 'path';
+import { textProcessor } from './processors';
+
+const docFiles: ReadonlyArray<string> = [
+  'neo-one-client-core/src/types.ts',
+  'neo-one-client-core/src/Client.ts',
+  'neo-one-client-core/src/DeveloperClient.ts',
+  'neo-one-client-core/src/Hash256.ts',
+  'neo-one-client-core/src/user/LocalUserAccountProvider.ts',
+  'neo-one-client-core/src/user/LocalKeyStore.ts',
+  'neo-one-client-core/src/user/LocalMemoryStore.ts',
+  'neo-one-client-core/src/user/LocalStringStore.ts',
+  'neo-one-client-core/src/provider/JSONRPCProvider.ts',
+  'neo-one-client-core/src/provider/NEOONEDataProvider.ts',
+  'neo-one-client-core/src/provider/NEOONEOneDataProvider.ts',
+  'neo-one-client-core/src/provider/NEOONEProvider.ts',
+  'neo-one-client-common/src/types.ts',
+  'neo-one-client-common/src/helpers.ts',
+  'neo-one-smart-contract/src/index.d.ts',
+];
+
+// Configuration of the typescript processor
+(typescript as any)
+  // Configure additional jsdoc style tags to be recognized by the processor
+  .config(function(parseTagsProcessor: any) {
+    const tagDefs = parseTagsProcessor.tagDefinitions;
+    // Register '@example' tags with the processor. Multiple instances allowed.
+    tagDefs.push({ name: 'example', multi: true });
+    // Register '@param' tags with the processor.  Multiple instances allowed.
+    tagDefs.push({ name: 'param', multi: true });
+    // Register '@internal' tags with the processor.
+    tagDefs.push({ name: 'internal' });
+  })
+  // Configure output paths for additional doc types
+  .config(function(computeIdsProcessor: any, computePathsProcessor: any) {
+    // Configure ID for "getters". Must be manually configured to avoid potential conflict with a property.
+    computeIdsProcessor.idTemplates.push({
+      docTypes: ['get-accessor-info'],
+      // Template for constructing "getter" id. If we had same-named properties, we would need to use a different ID.
+      idTemplate: '${containerDoc.id}.${name}',
+      getAliases(doc: any) {
+        return doc.containerDoc.aliases.map((alias: string) => `${alias}.${doc.name}`);
+      },
+    });
+
+    // Configure output path for "getters". Must be manually configured to avoid potential conflict.
+    computePathsProcessor.pathTemplates.push({
+      docTypes: ['get-accessor-info'],
+      // Template for constructing "getter" path. If we had same-named properties, we would need to use a different path.
+      pathTemplate: '${containerDoc.path}#${name}',
+      getOutputPath() {
+        // These docs are not written to their own file, instead they are part of their class doc
+      },
+    });
+
+    // Configure output path for overloaded functions. Each overload needs a unique output file to avoid conflict.
+    computePathsProcessor.pathTemplates.push({
+      docTypes: ['function-overload'],
+      outputPathTemplate: 'partials/modules/${path}/index.html',
+      // Template to generate unique path for each overload of a function.
+      pathTemplate: '${moduleDoc.path}/${name}${parameterDocs.length}',
+    });
+  });
+
+// Create a dgeni Package which defines how the code and comments are parsed into docs.
+export const dgeniSetup = new Package('neo-one-docs', [
+  // The typescript processor from dgeni-packages is used to parse typescript files.
+  typescript,
+  // The nunjucks processor from dgeni-packages is used to output docs to a template.
+  nunjucks,
+])
+  // Register our custom processor for our specific use case.
+  .processor(textProcessor)
+  // Configure subprocessors from typescript and nunjucks
+  .config(function(readFilesProcessor: any, readTypeScriptModules: any, templateFinder: any, writeFilesProcessor: any) {
+    // Register basePath used for resolving paths to typescript files to parse.
+    readTypeScriptModules.basePath = path.resolve(__dirname, '../../..');
+    // Register basePath used for resolving paths to javascript files to parse. Unused in our case, but needs to exist.
+    readFilesProcessor.basePath = '';
+
+    // Register typescript files to parse. Paths relative to basePath.
+    readTypeScriptModules.sourceFiles = docFiles;
+    // Register javascript files to parse. None needed.
+    readFilesProcessor.sourceFiles = [];
+
+    // Directory where output template can be found.
+    templateFinder.templateFolders.unshift(path.resolve(__dirname, 'templates'));
+    // File name of the output template in the registered directory.
+    templateFinder.templatePatterns.unshift('template.txt');
+
+    // Directory to output generated files.
+    writeFilesProcessor.outputFolder = path.resolve(__dirname, '../../../neo-one-website/src/utils/build');
+  });

packages/neo-one-website/src/utils/getReferences.ts

@@ -0,0 +1,176 @@
+import { Dgeni } from 'dgeni';
+import * as fs from 'fs-extra';
+import _ from 'lodash';
+import * as path from 'path';
+import { dgeniSetup } from './dgeniSetup';
+import { ModuleLinks, ModuleLinksPaths, tokenizeDocText } from './tokenizeDocText';
+
+interface Parameter {
+  readonly name: string;
+  readonly type: ReadonlyArray<string>;
+  readonly description: ReadonlyArray<string>;
+}
+
+interface Property extends Parameter {}
+
+interface MethodBase {
+  readonly name: string;
+  readonly text: ReadonlyArray<string>;
+  readonly parameters: ReadonlyArray<Parameter>;
+}
+
+interface Method extends MethodBase {
+  readonly description: ReadonlyArray<string>;
+  readonly returns?: ReadonlyArray<string>;
+  readonly internal?: boolean;
+}
+
+const BASE_PATH = path.resolve(__dirname, 'build', 'partials', 'modules');
+
+const dgenerate = async () => {
+  const dgeni = new Dgeni([dgeniSetup]);
+
+  await fs.emptyDir(path.resolve(__dirname, 'build'));
+  await dgeni.generate();
+};
+
+const mapModule = (moduleName: string) => {
+  switch (moduleName) {
+    case 'neo-one-client-common':
+      return '@neo-one/client';
+    case 'neo-one-client-core':
+      return '@neo-one/client';
+    case 'neo-one-smart-contract':
+      return '@neo-one/smart-contract';
+    default:
+      return '';
+  }
+};
+
+const getLinksFromModule = async (currentPath: string, dirName: string): Promise<ModuleLinksPaths> => {
+  const modulePath = path.resolve(currentPath, dirName);
+  const dirsFiles = await fs.readdir(modulePath);
+  const dirs = dirsFiles.filter((name) => name !== 'index.json');
+
+  if (_.isEmpty(dirs)) {
+    return {
+      links: [dirName],
+      paths: [path.resolve(modulePath, 'index.json')],
+    };
+  }
+
+  const links = await Promise.all(dirs.map(async (dir) => getLinksFromModule(path.resolve(currentPath, dirName), dir)));
+
+  return {
+    links: _.flatten(links.map((link) => link.links)),
+    paths: _.flatten(links.map((link) => link.paths)),
+  };
+};
+
+const getLinks = async (): Promise<{ readonly [moduleName: string]: ModuleLinksPaths }> => {
+  const moduleNames = await fs.readdir(BASE_PATH);
+
+  return moduleNames.reduce(async (acc, moduleName) => {
+    const moduleLinks = await getLinksFromModule(BASE_PATH, moduleName);
+    const accRes = await acc;
+
+    if (Object.keys(accRes).includes(mapModule(moduleName))) {
+      const accModuleLinks = Object.entries(accRes).find(([name]) => name === mapModule(moduleName));
+
+      return {
+        ...accRes,
+        [mapModule(moduleName)]: {
+          links:
+            accModuleLinks === undefined
+              ? moduleLinks.links
+              : (accModuleLinks[1] as ModuleLinksPaths).links.concat(moduleLinks.links),
+          paths:
+            accModuleLinks === undefined
+              ? moduleLinks.paths
+              : (accModuleLinks[1] as ModuleLinksPaths).paths.concat(moduleLinks.paths),
+        },
+      };
+    }
+
+    return {
+      ...accRes,
+      [mapModule(moduleName)]: moduleLinks,
+    };
+  }, Promise.resolve({}));
+};
+
+const extractParametersProperties = (values: ReadonlyArray<Parameter | Property>, links: ModuleLinks) =>
+  values
+    .filter((param) => !_.isEmpty(param))
+    .map((param) => ({
+      name: param.name,
+      type: tokenizeDocText(param.type, links),
+      description: tokenizeDocText(param.description, links),
+    }));
+
+const extractConstructor = (constructorDoc: MethodBase, links: ModuleLinks) => ({
+  name: constructorDoc.name,
+  text: tokenizeDocText(constructorDoc.text, links),
+  parameters: extractParametersProperties(constructorDoc.parameters, links),
+});
+
+const extractMethods = (methods: ReadonlyArray<Method>, links: ModuleLinks) =>
+  methods
+    .filter((method) => !_.isEmpty(method))
+    .filter((method) => !method.internal)
+    .map((method) => ({
+      name: method.name,
+      text: tokenizeDocText(method.text, links),
+      parameters: extractParametersProperties(method.parameters, links),
+      description: tokenizeDocText(method.description, links),
+      returns: method.returns === undefined ? undefined : tokenizeDocText(method.returns, links),
+    }));
+
+const getReference = async (refPath: string, links: ModuleLinks) => {
+  const contents = await fs.readJSON(refPath);
+
+  return {
+    name: contents.name,
+    docType: contents.docType,
+    text: tokenizeDocText(contents.text, links),
+    description: tokenizeDocText(contents.description, links),
+    parameters: contents.parameters === undefined ? undefined : extractParametersProperties(contents.parameters, links),
+    constructorDoc:
+      contents.constructorDoc === undefined ? undefined : extractConstructor(contents.constructorDoc, links),
+    properties: contents.properties === undefined ? undefined : extractParametersProperties(contents.properties, links),
+    methods: contents.methods === undefined ? undefined : extractMethods(contents.methods, links),
+    returns: contents.returns === undefined ? undefined : tokenizeDocText(contents.returns, links),
+    examples:
+      contents.examples === undefined
+        ? undefined
+        : contents.examples.map((example: ReadonlyArray<string>) => tokenizeDocText(example, links)),
+  };
+};
+
+const getSidebar = (links: ModuleLinks) => ({
+  title: 'Packages',
+  subsections: Object.keys(links).map((subsection) => ({
+    slug: subsection,
+    title: subsection,
+  })),
+});
+
+export const getReferences = async () => {
+  await dgenerate();
+  const links = await getLinks();
+
+  return Object.entries(links).map(async ([moduleName, moduleLinksPaths]) => {
+    const refItems = await Promise.all(moduleLinksPaths.paths.map(async (refPath) => getReference(refPath, links)));
+
+    return {
+      title: moduleName,
+      type: 'All',
+      content: {
+        type: 'referenceItems',
+        value: refItems,
+      },
+      current: '@neo-one/client',
+      sidebar: getSidebar(links),
+    };
+  });
+};

packages/neo-one-website/src/utils/processors/index.ts

@@ -0,0 +1 @@
+export * from './textProcessor';

packages/neo-one-website/src/utils/processors/textProcessor.ts

@@ -0,0 +1,48 @@
+// tslint:disable only-arrow-functions no-object-mutation no-any
+
+const extractTextWithoutComments = (declaration: ts.Declaration) =>
+  declaration
+    .getText(declaration.getSourceFile())
+    .split('\n')
+    .filter((line: string) => {
+      const trimmedLine = line.trim();
+
+      return trimmedLine !== '/**' && trimmedLine.charAt(0) !== '*' && trimmedLine.charAt(0) !== '/';
+    })
+    .join('\n');
+
+const extractBasicText = (text: string) => text.substr(0, text.indexOf(' {'));
+
+const separateLines = (text: string) => text.split('\n').filter((line: string) => line !== '');
+
+export function textProcessor() {
+  return {
+    $runAfter: ['paths-computed'],
+    $runBefore: ['rendering-docs'],
+    $process(docs: ReadonlyArray<any>) {
+      docs.forEach((doc: any) => {
+        const text = extractTextWithoutComments(doc.declaration);
+
+        doc.text =
+          doc.docType === 'class' || doc.parameterDocs !== undefined
+            ? extractBasicText(text).split('\n')
+            : text.split('\n');
+
+        doc.description = separateLines(doc.description);
+        if (doc.returns) {
+          doc.returns.description = separateLines(doc.returns.description);
+        }
+        if (doc.type) {
+          doc.type = separateLines(doc.type);
+        }
+        if (doc.example) {
+          doc.example = doc.example.map(separateLines);
+        }
+
+        if (doc.outputPath) {
+          doc.outputPath = doc.outputPath.replace('.html', '.json');
+        }
+      });
+    },
+  };
+}

packages/neo-one-website/src/utils/templates/template.txt

@@ -0,0 +1,56 @@
+{
+  "description": [{% for line in doc.description %} "{{line}}", {% endfor %} ""], {% if doc.constructorDoc %}
+  "constructorDoc": {
+    "name": "{{doc.constructorDoc.name}}",
+    "text": [{% for line in doc.constructorDoc.text %} "{{line}}", {% endfor %} ""],
+    "parameters": [{% for param in doc.constructorDoc.parameterDocs %}
+          {
+            "name": "{{ param.name }}",
+            "type": [{% for line in param.type %} "{{line}}", {% endfor %} ""],
+            "description": [{% for line in param.content %} "{{line}}", {% endfor %} ""]
+          }, {% endfor %}
+          {}
+       ]
+  }, {% endif %}
+  "text": [{% for line in doc.text %} "{{line}}", {% endfor %} ""], {% if doc.parameterDocs %}
+  "parameters": [ {% for param in doc.parameterDocs %}
+    {
+      "name": "{{ param.name }}",
+      "type": [{% for line in param.type %} "{{line}}", {% endfor %} ""],
+      "description": [{% for line in param.description %} "{{line}}", {% endfor %} ""]
+    }, {% endfor %}
+    {}
+  ], {% endif %} {% if doc.docType in ['interface', 'class', 'enum'] %}
+  "properties": [{% for member in doc.members %} {% if member.parameterDocs === undefined %}
+    {
+      "name": "{{member.name}}",
+      "type": [{% for line in member.type %} "{{line}}", {% endfor %} ""],
+      "description": [{% for line in member.description %} "{{line}}", {% endfor %} ""]
+    }, {% endif %} {% endfor %}
+    {}
+  ], {% endif %} {% if doc.docType in ['interface', 'class'] %}
+  "methods": [{% for member in doc.members %} {% if member.parameterDocs %}
+     {
+       "text": [{% for line in member.text %} "{{line}}", {% endfor %} ""],
+       "description": [{% for line in member.description %} "{{line}}", {% endfor %} ""], {% if member.returns %}
+       "returns": [{% for line in member.returns.description %} "{{line}}", {% endfor %} ""], {% endif %}
+       "parameters": [{% for param in member.parameterDocs %}
+          {
+            "name": "{{ param.name }}",
+            "type": [{% for line in param.type %} "{{line}}", {% endfor %} ""],
+            "description": [{% for line in param.content %} "{{line}}", {% endfor %} ""]
+          }, {% endfor %}
+          {}
+       ], {% if member.internal !== undefined %}
+       "internal": true, {% endif %}
+       "name": "{{member.name}}",
+       "type": [{% for line in member.type %} "{{line}}", {% endfor %} ""]
+      },{% endif %} {% endfor %}
+      {}
+    ],
+  {% endif %} {% if doc.returns %}
+  "returns": [{% for line in doc.returns.description %} "{{line}}", {% endfor %} ""], {% endif %} {% if doc.example %}
+  "examples": [{% for example in doc.example %} [{% for line in example %} "{{line}}", {% endfor %} ""], {% endfor %} []], {% endif %}
+  "name": "{{ doc.name }}",
+  "docType": "{{ doc.docType }}"
+}