Skip to content

Commit

Permalink
tools: produce JSON documentation using unified/remark/rehype
Browse files Browse the repository at this point in the history 
PR-URL: #21697
Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>
Reviewed-By: Trivikram Kamat <trivikr.dev@gmail.com>
  • Loading branch information
@rubys @targos
rubys authored and targos committed Aug 7, 2018
1 parent 40af976 commit c85d00b
Show file tree
Hide file tree
Showing 102 changed files with 494 additions and 23,127 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ deps/npm/node_modules/.bin/
/*.pkg
/SHASUMS*.txt*

# api docs artifacts
tools/doc/node_modules

# test artifacts
tools/remark-cli/node_modules
tools/remark-preset-lint-node/node_modules
Expand Down
16 changes: 8 additions & 8 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ ifeq ($(OSTYPE),aix)
DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp
endif

test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS)
test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
$(RM) -r test/addons/??_*/
[ -x $(NODE) ] && $(NODE) $< || node $<
touch $@
Expand Down Expand Up @@ -633,15 +633,15 @@ available-node = \
run-npm-install = $(PWD)/$(NPM) install --production --no-package-lock
run-npm-ci = $(PWD)/$(NPM) ci

gen-json = tools/doc/generate.js --format=json $< > $@
gen-html = tools/doc/generate.js --node-version=$(FULLVERSION) --format=html \
--analytics=$(DOCS_ANALYTICS) $< > $@
tools/doc/node_modules/js-yaml/package.json:
cd tools/doc && $(call available-node,$(run-npm-install))

out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
$(call available-node, $(gen-json))
gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \
--analytics=$(DOCS_ANALYTICS) $< --output-directory=out/doc/api

out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
$(call available-node, $(gen-html))
out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \
tools/doc/html.js tools/doc/json.js
$(call available-node, $(gen-api))

out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
$(call available-node, tools/doc/allhtml.js)
Expand Down
4 changes: 2 additions & 2 deletions doc/api/https.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ changes:
pr-url: https://github.com/nodejs/node/pull/10638
description: The `options` parameter can be a WHATWG `URL` object.
-->
- `url` {string | URL}
* `url` {string | URL}
* `options` {Object | string | URL} Accepts the same `options` as
[`https.request()`][], with the `method` always set to `GET`.
* `callback` {Function}
Expand Down Expand Up @@ -174,7 +174,7 @@ changes:
pr-url: https://github.com/nodejs/node/pull/10638
description: The `options` parameter can be a WHATWG `URL` object.
-->
- `url` {string | URL}
* `url` {string | URL}
* `options` {Object | string | URL} Accepts all `options` from
[`http.request()`][], with some differences in default values:
- `protocol` **Default:** `'https:'`
Expand Down
30 changes: 28 additions & 2 deletions test/doctool/test-doctool-html.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,34 @@ try {
const assert = require('assert');
const { readFile } = require('fs');
const fixtures = require('../common/fixtures');
const toHTML = require('../../tools/doc/html.js');
const html = require('../../tools/doc/html.js');
const path = require('path');

module.paths.unshift(
path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
const unified = require('unified');
const markdown = require('remark-parse');
const remark2rehype = require('remark-rehype');
const raw = require('rehype-raw');
const htmlStringify = require('rehype-stringify');

function toHTML({ input, filename, nodeVersion, analytics }, cb) {
const content = unified()
.use(markdown)
.use(html.firstHeader)
.use(html.preprocessText)
.use(html.preprocessElements, { filename })
.use(html.buildToc, { filename })
.use(remark2rehype, { allowDangerousHTML: true })
.use(raw)
.use(htmlStringify)
.processSync(input);

html.toHTML(
{ input, content, filename, nodeVersion, analytics },
cb
);
}

// Test data is a list of objects with two properties.
// The file property is the file path.
Expand Down Expand Up @@ -80,7 +107,6 @@ testData.forEach(({ file, html, analyticsId }) => {

readFile(file, 'utf8', common.mustCall((err, input) => {
assert.ifError(err);

toHTML(
{
input: input,
Expand Down
44 changes: 30 additions & 14 deletions test/doctool/test-doctool-json.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,27 @@ try {

const assert = require('assert');
const fs = require('fs');
const path = require('path');
const fixtures = require('../common/fixtures');
const json = require('../../tools/doc/json.js');

module.paths.unshift(
path.join(__dirname, '..', '..', 'tools', 'doc', 'node_modules'));
const unified = require('unified');
const markdown = require('remark-parse');

function toJSON(input, filename, cb) {
function nullCompiler() {
this.Compiler = (tree) => tree;
}

unified()
.use(markdown)
.use(json.jsonAPI, { filename })
.use(nullCompiler)
.process(input, cb);
}

// Outputs valid json with the expected fields when given simple markdown
// Test data is a list of objects with two properties.
// The file property is the file path.
Expand All @@ -21,15 +39,16 @@ const testData = [
{
file: fixtures.path('sample_document.md'),
json: {
type: 'module',
source: 'foo',
modules: [{
textRaw: 'Sample Markdown',
name: 'sample_markdown',
modules: [{
textRaw: 'Seussian Rhymes',
name: 'seussian_rhymes',
desc: '<ol>\n<li>fish</li>\n<li><p>fish</p>\n</li>\n<li>' +
'<p>Red fish</p>\n</li>\n<li>Blue fish</li>\n</ol>\n',
desc: '<ol>\n<li>fish</li>\n<li>fish</li>\n</ol>\n' +
'<ul>\n<li>Red fish</li>\n<li>Blue fish</li>\n</ul>',
type: 'module',
displayName: 'Seussian Rhymes'
}],
Expand All @@ -41,6 +60,7 @@ const testData = [
{
file: fixtures.path('order_of_end_tags_5873.md'),
json: {
type: 'module',
source: 'foo',
modules: [{
textRaw: 'Title',
Expand All @@ -55,15 +75,10 @@ const testData = [
signatures: [
{
params: [{
textRaw: '`array` {Array} ',
textRaw: '`array` {Array}',
name: 'array',
type: 'Array'
}]
},
{
params: [{
name: 'array'
}]
}
]
}],
Expand All @@ -78,6 +93,7 @@ const testData = [
{
file: fixtures.path('doc_with_yaml.md'),
json: {
type: 'module',
source: 'foo',
modules: [
{
Expand All @@ -92,7 +108,7 @@ const testData = [
changes: []
},
desc: '<p>Describe <code>Foobar</code> in more detail ' +
'here.</p>\n',
'here.</p>',
type: 'module',
displayName: 'Foobar'
},
Expand All @@ -110,7 +126,7 @@ const testData = [
]
},
desc: '<p>Describe <code>Foobar II</code> in more detail ' +
'here. fg(1)</p>\n',
'here. fg(1)</p>',
type: 'module',
displayName: 'Foobar II'
},
Expand All @@ -123,15 +139,15 @@ const testData = [
changes: []
},
desc: '<p>Describe <code>Deprecated thingy</code> in more ' +
'detail here. fg(1p)</p>\n',
'detail here. fg(1p)</p>',
type: 'module',
displayName: 'Deprecated thingy'
},
{
textRaw: 'Something',
name: 'something',
desc: '<!-- This is not a metadata comment -->\n<p>' +
'Describe <code>Something</code> in more detail here.</p>\n',
'Describe <code>Something</code> in more detail here.</p>',
type: 'module',
displayName: 'Something'
}
Expand All @@ -147,9 +163,9 @@ const testData = [
testData.forEach((item) => {
fs.readFile(item.file, 'utf8', common.mustCall((err, input) => {
assert.ifError(err);
json(input, 'foo', common.mustCall((err, output) => {
toJSON(input, 'foo', common.mustCall((err, output) => {
assert.ifError(err);
assert.deepStrictEqual(output, item.json);
assert.deepStrictEqual(output.json, item.json);
}));
}));
});
59 changes: 39 additions & 20 deletions tools/doc/generate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,53 +22,72 @@
'use strict';

const fs = require('fs');
const path = require('path');
const unified = require('unified');
const markdown = require('remark-parse');
const remark2rehype = require('remark-rehype');
const raw = require('rehype-raw');
const htmlStringify = require('rehype-stringify');

const html = require('./html');
const json = require('./json');

// Parse the args.
// Don't use nopt or whatever for this. It's simple enough.

const args = process.argv.slice(2);
let format = 'json';
let filename = null;
let nodeVersion = null;
let analytics = null;
let outputDir = null;

args.forEach(function(arg) {
if (!arg.startsWith('--')) {
filename = arg;
} else if (arg.startsWith('--format=')) {
format = arg.replace(/^--format=/, '');
} else if (arg.startsWith('--node-version=')) {
nodeVersion = arg.replace(/^--node-version=/, '');
} else if (arg.startsWith('--analytics=')) {
analytics = arg.replace(/^--analytics=/, '');
} else if (arg.startsWith('--output-directory=')) {
outputDir = arg.replace(/^--output-directory=/, '');
}
});

nodeVersion = nodeVersion || process.version;

if (!filename) {
throw new Error('No input file specified');
} else if (!outputDir) {
throw new Error('No output directory specified');
}


fs.readFile(filename, 'utf8', (er, input) => {
if (er) throw er;
switch (format) {
case 'json':
require('./json.js')(input, filename, (er, obj) => {
if (er) throw er;
console.log(JSON.stringify(obj, null, 2));
});
break;

case 'html':
require('./html')({ input, filename, nodeVersion, analytics },
(err, html) => {
if (err) throw err;
console.log(html);
});
break;
const content = unified()
.use(markdown)
.use(json.jsonAPI, { filename })
.use(html.firstHeader)
.use(html.preprocessText)
.use(html.preprocessElements, { filename })
.use(html.buildToc, { filename })
.use(remark2rehype, { allowDangerousHTML: true })
.use(raw)
.use(htmlStringify)
.processSync(input);

default:
throw new Error(`Invalid format: ${format}`);
}
const basename = path.basename(filename, '.md');

html.toHTML(
{ input, content, filename, nodeVersion, analytics },
(err, html) => {
const target = path.join(outputDir, `${basename}.html`);
if (err) throw err;
fs.writeFileSync(target, html);
}
);

const target = path.join(outputDir, `${basename}.json`);
fs.writeFileSync(target, JSON.stringify(content.json, null, 2));
});
25 changes: 8 additions & 17 deletions tools/doc/html.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@ const visit = require('unist-util-visit');
const markdown = require('remark-parse');
const remark2rehype = require('remark-rehype');
const raw = require('rehype-raw');
const html = require('rehype-stringify');
const htmlStringify = require('rehype-stringify');
const path = require('path');
const typeParser = require('./type-parser.js');

module.exports = toHTML;
module.exports = {
toHTML, firstHeader, preprocessText, preprocessElements, buildToc
};

const docPath = path.resolve(__dirname, '..', '..', 'doc');

Expand All @@ -54,26 +56,15 @@ const gtocHTML = unified()
.use(remark2rehype, { allowDangerousHTML: true })
.use(raw)
.use(navClasses)
.use(html)
.use(htmlStringify)
.processSync(gtocMD).toString();

const templatePath = path.join(docPath, 'template.html');
const template = fs.readFileSync(templatePath, 'utf8');

function toHTML({ input, filename, nodeVersion, analytics }, cb) {
function toHTML({ input, content, filename, nodeVersion, analytics }, cb) {
filename = path.basename(filename, '.md');

const content = unified()
.use(markdown)
.use(firstHeader)
.use(preprocessText)
.use(preprocessElements, { filename })
.use(buildToc, { filename })
.use(remark2rehype, { allowDangerousHTML: true })
.use(raw)
.use(html)
.processSync(input);

const id = filename.replace(/\W+/g, '-');

let HTML = template.replace('__ID__', id)
Expand Down Expand Up @@ -296,7 +287,7 @@ function parseYAML(text) {
.use(markdown)
.use(remark2rehype, { allowDangerousHTML: true })
.use(raw)
.use(html)
.use(htmlStringify)
.processSync(change.description).toString();

result += `<tr><td>${change.version}</td>\n` +
Expand Down Expand Up @@ -381,7 +372,7 @@ function buildToc({ filename }) {
.use(markdown)
.use(remark2rehype, { allowDangerousHTML: true })
.use(raw)
.use(html)
.use(htmlStringify)
.processSync(toc).toString();
};
}
Expand Down
Loading

0 comments on commit c85d00b

Please sign in to comment.