From de08352f4e83bd2836baa0536cca16058c5d146b Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Wed, 25 Apr 2018 18:24:27 +0300 Subject: [PATCH 1/2] tools: overhaul tools/doc/html.js --- tools/doc/html.js | 412 ++++++++++++++++++---------------------------- 1 file changed, 159 insertions(+), 253 deletions(-) diff --git a/tools/doc/html.js b/tools/doc/html.js index 00fd48d8e6631f..7bab4003ab72b8 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -29,17 +29,10 @@ const typeParser = require('./type-parser.js'); module.exports = toHTML; -const STABILITY_TEXT_REG_EXP = /(.*:)\s*(\d)([\s\S]*)/; -const DOC_CREATED_REG_EXP = //; - -// Customized heading without id attribute. +// Make `marked` to not automatically insert id attributes in headings. const renderer = new marked.Renderer(); -renderer.heading = function(text, level) { - return `${text}\n`; -}; -marked.setOptions({ - renderer: renderer -}); +renderer.heading = (text, level) => `${text}\n`; +marked.setOptions({ renderer }); const docPath = path.resolve(__dirname, '..', '..', 'doc'); @@ -47,91 +40,39 @@ const gtocPath = path.join(docPath, 'api', '_toc.md'); const gtocMD = fs.readFileSync(gtocPath, 'utf8').replace(/^@\/\/.*$/gm, ''); const gtocHTML = marked(gtocMD).replace( / ` ` type === 'heading'); + const section = firstHeading ? firstHeading.text : 'Index'; - // Get the section. - const section = getSection(lexed); + preprocessText(lexed); + preprocessElements(lexed); - filename = path.basename(filename, '.md'); + // Generate the table of contents. This mutates the lexed contents in-place. + const toc = buildToc(lexed, filename); - parseText(lexed); - lexed = preprocessElements(lexed); - - // Generate the table of contents. - // This mutates the lexed contents in-place. - buildToc(lexed, filename, function(er, toc) { - if (er) return cb(er); - - const id = toID(path.basename(filename)); - - template = template.replace(/__ID__/g, id); - template = template.replace(/__FILENAME__/g, filename); - template = template.replace(/__SECTION__/g, section || 'Index'); - template = template.replace(/__VERSION__/g, nodeVersion); - template = template.replace(/__TOC__/g, toc); - template = template.replace( - /__GTOC__/g, - gtocHTML.replace(`class="nav-${id}`, `class="nav-${id} active`) - ); - - if (opts.analytics) { - template = template.replace( - '', - analyticsScript(opts.analytics) - ); - } + const id = filename.replace(/\W+/g, '-'); - template = template.replace(/__ALTDOCS__/, altDocs(filename)); + let HTML = template.replace('__ID__', id) + .replace(/__FILENAME__/g, filename) + .replace('__SECTION__', section) + .replace(/__VERSION__/g, nodeVersion) + .replace('__TOC__', toc) + .replace('__GTOC__', gtocHTML.replace( + `class="nav-${id}`, `class="nav-${id} active`)); - // Content has to be the last thing we do with the lexed tokens, - // because it's destructive. - const content = marked.parser(lexed); - template = template.replace(/__CONTENT__/g, content); - - cb(null, template); - }); -} - -function analyticsScript(analytics) { - return ` + if (analytics) { + HTML = HTML.replace('', ` - `; + `); + } + + const docCreated = input.match( + //); + if (docCreated) { + HTML = HTML.replace('__ALTDOCS__', altDocs(filename, docCreated)); + } else { + console.error(`Failed to add alternative version links to ${filename}`); + HTML = HTML.replace('__ALTDOCS__', ''); + } + + // Content insertion has to be the last thing we do with the lexed tokens, + // because it's destructive. + HTML = HTML.replace('__CONTENT__', marked.parser(lexed)); + + cb(null, HTML); } // Replace placeholders in text tokens. function replaceInText(text) { + if (text === '') return text; return linkJsTypeDocs(linkManPages(text)); } -function altDocs(filename) { - if (!docCreated) { - console.error(`Failed to add alternative version links to ${filename}`); - return ''; - } - - function lte(v) { - const ns = v.num.split('.'); - if (docCreated[1] > +ns[0]) - return false; - if (docCreated[1] < +ns[0]) - return true; - return docCreated[2] <= +ns[1]; - } - +function altDocs(filename, docCreated) { + const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number); + const host = 'https://nodejs.org'; const versions = [ { num: '10.x' }, { num: '9.x' }, @@ -179,113 +124,101 @@ function altDocs(filename) { { num: '0.10.x' } ]; - const host = 'https://nodejs.org'; - const href = (v) => `${host}/docs/latest-v${v.num}/api/${filename}.html`; - - function li(v) { - let html = `
  • ${v.num}`; + const getHref = (versionNum) => + `${host}/docs/latest-v${versionNum}/api/${filename}.html`; - if (v.lts) - html += ' LTS'; + const wrapInListItem = (version) => + `
  • ${version.num}` + + `${version.lts ? ' LTS' : ''}
    • `; - return html + ''; + function isDocInVersion(version) { + const [versionMajor, versionMinor] = version.num.split('.').map(Number); + if (docCreatedMajor > versionMajor) return false; + if (docCreatedMajor < versionMajor) return true; + return docCreatedMinor <= versionMinor; } - const lis = versions.filter(lte).map(li).join('\n'); - - if (!lis.length) - return ''; + const list = versions.filter(isDocInVersion).map(wrapInListItem).join('\n'); - return ` + return list ? `
  • View another version ▼ -
      ${lis}
    +
      ${list}
    • - `; + ` : ''; } // Handle general body-text replacements. // For example, link man page references to the actual page. -function parseText(lexed) { - lexed.forEach(function(tok) { - if (tok.type === 'table') { - if (tok.cells) { - tok.cells.forEach((row, x) => { - row.forEach((_, y) => { - if (tok.cells[x] && tok.cells[x][y]) { - tok.cells[x][y] = replaceInText(tok.cells[x][y]); - } - }); - }); +function preprocessText(lexed) { + lexed.forEach((token) => { + if (token.type === 'table') { + if (token.header) { + token.header = token.header.map(replaceInText); } - if (tok.header) { - tok.header.forEach((_, i) => { - if (tok.header[i]) { - tok.header[i] = replaceInText(tok.header[i]); - } + if (token.cells) { + token.cells.forEach((row, i) => { + token.cells[i] = row.map(replaceInText); }); } - } else if (tok.text && tok.type !== 'code') { - tok.text = replaceInText(tok.text); + } else if (token.text && token.type !== 'code') { + token.text = replaceInText(token.text); } }); } // Preprocess stability blockquotes and YAML blocks. -function preprocessElements(input) { - var state = null; - const output = []; +function preprocessElements(lexed) { + const STABILITY_RE = /(.*:)\s*(\d)([\s\S]*)/; + let state = null; let headingIndex = -1; let heading = null; - output.links = input.links; - input.forEach(function(tok, index) { - if (tok.type === 'heading') { + lexed.forEach((token, index) => { + if (token.type === 'heading') { headingIndex = index; - heading = tok; + heading = token; } - if (tok.type === 'html' && common.isYAMLBlock(tok.text)) { - tok.text = parseYAML(tok.text); + if (token.type === 'html' && common.isYAMLBlock(token.text)) { + token.text = parseYAML(token.text); } - if (tok.type === 'blockquote_start') { + if (token.type === 'blockquote_start') { state = 'MAYBE_STABILITY_BQ'; - return; + lexed[index] = { type: 'space' }; } - if (tok.type === 'blockquote_end' && state === 'MAYBE_STABILITY_BQ') { + if (token.type === 'blockquote_end' && state === 'MAYBE_STABILITY_BQ') { state = null; - return; + lexed[index] = { type: 'space' }; } - if (tok.type === 'paragraph' && state === 'MAYBE_STABILITY_BQ') { - if (tok.text.match(/Stability:.*/g)) { - const stabilityMatch = tok.text.match(STABILITY_TEXT_REG_EXP); - const stability = Number(stabilityMatch[2]); + if (token.type === 'paragraph' && state === 'MAYBE_STABILITY_BQ') { + if (token.text.includes('Stability:')) { + const [, prefix, number, explication] = token.text.match(STABILITY_RE); const isStabilityIndex = index - 2 === headingIndex || // General. index - 3 === headingIndex; // With api_metadata block. if (heading && isStabilityIndex) { - heading.stability = stability; + heading.stability = number; headingIndex = -1; heading = null; } - tok.text = parseAPIHeader(tok.text).replace(/\n/g, ' '); - output.push({ type: 'html', text: tok.text }); - return; + token.text = `
    ` + + '' + + `${prefix} ${number}${explication}
    ` + .replace(/\n/g, ' '); + lexed[index] = { type: 'html', text: token.text }; } else if (state === 'MAYBE_STABILITY_BQ') { - output.push({ type: 'blockquote_start' }); state = null; + lexed[index - 1] = { type: 'blockquote_start' }; } } - output.push(tok); }); - - return output; } function parseYAML(text) { const meta = common.extractAndParseYAML(text); - const html = ['
    ']; + let html = '
    \n'; const added = { description: '' }; const deprecated = { description: '' }; @@ -302,153 +235,126 @@ function parseYAML(text) { } if (meta.changes.length > 0) { - let changes = meta.changes.slice(); - if (added.description) changes.push(added); - if (deprecated.description) changes.push(deprecated); + if (added.description) meta.changes.push(added); + if (deprecated.description) meta.changes.push(deprecated); - changes = changes.sort((a, b) => versionSort(a.version, b.version)); + meta.changes.sort((a, b) => versionSort(a.version, b.version)); - html.push('
    History'); - html.push(''); - html.push(''); + html += '
    History\n' + + '
    VersionChanges
    \n\n'; - changes.forEach((change) => { - html.push(``); - html.push(``); + meta.changes.forEach((change) => { + html += `\n` + + `\n`; }); - html.push('
    VersionChanges
    ${change.version}${marked(change.description)}
    ${change.version}${marked(change.description)}
    '); - html.push('
    '); + html += '\n\n'; } else { - html.push(`${added.description}${deprecated.description}`); + html += `${added.description}${deprecated.description}\n`; } - html.push('
    '); - return html.join('\n'); + html += '
    '; + return html; } // Syscalls which appear in the docs, but which only exist in BSD / macOS. const BSD_ONLY_SYSCALLS = new Set(['lchmod']); +const MAN_PAGE = /(^|\s)([a-z.]+)\((\d)([a-z]?)\)/gm; // Handle references to man pages, eg "open(2)" or "lchmod(2)". // Returns modified text, with such refs replaced with HTML links, for example // 'open(2)'. function linkManPages(text) { return text.replace( - /(^|\s)([a-z.]+)\((\d)([a-z]?)\)/gm, - (match, beginning, name, number, optionalCharacter) => { - // Name consists of lowercase letters, number is a single digit. + MAN_PAGE, (match, beginning, name, number, optionalCharacter) => { + // Name consists of lowercase letters, + // number is a single digit with an optional lowercase letter. const displayAs = `${name}(${number}${optionalCharacter})`; + if (BSD_ONLY_SYSCALLS.has(name)) { - return `${beginning}${displayAs}`; - } else { - return `${beginning}${displayAs}`; + return `${beginning}${displayAs}`; } + return `${beginning}${displayAs}`; }); } +const TYPE_SIGNATURE = /\{[^}]+\}/g; function linkJsTypeDocs(text) { const parts = text.split('`'); - var i; - var typeMatches; // Handle types, for example the source Markdown might say - // "This argument should be a {Number} or {String}". - for (i = 0; i < parts.length; i += 2) { - typeMatches = parts[i].match(/\{([^}]+)\}/g); + // "This argument should be a {number} or {string}". + for (let i = 0; i < parts.length; i += 2) { + const typeMatches = parts[i].match(TYPE_SIGNATURE); if (typeMatches) { - typeMatches.forEach(function(typeMatch) { + typeMatches.forEach((typeMatch) => { parts[i] = parts[i].replace(typeMatch, typeParser.toLink(typeMatch)); }); } } - // TODO: maybe put more stuff here? return parts.join('`'); } -function parseAPIHeader(text) { - const classNames = 'api_stability api_stability_$2'; - const docsUrl = 'documentation.html#documentation_stability_index'; - - text = text.replace( - STABILITY_TEXT_REG_EXP, - `
    $1 $2$3
    ` - ); - return text; -} - -// Section is just the first heading. -function getSection(lexed) { - for (var i = 0, l = lexed.length; i < l; i++) { - var tok = lexed[i]; - if (tok.type === 'heading') return tok.text; - } - return ''; -} - -function getMark(anchor) { - return `#`; -} - -function buildToc(lexed, filename, cb) { - var toc = []; - var depth = 0; - +function buildToc(lexed, filename) { const startIncludeRefRE = /^\s*\s*$/; - const endIncludeRefRE = /^\s*\s*$/; + const endIncludeRefRE = /^\s*\s*$/; const realFilenames = [filename]; - - lexed.forEach(function(tok) { - // Keep track of the current filename along @include directives. - if (tok.type === 'html') { - let match; - if ((match = tok.text.match(startIncludeRefRE)) !== null) - realFilenames.unshift(match[1]); - else if (tok.text.match(endIncludeRefRE)) + const idCounters = Object.create(null); + let toc = ''; + let depth = 0; + + lexed.forEach((token) => { + // Keep track of the current filename along comment wrappers of inclusions. + if (token.type === 'html') { + const [, includedFileName] = token.text.match(startIncludeRefRE) || []; + if (includedFileName !== undefined) + realFilenames.unshift(includedFileName); + else if (endIncludeRefRE.test(token.text)) realFilenames.shift(); } - if (tok.type !== 'heading') return; - if (tok.depth - depth > 1) { - return cb(new Error('Inappropriate heading level\n' + - JSON.stringify(tok))); + if (token.type !== 'heading') return; + + if (token.depth - depth > 1) { + throw new Error(`Inappropriate heading level:\n${JSON.stringify(token)}`); } - depth = tok.depth; + depth = token.depth; const realFilename = path.basename(realFilenames[0], '.md'); - const apiName = tok.text.trim(); - const id = getId(`${realFilename}_${apiName}`); - toc.push(new Array((depth - 1) * 2 + 1).join(' ') + - `* ` + - `${tok.text}`); - tok.text += getMark(id); - if (realFilename === 'errors' && apiName.startsWith('ERR_')) { - tok.text += getMark(apiName); + const headingText = token.text.trim(); + const id = getId(`${realFilename}_${headingText}`, idCounters); + toc += ' '.repeat((depth - 1) * 2) + + `* ` + + `${token.text}\n`; + token.text += `#`; + if (realFilename === 'errors' && headingText.startsWith('ERR_')) { + token.text += `#`; } }); - toc = marked.parse(toc.join('\n')); - cb(null, toc); + return marked(toc); } -const idCounters = {}; -function getId(text) { - text = text.toLowerCase(); - text = text.replace(/[^a-z0-9]+/g, '_'); - text = text.replace(/^_+|_+$/, ''); - text = text.replace(/^([^a-z])/, '_$1'); - if (idCounters.hasOwnProperty(text)) { - text += `_${++idCounters[text]}`; - } else { - idCounters[text] = 0; +const notAlphaNumerics = /[^a-z0-9]+/g; +const edgeUnderscores = /^_+|_+$/g; +const notAlphaStart = /^[^a-z]/; +function getId(text, idCounters) { + text = text.toLowerCase() + .replace(notAlphaNumerics, '_') + .replace(edgeUnderscores, '') + .replace(notAlphaStart, '_$&'); + if (idCounters[text] !== undefined) { + return `${text}_${++idCounters[text]}`; } + idCounters[text] = 0; return text; } -const numberRe = /^(\d*)/; +const numberRe = /^\d*/; function versionSort(a, b) { a = a.trim(); b = b.trim(); @@ -456,5 +362,5 @@ function versionSort(a, b) { while (i < a.length && i < b.length && a[i] === b[i]) i++; a = a.substr(i); b = b.substr(i); - return +b.match(numberRe)[1] - +a.match(numberRe)[1]; + return +b.match(numberRe)[0] - +a.match(numberRe)[0]; } From 071e34a568588662c16ec3c1d5a36c03256ed18e Mon Sep 17 00:00:00 2001 From: Vse Mozhet Byt Date: Wed, 9 May 2018 01:48:53 +0300 Subject: [PATCH 2/2] fixup: reorder function declarations --- tools/doc/html.js | 186 +++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/tools/doc/html.js b/tools/doc/html.js index 7bab4003ab72b8..60cacd5b791bcb 100644 --- a/tools/doc/html.js +++ b/tools/doc/html.js @@ -103,51 +103,6 @@ function toHTML({ input, filename, nodeVersion, analytics }, cb) { cb(null, HTML); } -// Replace placeholders in text tokens. -function replaceInText(text) { - if (text === '') return text; - return linkJsTypeDocs(linkManPages(text)); -} - -function altDocs(filename, docCreated) { - const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number); - const host = 'https://nodejs.org'; - const versions = [ - { num: '10.x' }, - { num: '9.x' }, - { num: '8.x', lts: true }, - { num: '7.x' }, - { num: '6.x', lts: true }, - { num: '5.x' }, - { num: '4.x', lts: true }, - { num: '0.12.x' }, - { num: '0.10.x' } - ]; - - const getHref = (versionNum) => - `${host}/docs/latest-v${versionNum}/api/${filename}.html`; - - const wrapInListItem = (version) => - `
  • ${version.num}` + - `${version.lts ? ' LTS' : ''}
    • `; - - function isDocInVersion(version) { - const [versionMajor, versionMinor] = version.num.split('.').map(Number); - if (docCreatedMajor > versionMajor) return false; - if (docCreatedMajor < versionMajor) return true; - return docCreatedMinor <= versionMinor; - } - - const list = versions.filter(isDocInVersion).map(wrapInListItem).join('\n'); - - return list ? ` -
  • - View another version ▼ -
      ${list}
    -
    • - ` : ''; -} - // Handle general body-text replacements. // For example, link man page references to the actual page. function preprocessText(lexed) { @@ -168,6 +123,53 @@ function preprocessText(lexed) { }); } +// Replace placeholders in text tokens. +function replaceInText(text) { + if (text === '') return text; + return linkJsTypeDocs(linkManPages(text)); +} + +// Syscalls which appear in the docs, but which only exist in BSD / macOS. +const BSD_ONLY_SYSCALLS = new Set(['lchmod']); +const MAN_PAGE = /(^|\s)([a-z.]+)\((\d)([a-z]?)\)/gm; + +// Handle references to man pages, eg "open(2)" or "lchmod(2)". +// Returns modified text, with such refs replaced with HTML links, for example +// 'open(2)'. +function linkManPages(text) { + return text.replace( + MAN_PAGE, (match, beginning, name, number, optionalCharacter) => { + // Name consists of lowercase letters, + // number is a single digit with an optional lowercase letter. + const displayAs = `${name}(${number}${optionalCharacter})`; + + if (BSD_ONLY_SYSCALLS.has(name)) { + return `${beginning}${displayAs}`; + } + return `${beginning}${displayAs}`; + }); +} + +const TYPE_SIGNATURE = /\{[^}]+\}/g; +function linkJsTypeDocs(text) { + const parts = text.split('`'); + + // Handle types, for example the source Markdown might say + // "This argument should be a {number} or {string}". + for (let i = 0; i < parts.length; i += 2) { + const typeMatches = parts[i].match(TYPE_SIGNATURE); + if (typeMatches) { + typeMatches.forEach((typeMatch) => { + parts[i] = parts[i].replace(typeMatch, typeParser.toLink(typeMatch)); + }); + } + } + + return parts.join('`'); +} + // Preprocess stability blockquotes and YAML blocks. function preprocessElements(lexed) { const STABILITY_RE = /(.*:)\s*(\d)([\s\S]*)/; @@ -257,45 +259,15 @@ function parseYAML(text) { return html; } -// Syscalls which appear in the docs, but which only exist in BSD / macOS. -const BSD_ONLY_SYSCALLS = new Set(['lchmod']); -const MAN_PAGE = /(^|\s)([a-z.]+)\((\d)([a-z]?)\)/gm; - -// Handle references to man pages, eg "open(2)" or "lchmod(2)". -// Returns modified text, with such refs replaced with HTML links, for example -// 'open(2)'. -function linkManPages(text) { - return text.replace( - MAN_PAGE, (match, beginning, name, number, optionalCharacter) => { - // Name consists of lowercase letters, - // number is a single digit with an optional lowercase letter. - const displayAs = `${name}(${number}${optionalCharacter})`; - - if (BSD_ONLY_SYSCALLS.has(name)) { - return `${beginning}${displayAs}`; - } - return `${beginning}${displayAs}`; - }); -} - -const TYPE_SIGNATURE = /\{[^}]+\}/g; -function linkJsTypeDocs(text) { - const parts = text.split('`'); - - // Handle types, for example the source Markdown might say - // "This argument should be a {number} or {string}". - for (let i = 0; i < parts.length; i += 2) { - const typeMatches = parts[i].match(TYPE_SIGNATURE); - if (typeMatches) { - typeMatches.forEach((typeMatch) => { - parts[i] = parts[i].replace(typeMatch, typeParser.toLink(typeMatch)); - }); - } - } - - return parts.join('`'); +const numberRe = /^\d*/; +function versionSort(a, b) { + a = a.trim(); + b = b.trim(); + let i = 0; // Common prefix length. + while (i < a.length && i < b.length && a[i] === b[i]) i++; + a = a.substr(i); + b = b.substr(i); + return +b.match(numberRe)[0] - +a.match(numberRe)[0]; } function buildToc(lexed, filename) { @@ -354,13 +326,41 @@ function getId(text, idCounters) { return text; } -const numberRe = /^\d*/; -function versionSort(a, b) { - a = a.trim(); - b = b.trim(); - let i = 0; // Common prefix length. - while (i < a.length && i < b.length && a[i] === b[i]) i++; - a = a.substr(i); - b = b.substr(i); - return +b.match(numberRe)[0] - +a.match(numberRe)[0]; +function altDocs(filename, docCreated) { + const [, docCreatedMajor, docCreatedMinor] = docCreated.map(Number); + const host = 'https://nodejs.org'; + const versions = [ + { num: '10.x' }, + { num: '9.x' }, + { num: '8.x', lts: true }, + { num: '7.x' }, + { num: '6.x', lts: true }, + { num: '5.x' }, + { num: '4.x', lts: true }, + { num: '0.12.x' }, + { num: '0.10.x' } + ]; + + const getHref = (versionNum) => + `${host}/docs/latest-v${versionNum}/api/${filename}.html`; + + const wrapInListItem = (version) => + `
  • ${version.num}` + + `${version.lts ? ' LTS' : ''}
    • `; + + function isDocInVersion(version) { + const [versionMajor, versionMinor] = version.num.split('.').map(Number); + if (docCreatedMajor > versionMajor) return false; + if (docCreatedMajor < versionMajor) return true; + return docCreatedMinor <= versionMinor; + } + + const list = versions.filter(isDocInVersion).map(wrapInListItem).join('\n'); + + return list ? ` +
  • + View another version ▼ +
      ${list}
    +
    • + ` : ''; }