partial review addressed

Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>

bin/cli.mjs

@@ -9,8 +9,8 @@ import { coerce } from 'semver';
 import { DOC_NODE_CHANGELOG_URL, DOC_NODE_VERSION } from '../src/constants.mjs';
 import createGenerator from '../src/generators.mjs';
 import generators from '../src/generators/index.mjs';
-import createLoader from '../src/loader.mjs';
-import createParser from '../src/parser.mjs';
+import { createMarkdownLoader } from '../src/loader.mjs';
+import { createMarkdownParser } from '../src/parser.mjs';
 import createNodeReleases from '../src/releases.mjs';
 
 const availableGenerators = Object.keys(generators);
@@ -68,25 +68,23 @@ program
  */
 const { input, output, target = [], version, changelog } = program.opts();
 
-const { loadMarkdownFiles, loadJsFiles } = createLoader();
-const { parseApiDocs, parseJsSources } = createParser();
+const { loadFiles } = createMarkdownLoader();
+const { parseApiDocs } = createMarkdownParser();
 
-const apiDocFiles = loadMarkdownFiles(input);
+const apiDocFiles = loadFiles(input);
 
 const parsedApiDocs = await parseApiDocs(apiDocFiles);
 
-const sourceFiles = loadJsFiles(input);
-
-const parsedJsFiles = await parseJsSources(sourceFiles);
-
-const { runGenerators } = createGenerator(parsedApiDocs, parsedJsFiles);
+const { runGenerators } = createGenerator(parsedApiDocs);
 
 // Retrieves Node.js release metadata from a given Node.js version and CHANGELOG.md file
 const { getAllMajors } = createNodeReleases(changelog);
 
 await runGenerators({
   // A list of target modes for the API docs parser
   generators: target,
+  // Resolved `input` to be used
+  input: input,
   // Resolved `output` path to be used
   output: resolve(output),
   // Resolved SemVer of current Node.js version

src/generators/api-links/index.mjs

@@ -30,6 +30,8 @@ export default {
   description:
     'Creates a mapping of publicly accessible functions to their source locations in the Node.js repository.',
 
+  // Unlike the rest of the generators, this utilizes Javascript sources being
+  // passed into the input field rather than Markdown.
   dependsOn: 'ast-js',
 
   /**

src/generators/api-links/utils/getBaseGitHubUrl.mjs

@@ -13,14 +13,14 @@ export function getBaseGitHubUrl(cwd) {
     //  Ex/ git@github.com:nodejs/node.git -> https://github.com/nodejs/node.git
     let [, repository] = url.split(':');
 
-    // Trim off the trailing .git if it exists
-    if (repository.endsWith('.git')) {
-      repository = repository.substring(0, repository.length - 4);
-    }
-
     url = `https://github.com/${repository}`;
   }
 
+  // https://github.com/nodejs/node.git -> https://github.com/nodejs/node
+  if (url.endsWith('.git')) {
+    url = url.substring(0, url.length - 4);
+  }
+
   return url;
 }
 

src/generators/ast-js/index.mjs

@@ -0,0 +1,47 @@
+import { createJsLoader } from '../../loader.mjs';
+import { createJsParser } from '../../parser.mjs';
+
+/**
+ * This generator parses Javascript sources passed into the generator's input
+ * field. This is separate from the Markdown parsing step since it's not as
+ * commonly used and can take up a significant amount of memory.
+ *
+ * Putting this with the rest of the generators allows it to be lazily loaded
+ * so we're only parsing the Javascript sources when we need to.
+ *
+ * @typedef {unknown} Input
+ *
+ * @type {import('../types.d.ts').GeneratorMetadata<Input, Array<JsProgram>>}
+ */
+export default {
+  name: 'ast-js',
+
+  version: '1.0.0',
+
+  description: 'Parses Javascript source files passed into the input.',
+
+  dependsOn: 'ast',
+
+  /**
+   * @param {Input} _
+   * @param {Partial<GeneratorOptions>} options
+   */
+  async generate(_, options) {
+    const { loadFiles } = createJsLoader();
+
+    if (!options.input) {
+      return [];
+    }
+
+    // Load all of the Javascript sources into memory
+    const sourceFiles = loadFiles(options.input);
+
+    const { parseJsSources } = createJsParser();
+
+    // Parse the Javascript sources into ASTs
+    const parsedJsFiles = await parseJsSources(sourceFiles);
+
+    // Return the ASTs so they can be used in another generator
+    return parsedJsFiles;
+  },
+};

src/generators/index.mjs

@@ -8,6 +8,7 @@ import legacyJson from './legacy-json/index.mjs';
 import legacyJsonAll from './legacy-json-all/index.mjs';
 import addonVerify from './addon-verify/index.mjs';
 import apiLinks from './api-links/index.mjs';
+import astJs from './ast-js/index.mjs';
 
 export default {
   'json-simple': jsonSimple,
@@ -18,4 +19,5 @@ export default {
   'legacy-json-all': legacyJsonAll,
   'addon-verify': addonVerify,
   'api-links': apiLinks,
+  'ast-js': astJs,
 };

src/generators/types.d.ts

@@ -9,6 +9,10 @@ declare global {
 
   // This is the runtime config passed to the API doc generators
   export interface GeneratorOptions {
+    // The path to the input source files. This parameter accepts globs and can
+    // be a glob when passed to a generator.
+    input: string | string[];
+
     // The path used to output generated files, this is to be considered
     // the base path that any generator will use for generating files
     // This parameter accepts globs but when passed to generators will contain

src/index.mjs

@@ -1,8 +1,8 @@
 export * as constants from './constants.mjs';
 export { default as generators } from './generators/index.mjs';
 export { default as createGenerator } from './generators.mjs';
-export { default as createLoader } from './loader.mjs';
+export * from './loader.mjs';
 export { default as createMetadata } from './metadata.mjs';
-export { default as createParser } from './parser.mjs';
+export * from './parser.mjs';
 export { default as createQueries } from './queries.mjs';
 export { default as createNodeReleases } from './releases.mjs';

src/loader.mjs

@@ -11,7 +11,7 @@ import { VFile } from 'vfile';
  * could be used for different things, but here we want to use it to load
  * Markdown files and transform them into VFiles
  */
-const createLoader = () => {
+export const createMarkdownLoader = () => {
   /**
    * Loads API Doc files and transforms it into VFiles
    *
@@ -21,7 +21,7 @@ const createLoader = () => {
    *
    * @see https://code.visualstudio.com/docs/editor/glob-patterns
    */
-  const loadMarkdownFiles = searchPath => {
+  const loadFiles = searchPath => {
     const resolvedFiles = globSync(searchPath).filter(
       filePath => extname(filePath) === '.md'
     );
@@ -33,12 +33,19 @@ const createLoader = () => {
     });
   };
 
+  return { loadFiles };
+};
+
+/**
+ * This creates a "loader" for loading Javascript source files into VFiles.
+ */
+export const createJsLoader = () => {
   /**
    * Loads the JavaScript source files and transforms them into VFiles
    *
-   * @param {Array<string>} searchPath
+   * @param {string | Array<string>} searchPath
    */
-  const loadJsFiles = searchPath => {
+  const loadFiles = searchPath => {
     const resolvedFiles = globSync(searchPath).filter(
       filePath => extname(filePath) === '.js'
     );
@@ -50,7 +57,5 @@ const createLoader = () => {
     });
   };
 
-  return { loadMarkdownFiles, loadJsFiles };
+  return { loadFiles };
 };
-
-export default createLoader;

src/parser.mjs

@@ -16,7 +16,7 @@ import { createNodeSlugger } from './utils/slugger.mjs';
 /**
  * Creates an API doc parser for a given Markdown API doc file
  */
-const createParser = () => {
+export const createMarkdownParser = () => {
   // Creates an instance of the Remark processor with GFM support
   // which is used for stringifying the AST tree back to Markdown
   const remarkProcessor = getRemark();
@@ -183,38 +183,40 @@ const createParser = () => {
     return resolvedApiDocEntries.flat();
   };
 
+  return { parseApiDocs, parseApiDoc };
+};
+
+/**
+ * Creates a Javascript source parser for a given source file
+ */
+export const createJsParser = () => {
   /**
    * Parses a given JavaScript file into an ESTree AST representation of it
    *
-   * @param {import('vfile').VFile | Promise<import('vfile').VFile>} apiDoc
+   * @param {import('vfile').VFile | Promise<import('vfile').VFile>} sourceFile
    * @returns {Promise<JsProgram>}
    */
-  const parseJsSource = async apiDoc => {
+  const parseJsSource = async sourceFile => {
     // We allow the API doc VFile to be a Promise of a VFile also,
     // hence we want to ensure that it first resolves before we pass it to the parser
-    const resolvedApiDoc = await Promise.resolve(apiDoc);
+    const resolvedSourceFile = await Promise.resolve(sourceFile);
 
-    if (typeof resolvedApiDoc.value !== 'string') {
+    if (typeof resolvedSourceFile.value !== 'string') {
       throw new TypeError(
-        `expected resolvedApiDoc.value to be string but got ${typeof resolvedApiDoc.value}`
+        `expected resolvedSourceFile.value to be string but got ${typeof resolvedSourceFile.value}`
       );
     }
 
-    try {
-      const res = acorn.parse(resolvedApiDoc.value, {
-        allowReturnOutsideFunction: true,
-        ecmaVersion: 'latest',
-        locations: true,
-      });
-
-      return {
-        ...res,
-        path: resolvedApiDoc.path,
-      };
-    } catch (err) {
-      console.log(`error parsing ${resolvedApiDoc.basename}`);
-      throw err;
-    }
+    const res = acorn.parse(resolvedSourceFile.value, {
+      allowReturnOutsideFunction: true,
+      ecmaVersion: 'latest',
+      locations: true,
+    });
+
+    return {
+      ...res,
+      path: resolvedSourceFile.path,
+    };
   };
 
   /**
@@ -231,7 +233,5 @@ const createParser = () => {
     return resolvedApiDocEntries;
   };
 
-  return { parseApiDocs, parseApiDoc, parseJsSources, parseJsSource };
+  return { parseJsSource, parseJsSources };
 };
-
-export default createParser;