From e54e7acc2328a2eefd7feff293685b29ba6cd3b8 Mon Sep 17 00:00:00 2001 From: fmvilas Date: Fri, 20 Mar 2020 14:18:06 +0100 Subject: [PATCH 1/9] Add support for remote templates :tada: --- cli.js | 23 ++++---- lib/generator.js | 150 +++++++++++++++++++++++++++-------------------- lib/utils.js | 48 +++++++++++++++ 3 files changed, 147 insertions(+), 74 deletions(-) create mode 100644 lib/utils.js diff --git a/cli.js b/cli.js index f4d2a8327..0da836058 100755 --- a/cli.js +++ b/cli.js @@ -7,6 +7,7 @@ const packageInfo = require('./package.json'); const mkdirp = require('mkdirp'); const Generator = require('./lib/generator'); const Watcher = require('./lib/watcher'); +const { isLocalTemplate } = require('./lib/utils'); const red = text => `\x1b[31m${text}\x1b[0m`; const magenta = text => `\x1b[35m${text}\x1b[0m`; @@ -52,14 +53,13 @@ program asyncapiFile = path.resolve(asyncAPIPath); template = tmpl; }) - .option('-w, --watch', 'watches the templates directory and the AsyncAPI document for changes, and re-generate the files when they occur') - .option('-o, --output ', 'directory where to put the generated files (defaults to current directory)', parseOutput, process.cwd()) .option('-d, --disable-hook ', 'disable a specific hook', disableHooksParser) + .option('-i, --install', 'installs the template and its dependencies (defaults to false)') .option('-n, --no-overwrite ', 'glob or path of the file(s) to skip when regenerating', noOverwriteParser) + .option('-o, --output ', 'directory where to put the generated files (defaults to current directory)', parseOutput, process.cwd()) .option('-p, --param ', 'additional param to pass to templates', paramParser) - .option('-t, --templates ', 'directory where templates are located (defaults to internal templates directory)', Generator.DEFAULT_TEMPLATES_DIR, path.resolve(__dirname, 'templates')) - .option('--force-install', 'forces the installation of the template dependencies. By default, dependencies are installed and this flag is taken into account only if `node_modules` is not in place.') .option('--force-write', 'force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir (defaults to false)') + .option('--watch-template', 'watches the template directory and the AsyncAPI document, and re-generate the files when changes they occur') .parse(process.argv); if (!asyncapiFile) { @@ -76,15 +76,19 @@ mkdirp(program.output, async err => { } // If we want to watch for changes do that - if (program.watch) { - const watchDir = path.resolve(program.templates, template); - console.log(`[WATCHER] Watching for changes in the template directory ${magenta(watchDir)} and in the async api file ${magenta(asyncapiFile)}`); + if (program.watchTemplate) { + const watchDir = path.resolve(Generator.DEFAULT_TEMPLATES_DIR, template); + console.log(`[WATCHER] Watching for changes in the template directory ${magenta(watchDir)} and in the AsyncAPI file ${magenta(asyncapiFile)}`); + + if (!(await isLocalTemplate(watchDir))) { + console.warn(`WARNING: ${template} is a remote template. Changes may be lost on subsequent installations.`); + } const watcher = new Watcher([asyncapiFile, watchDir]); watcher.watch(async (changedFiles) => { console.clear(); console.log('[WATCHER] Change detected'); - for (const [key, value] of Object.entries(changedFiles)) { + for (const [, value] of Object.entries(changedFiles)) { let eventText; switch (value.eventType) { case 'changed': @@ -121,12 +125,11 @@ function generate(targetDir) { return new Promise(async (resolve, reject) => { try { const generator = new Generator(template, targetDir || path.resolve(os.tmpdir(), 'asyncapi-generator'), { - templatesDir: program.templates, templateParams: params, noOverwriteGlobs, disabledHooks, forceWrite: program.forceWrite, - forceInstall: program.forceInstall, + forceInstall: program.install, }); await generator.generateFromFile(asyncapiFile); diff --git a/lib/generator.js b/lib/generator.js index 89cd2dca5..91e774c7d 100644 --- a/lib/generator.js +++ b/lib/generator.js @@ -14,6 +14,13 @@ const Ajv = require('ajv'); const filenamify = require('filenamify'); const git = require('simple-git/promise'); const npmi = require('npmi'); +const { + convertMapToObject, + isFileSystemPath, + beautifyNpmiResult, + isLocalTemplate, + getLocalTemplateDetails, +} = require('./utils'); const ajv = new Ajv({ allErrors: true }); @@ -40,7 +47,8 @@ const NODE_MODULES_DIRNAME = 'node_modules'; const CONFIG_FILENAME = '.tp-config.json'; const PACKAGE_JSON_FILENAME = 'package.json'; const PACKAGE_LOCK_FILENAME = 'package-lock.json'; -const DEFAULT_TEMPLATES_DIR = path.resolve(__dirname, '..', 'templates'); +const ROOT_DIR = path.resolve(__dirname, '..'); +const DEFAULT_TEMPLATES_DIR = path.resolve(ROOT_DIR, 'node_modules'); const shouldIgnoreFile = filePath => filePath.startsWith(`${PARTIALS_DIRNAME}${path.sep}`) @@ -77,33 +85,24 @@ class Generator { * } * }); * - * @example Specifying a custom directory for templates - * const path = require('path'); - * const generator = new Generator('myTemplate', path.resolve(__dirname, 'example'), { - * templatesDir: path.resolve(__dirname, 'my-templates') - * }); - * * @param {String} templateName Name of the template to generate. * @param {String} targetDir Path to the directory where the files will be generated. * @param {Object} options - * @param {String} [options.templatesDir] Path to the directory where to find the given template. Defaults to internal `templates` directory. * @param {String} [options.templateParams] Optional parameters to pass to the template. Each template define their own params. * @param {String} [options.entrypoint] Name of the file to use as the entry point for the rendering process. Use in case you want to use only a specific template file. Note: this potentially avoids rendering every file in the template. * @param {String[]} [options.noOverwriteGlobs] List of globs to skip when regenerating the template. * @param {String[]} [options.disabledHooks] List of hooks to disable. * @param {String} [options.output='fs'] Type of output. Can be either 'fs' (default) or 'string'. Only available when entrypoint is set. * @param {Boolean} [options.forceWrite=false] Force writing of the generated files to given directory even if it is a git repo with unstaged files or not empty dir. Default is set to false. - * @param {Boolean} [options.forceInstall=false] Force the installation of the template dependencies. By default, dependencies are installed and this flag is taken into account only if `node_modules` is not in place. + * @param {Boolean} [options.forceInstall=false] Force the installation of the template and its dependencies. */ - constructor(templateName, targetDir, { templatesDir, templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, forceInstall = false } = {}) { + constructor(templateName, targetDir, { templateParams = {}, entrypoint, noOverwriteGlobs, disabledHooks, output = 'fs', forceWrite = false, forceInstall = false } = {}) { if (!templateName) throw new Error('No template name has been specified.'); if (!entrypoint && !targetDir) throw new Error('No target directory has been specified.'); if (!['fs', 'string'].includes(output)) throw new Error(`Invalid output type ${output}. Valid values are 'fs' and 'string'.`); this.templateName = templateName; this.targetDir = targetDir; - this.templatesDir = templatesDir || DEFAULT_TEMPLATES_DIR; - this.templateDir = path.resolve(this.templatesDir, this.templateName); this.entrypoint = entrypoint; this.noOverwriteGlobs = noOverwriteGlobs || []; this.disabledHooks = disabledHooks || []; @@ -111,10 +110,6 @@ class Generator { this.forceWrite = forceWrite; this.forceInstall = forceInstall; - // Config Nunjucks - this.nunjucks = new Nunjucks.Environment(new Nunjucks.FileSystemLoader(this.templateDir)); - this.nunjucks.addFilter('log', console.log); - // Load template configuration this.templateParams = {}; Object.keys(templateParams).forEach(key => { @@ -128,8 +123,6 @@ class Generator { } }); }); - this.loadTemplateConfig(); - this.registerHooks(); } /** @@ -159,7 +152,12 @@ class Generator { try { if (!this.forceWrite) await this.verifyTargetDir(this.targetDir); - await this.installTemplate(this.forceInstall); + const { name: templatePkgName, path: templatePkgPath } = await this.installTemplate(this.forceInstall); + this.templateDir = path.resolve(ROOT_DIR, templatePkgPath); + this.templateName = templatePkgName; + this.configNunjucks(); + this.loadTemplateConfig(); + this.registerHooks(); await this.registerFilters(); if (this.entrypoint) { @@ -275,22 +273,65 @@ class Generator { * const Generator = require('asyncapi-generator'); * const content = await Generator.getTemplateFile('html', '.partials/content.html'); * - * @example Obtaining the content of a file from a custom template - * const path = require('path'); - * const Generator = require('asyncapi-generator'); - * const content = await Generator.getTemplateFile('myTemplate', 'a/file.js', { - * templatesDir: path.resolve(__dirname, 'my-templates') - * }); - * * @static * @param {String} templateName Name of the template to generate. * @param {String} filePath Path to the file to render. Relative to the template directory. * @param {Object} options - * @param {String} [options.templatesDir] Path to the directory where to find the given template. Defaults to internal `templates` directory. * @return {Promise} */ - static async getTemplateFile(templateName, filePath, { templatesDir } = {}) { - return await readFile(path.resolve(templatesDir || DEFAULT_TEMPLATES_DIR, templateName, filePath), 'utf8'); + static async getTemplateFile(templateName, filePath) { + return await readFile(path.resolve(DEFAULT_TEMPLATES_DIR, templateName, filePath), 'utf8'); + } + + /** + * Downloads and installs a template and its dependencies. + * + * @param {Boolean} [force=false] Whether to force installation (and skip cache) or not. + */ + installTemplate(force = false) { + return new Promise(async (resolve, reject) => { + if (!force) { + try { + let installedPkg; + + if (isFileSystemPath(this.templateName)) { + const pkg = require(path.resolve(this.templateName, 'package.json')); + installedPkg = require(path.resolve(DEFAULT_TEMPLATES_DIR, pkg.name, 'package.json')); + if (installedPkg.name !== pkg.name) throw new Error('Package not installed'); + } else { // Template is not a filesystem path... + const templatePath = path.resolve(DEFAULT_TEMPLATES_DIR, this.templateName); + if (await isLocalTemplate(templatePath)) { + const { resolvedLink } = await getLocalTemplateDetails(templatePath); + console.warn(`WARNING: this template is already installed and pointing to your filesystem at ${resolvedLink}.`); + } + installedPkg = require(path.resolve(templatePath, 'package.json')); + } + + return resolve({ + name: installedPkg.name, + version: installedPkg.version, + path: path.resolve(DEFAULT_TEMPLATES_DIR, this.templateName), + }); + } catch (e) { + // We did our best. Proceed with installation... + } + } + + npmi({ + name: this.templateName, + forceInstall: force, + pkgName: 'dummy value so it does not force installation always', + npmLoad: { + loglevel: 'http', + save: false, + audit: false, + progress: false, + }, + }, (err, result) => { + if (err) return reject(err); + resolve(beautifyNpmiResult(result)); + }); + }); } /** @@ -330,16 +371,10 @@ class Generator { }); } - convertMapToObject(map) { - const tempObject = {}; - for (const [key, value] of map.entries()) { - tempObject[key] = value; - } - return tempObject; - } - /** + * Returns all the parameters on the AsyncAPI document. * + * @private * @param {AsyncAPIDocument} asyncapiDocument AsyncAPI document to use as the source. */ getAllParameters(asyncapiDocument) { @@ -373,11 +408,11 @@ class Generator { generateDirectoryStructure(asyncapiDocument) { const fileNamesForSeparation = { channel: asyncapiDocument.channels(), - message: this.convertMapToObject(asyncapiDocument.allMessages()), + message: convertMapToObject(asyncapiDocument.allMessages()), securityScheme: asyncapiDocument.components() ? asyncapiDocument.components().securitySchemes() : {}, schema: asyncapiDocument.components() ? asyncapiDocument.components().schemas() : {}, - parameter: this.convertMapToObject(this.getAllParameters(asyncapiDocument)), - everySchema: this.convertMapToObject(asyncapiDocument.allSchemas()), + parameter: convertMapToObject(this.getAllParameters(asyncapiDocument)), + everySchema: convertMapToObject(asyncapiDocument.allSchemas()), }; return new Promise((resolve, reject) => { @@ -611,6 +646,15 @@ class Generator { return !this.noOverwriteGlobs.some(globExp => minimatch(filePath, globExp)); } + /** + * Configures Nunjucks templating system + * @private + */ + configNunjucks() { + this.nunjucks = new Nunjucks.Environment(new Nunjucks.FileSystemLoader(this.templateDir)); + this.nunjucks.addFilter('log', console.log); + } + /** * Loads the template configuration. * @private @@ -757,30 +801,8 @@ class Generator { throw e; } } - - /** - * Installs template dependencies. - * - * @param {Boolean} [force=false] Whether to force installation or not. - */ - installTemplate(force = false) { - return new Promise(async (resolve, reject) => { - const nodeModulesDir = path.resolve(this.templateDir, 'node_modules'); - const templatePackageFile = path.resolve(this.templateDir, 'package.json'); - const templateDirExists = await exists(this.templateDir); - const templatePackageExists = await exists(templatePackageFile); - if (!templateDirExists) return reject(new Error(`Template "${this.templateName}" does not exist.`)); - if (!templatePackageExists) return reject(new Error(`Directory "${this.templateName}" is not a valid template. Please provide a package.json file.`)); - if (!force && await exists(nodeModulesDir)) return resolve(); - - npmi({ - path: this.templateDir, - }, err => { - if (err) return reject(err); - resolve(); - }); - }); - } } +Generator.DEFAULT_TEMPLATES_DIR = DEFAULT_TEMPLATES_DIR; + module.exports = Generator; diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 000000000..60a318638 --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,48 @@ +const fs = require('fs'); +const util = require('util'); +const path = require('path'); +const utils = module.exports; + +const lstat = util.promisify(fs.lstat); +const readlink = util.promisify(fs.readlink); + +utils.isFileSystemPath = (string) => { + return path.isAbsolute(string) + || string.startsWith(`.${path.sep}`) + || string.startsWith(`..${path.sep}`) + || string.startsWith('~'); +}; + +utils.beautifyNpmiResult = (result) => { + const [nameWithVersion, pkgPath] = result[0]; + const nameWithVersionArray = nameWithVersion.split('@'); + const version = nameWithVersionArray[nameWithVersionArray.length - 1]; + const name = nameWithVersionArray.splice(0, nameWithVersionArray.length - 1).join('@'); + + return { + name, + path: pkgPath, + version, + }; +}; + +utils.convertMapToObject = (map) => { + const tempObject = {}; + for (const [key, value] of map.entries()) { + tempObject[key] = value; + } + return tempObject; +}; + +utils.isLocalTemplate = async (templatePath) => { + const stats = await lstat(templatePath); + return stats.isSymbolicLink(); +}; + +utils.getLocalTemplateDetails = async (templatePath) => { + const linkTarget = await readlink(templatePath); + return { + link: linkTarget, + resolvedLink: path.resolve(path.dirname(templatePath), linkTarget), + }; +}; From 638d6d042248aa37e499310c427168850d61f704 Mon Sep 17 00:00:00 2001 From: fmvilas Date: Fri, 20 Mar 2020 18:50:43 +0100 Subject: [PATCH 2/9] Move some utility functions to utils package --- lib/generator.js | 12 +++++------- lib/utils.js | 43 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/generator.js b/lib/generator.js index 91e774c7d..cf9830f4b 100644 --- a/lib/generator.js +++ b/lib/generator.js @@ -1,6 +1,5 @@ const path = require('path'); const fs = require('fs'); -const util = require('util'); const xfs = require('fs.extra'); const walkSync = require('klaw-sync'); const minimatch = require('minimatch'); @@ -20,6 +19,11 @@ const { beautifyNpmiResult, isLocalTemplate, getLocalTemplateDetails, + readFile, + readDir, + writeFile, + copyFile, + exists, } = require('./utils'); const ajv = new Ajv({ allErrors: true }); @@ -34,12 +38,6 @@ parser.registerSchemaParser([ 'application/raml+yaml;version=1.0', ], ramlDtParser); -const readFile = util.promisify(fs.readFile); -const writeFile = util.promisify(fs.writeFile); -const copyFile = util.promisify(fs.copyFile); -const exists = util.promisify(fs.exists); -const readDir = util.promisify(fs.readdir); - const FILTERS_DIRNAME = '.filters'; const PARTIALS_DIRNAME = '.partials'; const HOOKS_DIRNAME = '.hooks'; diff --git a/lib/utils.js b/lib/utils.js index 60a318638..45fd12c1e 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -3,9 +3,20 @@ const util = require('util'); const path = require('path'); const utils = module.exports; -const lstat = util.promisify(fs.lstat); -const readlink = util.promisify(fs.readlink); +utils.lstat = util.promisify(fs.lstat); +utils.readlink = util.promisify(fs.readlink); +utils.readFile = util.promisify(fs.readFile); +utils.writeFile = util.promisify(fs.writeFile); +utils.copyFile = util.promisify(fs.copyFile); +utils.exists = util.promisify(fs.exists); +utils.readDir = util.promisify(fs.readdir); +/** + * Checks if a string is a filesystem path. + * @private + * @param {String} string The string to check. + * @returns {Boolean} Whether the string is a filesystem path or not. + */ utils.isFileSystemPath = (string) => { return path.isAbsolute(string) || string.startsWith(`.${path.sep}`) @@ -13,6 +24,12 @@ utils.isFileSystemPath = (string) => { || string.startsWith('~'); }; +/** + * Takes the result of calling the npmi module and returns it in a more consumable form. + * @private + * @param {Array} result The result of calling npmi. + * @returns {Object} + */ utils.beautifyNpmiResult = (result) => { const [nameWithVersion, pkgPath] = result[0]; const nameWithVersionArray = nameWithVersion.split('@'); @@ -26,6 +43,12 @@ utils.beautifyNpmiResult = (result) => { }; }; +/** + * Converts a Map into an object. + * @private + * @param {Map} map The map to transform. + * @returns {Object} + */ utils.convertMapToObject = (map) => { const tempObject = {}; for (const [key, value] of map.entries()) { @@ -34,13 +57,25 @@ utils.convertMapToObject = (map) => { return tempObject; }; +/** + * Checks if template is local or not (i.e., it's remote). + * @private + * @param {String} templatePath The path to the template. + * @returns {Promise} + */ utils.isLocalTemplate = async (templatePath) => { - const stats = await lstat(templatePath); + const stats = await utils.lstat(templatePath); return stats.isSymbolicLink(); }; +/** + * Gets the details (link and resolved link) of a local template. + * @private + * @param {String} templatePath The path to the template. + * @returns {Promise} + */ utils.getLocalTemplateDetails = async (templatePath) => { - const linkTarget = await readlink(templatePath); + const linkTarget = await utils.readlink(templatePath); return { link: linkTarget, resolvedLink: path.resolve(path.dirname(templatePath), linkTarget), From 5dc2e654ee221b894cb1c21f2f6a87e298bf83e5 Mon Sep 17 00:00:00 2001 From: fmvilas Date: Fri, 20 Mar 2020 18:50:56 +0100 Subject: [PATCH 3/9] Update docs --- README.md | 35 ++++++++++++++++------------------- cli.js | 2 +- docs/api.md | 33 +++------------------------------ 3 files changed, 20 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 9c1690abe..098eeafbc 100644 --- a/README.md +++ b/README.md @@ -34,40 +34,37 @@ asyncapi/generator -o ./output asyncapi.yml markdown ### From the command-line interface (CLI) ```bash - Usage: ag [options]