From 386d50e41c26c89dc5276613923ff3c743f35baf Mon Sep 17 00:00:00 2001 From: OnkarRuikar <87750369+OnkarRuikar@users.noreply.github.com> Date: Tue, 25 Apr 2023 16:03:25 +0530 Subject: [PATCH] feat(lint): Implement front matter validator and linter --- front-matter-config.json | 155 +++++++++++++++++++++ package.json | 5 + scripts/front-matter-linter.js | 238 +++++++++++++++++++++++++++++++++ yarn.lock | 51 +++++++ 4 files changed, 449 insertions(+) create mode 100644 front-matter-config.json create mode 100644 scripts/front-matter-linter.js diff --git a/front-matter-config.json b/front-matter-config.json new file mode 100644 index 000000000000000..04de645a0badce5 --- /dev/null +++ b/front-matter-config.json @@ -0,0 +1,155 @@ +{ + "lineWidth": 80, + "schema": { + "title": "Front matter schema", + "type": "object", + "additionalProperties": false, + "required": ["title", "slug"], + "properties": { + "title": { + "title": "Title", + "description": "Rendered page title and for SEO", + "type": "string", + "maxLength": 120 + }, + "short-title": { + "title": "Short Title", + "description": "To be used in sidebars", + "type": "string", + "maxLength": 60 + }, + "slug": { + "title": "slug", + "description": "URL path of the page", + "type": "string" + }, + "page-type": { + "title": "Page Type", + "description": "Type of the page", + "type": "string" + }, + "status": { + "title": "Status", + "description": "Browser compatibility status", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": ["deprecated", "experimental", "non-standard"] + } + }, + "browser-compat": { + "title": "Browser Compatibility", + "description": "Browser compatibility location", + "type": ["array", "string"] + }, + "spec-urls": { + "title": "Specification URLs", + "description": "Specification locations", + "type": ["array", "string"], + "format": "uri", + "items": { + "type": "string", + "format": "uri" + } + } + } + }, + "attribute-order": [ + "title", + "short-title", + "slug", + "page-type", + "status", + "browser-compat", + "spec-urls" + ], + "allowedPageTypes": { + "Glossary": ["glossary-definition", "glossary-disambiguation"], + "MDN/": ["landing-page", "mdn-community-guide", "mdn-writing-guide"], + "Web/SVG/": ["guide", "landing-page", "svg-attribute", "svg-element"], + "Web/HTML/": [ + "guide", + "landing-page", + "html-attribute", + "html-attribute-value", + "html-element" + ], + "Mozilla/Add-ons/WebExtensions/": [ + "guide", + "landing-page", + "webextension-api-function", + "webextension-api", + "webextension-api-event", + "webextension-api-property", + "webextension-api-type", + "webextension-manifest-key" + ], + "Web/API/": [ + "guide", + "landing-page", + "web-api-overview", + "web-api-global-function", + "web-api-global-property", + "web-api-interface", + "web-api-constructor", + "web-api-instance-method", + "web-api-instance-property", + "web-api-instance-event", + "web-api-static-method", + "web-api-static-property", + "web-api-event", + "webgl-extension", + "webgl-extension-method" + ], + "Web/CSS/": [ + "guide", + "landing-page", + "css-at-rule", + "css-at-rule-descriptor", + "css-combinator", + "css-function", + "css-keyword", + "css-media-feature", + "css-module", + "css-property", + "css-pseudo-class", + "css-pseudo-element", + "css-selector", + "css-shorthand-property", + "css-type" + ], + "Web/JavaScript/": [ + "guide", + "landing-page", + "javascript-class", + "javascript-constructor", + "javascript-error", + "javascript-function", + "javascript-global-property", + "javascript-instance-accessor-property", + "javascript-instance-data-property", + "javascript-instance-method", + "javascript-language-feature", + "javascript-namespace", + "javascript-operator", + "javascript-statement", + "javascript-static-accessor-property", + "javascript-static-data-property", + "javascript-static-method" + ], + "Web/HTTP/Headers/Content-Security-Policy/": ["http-csp-directive"], + "Web/HTTP/Headers/Permissions-Policy/": [ + "http-permissions-policy-directive" + ], + "Web/HTTP/Headers/": ["guide", "http-header"], + "Web/HTTP/Methods/": ["http-method"], + "Web/HTTP/Status/": ["http-status-code"], + "Web/MathML/Element/": ["mathml-element"], + "Web/MathML/Global_attributes/": ["mathml-attribute"], + "Web/Accessibility/ARIA/Attributes/": ["aria-attribute"], + "Web/Accessibility/ARIA/Roles/": ["aria-role"], + "Web/HTTP/CORS/Errors/": ["http-cors-error"], + "global": ["guide", "landing-page"] + } +} diff --git a/package.json b/package.json index ed242a5a31a7d5c..968041748194ac4 100644 --- a/package.json +++ b/package.json @@ -11,10 +11,12 @@ "build": "env-cmd --silent cross-env CONTENT_ROOT=files BUILD_OUT_ROOT=build yari-build", "content": "env-cmd --silent cross-env CONTENT_ROOT=files yari-tool", "filecheck": "env-cmd --silent cross-env CONTENT_ROOT=files yari-filecheck --cwd=.", + "fix:fm": "node scripts/front-matter-linter.js --fix true", "fix:js": "prettier -w \"**/*.(m)?js\"", "fix:json": "prettier -w \"**/*.json(c)?\"", "fix:md": "markdownlint-cli2-fix \"**/*.md\"", "fix:yml": "prettier -w \"**/*.yml\"", + "lint:fm": "node scripts/front-matter-linter.js", "lint:js": "prettier -c \"**/*.(m)?js\"", "lint:json": "prettier -c \"**/*.json(c)?\"", "lint:md": "markdownlint-cli2 \"**/*.md\"", @@ -24,6 +26,9 @@ "up-to-date-check": "node scripts/up-to-date-check.js" }, "dependencies": { + "@apideck/better-ajv-errors": "^0.3.6", + "ajv": "^8.12.0", + "ajv-formats": "^2.1.1", "@mdn/yari": "2.18.1", "cross-env": "7.0.3", "env-cmd": "10.1.0", diff --git a/scripts/front-matter-linter.js b/scripts/front-matter-linter.js new file mode 100644 index 000000000000000..ecece0459e593b3 --- /dev/null +++ b/scripts/front-matter-linter.js @@ -0,0 +1,238 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; + +import YAML from "js-yaml"; +import AJV from "ajv"; +import addFormats from "ajv-formats"; +import { betterAjvErrors } from "@apideck/better-ajv-errors"; + +import { eachLimit } from "async"; +import cliProgress from "cli-progress"; + +import fdir_pkg from "fdir"; +const { fdir, PathsOutput } = fdir_pkg; + +import caporal from "@caporal/core"; +const { program } = caporal; + +const FM_RX = /(?<=^---\n)title[\s\S]+?(?=\n---$)/gm; + +function getRelativePath(filePath) { + return path.relative(process.cwd(), filePath); +} + +function areAttributesInOrder(frontMatter, ORDER) { + let prevIndex = -1; + let inOrder = true; + for (const attribute of Object.keys(frontMatter)) { + const index = ORDER.indexOf(attribute); + if (index === -1) { + continue; + } + if (index <= prevIndex) { + inOrder = false; + break; + } + prevIndex = index; + } + return inOrder; +} + +async function checkFrontMatter(filePath, options) { + let content = await fs.readFile(filePath, "utf-8"); + const frontMatter = content.match(FM_RX)[0]; + const fmObject = YAML.load(frontMatter); + + // find a validator for the file path + let validator = null; + for (const type of Object.keys(options.validators)) { + const rx = new RegExp(`^${type}`, "g"); + if (rx.test(fmObject.slug)) { + validator = options.validators[type]; + break; + } + } + if (validator === null) { + validator = options.validators["global"]; + } + + // validate and collect errors + const valid = validator(fmObject); + const validationErrors = betterAjvErrors({ + schema: validator.schema, + data: fmObject, + errors: validator.errors, + }); + const errors = []; + if (!valid) { + for (const error of validationErrors) { + let message = error.message.replace("{base}", "Front matter"); + if (error.context.allowedValues) { + message += `: \n\t${error.context.allowedValues.join(", ")}`; + } + errors.push(message); + } + } + + const isInOrder = areAttributesInOrder( + fmObject, + options.config["attribute-order"] + ); + let fixableError = null; + + if (!options.fix && !isInOrder) { + fixableError = `${getRelativePath( + filePath + )}\n\t Front matter attributes are not in required order: ${ORDER.join( + "->" + )}`; + } + + // if --fix option is true, fix the order and prettify + if (options.fix) { + for (const attr of options.config["attribute-order"]) { + const value = fmObject[attr]; + if (value) { + if (attr === "status" && Array.isArray(value) && value.length) { + fmObject[attr] = value; + } else if (attr === "browser-compat" || attr === "spec-urls") { + if (Array.isArray(value) && value.length === 1) { + fmObject[attr] = value[0]; + } else { + fmObject[attr] = value; + } + } else { + fmObject[attr] = value; + } + } + } + + let yml = YAML.dump(fmObject, { + skipInvalid: true, + lineWidth: options.config.lineWidth, + quotingType: '"', + }); + yml = yml.replace(/[\s\n]+$/g, ""); + // handle regex content like `$&`, `$1` etc. + yml = yml.replaceAll("$", "$$$"); + content = content.replace(frontMatter, yml); + + fs.writeFile(filePath, content); + } + + return [ + errors.length + ? `Error: ${getRelativePath(filePath)}\n${errors.join("\n")}` + : null, + fixableError, + ]; +} + +async function resolveDirectory(file) { + const stats = await fs.lstat(file); + if (stats.isDirectory()) { + const api = new fdir() + .withErrors() + .withFullPaths() + .filter((filePath) => filePath.endsWith("index.md")) + .crawl(file); + return api.withPromise(); + } else if (stats.isFile() && file.endsWith("index.md")) { + return [file]; + } else { + return []; + } +} + +// create ajv validators for each document type +function compileValidators(config) { + const ajv = new AJV({ allowUnionTypes: true, allErrors: true }); + addFormats.default(ajv); + const validators = {}; + + for (const type of Object.keys(config.allowedPageTypes)) { + const copy = JSON.parse(JSON.stringify(config.schema)); + + copy.properties["page-type"].enum = config.allowedPageTypes[type]; + validators[type] = ajv.compile(copy); + } + return validators; +} + +// lint front matter +async function lintFrontMatter(filesAndDirectories, options) { + const files = ( + await Promise.all(filesAndDirectories.map(resolveDirectory)) + ).flat(); + + options.config = JSON.parse( + await fs.readFile("./front-matter-config.json", "utf-8") + ); + options.validators = compileValidators(options.config); + + const progressBar = new cliProgress.SingleBar({ etaBuffer: 100 }); + progressBar.start(files.length, 0); + + const errors = []; + const fixableErrors = []; + await eachLimit(files, os.cpus().length, async (file) => { + try { + const [error, fixableError] = await checkFrontMatter(file, options); + error && errors.push(error); + fixableError && fixableErrors.push(fixableError); + } catch (err) { + errors.push(err); + } finally { + progressBar.increment(); + } + }); + progressBar.stop(); + console.log(errors.length, fixableErrors.length); + if (errors.length || fixableErrors.length) { + let msg = errors.map((error) => `${error}`).join("\n\n"); + + if (fixableErrors.length) { + msg += + "\n\nFollowing fixable errors can be fixed using '--fix true' option\n"; + msg += fixableErrors.map((error) => `${error}`).join("\n"); + } + throw new Error(msg); + } +} + +function tryOrExit(f) { + return async ({ options = {}, ...args }) => { + try { + await f({ options, ...args }); + } catch (error) { + if (options.verbose || options.v) { + console.error(chalk.red(error.stack)); + } + throw error; + } + }; +} + +program + .option("--fix", "Save corrected output", { + validator: program.BOOLEAN, + default: false, + }) + .argument("[files...]", "list of files and/or directories to check", { + default: ["./files/en-us"], + }) + .action( + tryOrExit(({ args, options, logger }) => { + const cwd = process.cwd(); + const files = (args.files || []).map((f) => path.resolve(cwd, f)); + if (!files.length) { + logger.info("No files to lint."); + return; + } + return lintFrontMatter(files, options); + }) + ); + +program.run(); diff --git a/yarn.lock b/yarn.lock index 7f3ab250c5885ed..c032d8e44d1077d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,15 @@ # yarn lockfile v1 +"@apideck/better-ajv-errors@^0.3.6": + version "0.3.6" + resolved "https://registry.yarnpkg.com/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz#957d4c28e886a64a8141f7522783be65733ff097" + integrity sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA== + dependencies: + json-schema "^0.4.0" + jsonpointer "^5.0.0" + leven "^3.1.0" + "@caporal/core@^2.0.2": version "2.0.2" resolved "https://registry.yarnpkg.com/@caporal/core/-/core-2.0.2.tgz#b7dd808cc58caa45786cf4b5b1603b37bf77ac72" @@ -322,6 +331,13 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" +ajv-formats@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ajv-formats/-/ajv-formats-2.1.1.tgz#6e669400659eb74973bbf2e33327180a0996b520" + integrity sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA== + dependencies: + ajv "^8.0.0" + ajv@^6.10.2: version "6.12.5" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.5.tgz#19b0e8bae8f476e5ba666300387775fb1a00a4da" @@ -332,6 +348,16 @@ ajv@^6.10.2: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ajv@^8.0.0, ajv@^8.12.0: + version "8.12.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" + integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + ansi-escapes@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b" @@ -2928,6 +2954,16 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== +json-schema-traverse@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" + integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== + +json-schema@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" + integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== + jsonfile@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.0.1.tgz#98966cba214378c8c84b82e085907b40bf614179" @@ -2937,6 +2973,11 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" +jsonpointer@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559" + integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ== + junk@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/junk/-/junk-3.1.0.tgz#31499098d902b7e98c5d9b9c80f43457a88abfa1" @@ -2968,6 +3009,11 @@ kuler@1.0.x: dependencies: colornames "^1.1.1" +leven@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" + integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A== + lilconfig@2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.1.0.tgz#78e23ac89ebb7e1bfbf25b18043de756548e7f52" @@ -4584,6 +4630,11 @@ replace-ext@^2.0.0: resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-2.0.0.tgz#9471c213d22e1bcc26717cd6e50881d88f812b06" integrity sha512-UszKE5KVK6JvyD92nzMn9cDapSk6w/CaFZ96CnmDMUqH9oowfxF/ZjRITD25H4DnOQClLA4/j7jLGXXLVKxAug== +require-from-string@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" + integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== + requires-port@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"