From a0db7dd0135cf63832ae6fe31e29c54fcf19ff85 Mon Sep 17 00:00:00 2001 From: OnkarRuikar <87750369+OnkarRuikar@users.noreply.github.com> Date: Thu, 23 Feb 2023 20:39:19 +0530 Subject: [PATCH 1/3] feat(lint): Implement front matter validator and linter --- fmlint/cli.ts | 42 ++++++ fmlint/front-matter-config.json | 127 ++++++++++++++++++ fmlint/linter.ts | 226 ++++++++++++++++++++++++++++++++ fmlint/types.ts | 31 +++++ package.json | 4 + tsconfig.dist.json | 10 +- yarn.lock | 12 +- 7 files changed, 450 insertions(+), 2 deletions(-) create mode 100644 fmlint/cli.ts create mode 100644 fmlint/front-matter-config.json create mode 100644 fmlint/linter.ts create mode 100644 fmlint/types.ts diff --git a/fmlint/cli.ts b/fmlint/cli.ts new file mode 100644 index 000000000000..a1be10ed9b70 --- /dev/null +++ b/fmlint/cli.ts @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +import path from "node:path"; +import { CliArgsAndOptions } from "./types.js"; +import caporal from "@caporal/core"; +import { runLinter } from "./linter.js"; +import { fileURLToPath } from "node:url"; +import { CONTENT_ROOT, CONTENT_TRANSLATED_ROOT } from "../libs/env/index.js"; + +const { program } = caporal; +const appDirectory = new URL("fmlint", import.meta.url); +const resolveApp = (relativePath) => + fileURLToPath(new URL(relativePath, appDirectory)); + +program + .version("0.0.0") + .option("--cwd ", "Explicit current-working-directory", { + validator: program.STRING, + default: process.cwd(), + }) + .option("--config ", "Front matter config file location", { + validator: program.STRING, + default: resolveApp("front-matter-config.json"), + }) + .option("--fix", "Save formatted/corrected output", { + validator: program.BOOLEAN, + default: false, + }) + .argument("[files...]", "list of files and/or directories to check", { + default: [CONTENT_ROOT, CONTENT_TRANSLATED_ROOT].filter(Boolean), + }) + .action(({ args, options, logger }: CliArgsAndOptions) => { + const cwd = options.cwd || process.cwd(); + const files = (args.files || []).map((f) => path.resolve(cwd, f)); + if (!files.length) { + logger.info("No files to lint."); + return; + } + return runLinter(files, options); + }); + +program.run(); diff --git a/fmlint/front-matter-config.json b/fmlint/front-matter-config.json new file mode 100644 index 000000000000..d3bf1cb2de78 --- /dev/null +++ b/fmlint/front-matter-config.json @@ -0,0 +1,127 @@ +{ + "lineWidth": 80, + "schema": { + "title": "Front matter schema", + "type": "object", + "additionalProperties": false, + "required": ["title", "slug"], + "properties": { + "title": { + "title": "Title", + "description": "Rendered page title and for SEO", + "type": "string", + "maxLength": 150 + }, + "slug": { + "title": "slug", + "description": "URL path of the page", + "type": "string" + }, + "page-type": { + "title": "Page Type", + "type": "string" + }, + "status": { + "title": "Status", + "description": "Browser compatibility status", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": ["experimental", "non-standard", "deprecated"] + } + }, + "browser-compat": { + "title": "Browser Compatibility", + "description": "Browser compatibility location", + "type": ["array", "string"] + }, + "spec-urls": { + "title": "Specification URLs", + "description": "Specification locations", + "type": ["array", "string"], + "format": "uri", + "items": { + "type": "string", + "format": "uri" + } + } + } + }, + "allowedPageTypes": { + "Glossary": ["glossary-definition", "glossary-disambiguation"], + "MDN/": ["landing-page", "mdn-community-guide", "mdn-writing-guide"], + "Web/SVG/": ["guide", "landing-page", "svg-attribute", "svg-element"], + "Web/HTML/": [ + "guide", + "landing-page", + "html-attribute", + "html-attribute-value", + "html-element" + ], + "Mozilla/Add-ons/WebExtensions/": [ + "guide", + "landing-page", + "webextension-api-function", + "webextension-api", + "webextension-api-event", + "webextension-api-property", + "webextension-api-type", + "webextension-manifest-key" + ], + "Web/API/": [ + "guide", + "landing-page", + "web-api-overview", + "web-api-global-function", + "web-api-global-property", + "web-api-interface", + "web-api-constructor", + "web-api-instance-method", + "web-api-instance-property", + "web-api-instance-event", + "web-api-static-method", + "web-api-static-property", + "web-api-event", + "webgl-extension", + "webgl-extension-method" + ], + "Web/CSS/": [ + "guide", + "landing-page", + "css-at-rule", + "css-at-rule-descriptor", + "css-combinator", + "css-function", + "css-keyword", + "css-media-feature", + "css-module", + "css-property", + "css-pseudo-class", + "css-pseudo-element", + "css-selector", + "css-shorthand-property", + "css-type" + ], + "Web/JavaScript/": [ + "guide", + "landing-page", + "javascript-class", + "javascript-constructor", + "javascript-error", + "javascript-function", + "javascript-global-property", + "javascript-instance-accessor-property", + "javascript-instance-data-property", + "javascript-instance-method", + "javascript-language-feature", + "javascript-namespace", + "javascript-operator", + "javascript-statement", + "javascript-static-accessor-property", + "javascript-static-data-property", + "javascript-static-method" + ], + "global": ["guide", "landing-page"] + } +} diff --git a/fmlint/linter.ts b/fmlint/linter.ts new file mode 100644 index 000000000000..f6ae9a966165 --- /dev/null +++ b/fmlint/linter.ts @@ -0,0 +1,226 @@ +import { LinterOptions, FMConfig, ValidationError } from "./types.js"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; + +import YAML from "js-yaml"; +import _Ajv from "ajv"; + +import addFormats from "ajv-formats"; +import { betterAjvErrors } from "@apideck/better-ajv-errors"; + +import { eachLimit } from "async"; +import cliProgress from "cli-progress"; +import { fdir, PathsOutput } from "fdir"; + +const ORDER = [ + "title", + "slug", + "page-type", + "status", + "browser-compat", + "spec-urls", +]; +const FM_RX = /(?<=^---\n)title[\s\S]+?(?=\n---$)/gm; + +function getRelativePath(filePath: string): string { + return path.relative(process.cwd(), filePath); +} + +function areAttributesInOrder(fm: object): boolean { + let prevIndex = -1; + let inOrder = true; + for (const attribute of Object.keys(fm)) { + const index = ORDER.indexOf(attribute); + if (index === -1) { + continue; + } + if (index <= prevIndex) { + inOrder = false; + break; + } + prevIndex = index; + } + return inOrder; +} + +export async function checkFrontMatter( + filePath: string, + options: LinterOptions +) { + let content = await fs.readFile(filePath, "utf-8"); + const frontMatter = content.match(FM_RX)[0]; + let fmObject = YAML.load(frontMatter); + + // find a validator for the file path + let validator = null; + for (const type of Object.keys(options.validators)) { + const rx = new RegExp(`^${type}`, "g"); + if (rx.test(fmObject.slug)) { + validator = options.validators[type]; + break; + } + } + if (validator === null) { + validator = options.validators["global"]; + } + + // validate and collect errors + const valid = validator(fmObject); + const validationErrors: ValidationError[] = betterAjvErrors({ + schema: validator.schema, + data: fmObject, + errors: validator.errors, + }); + const errors = []; + if (!valid) { + for (const error of validationErrors) { + let message = error.message.replace("{base}", "Front matter"); + if (error.context.allowedValues) { + message += `: \n\t${error.context.allowedValues.join(", ")}`; + } + errors.push(message); + } + } + + const inOrder = areAttributesInOrder(fmObject); + let fixableError = null; + if (!options.fix && !inOrder) { + fixableError = `${getRelativePath( + filePath + )}\n\t Front matter attributes are not in required order: ${ORDER.join( + "->" + )}`; + } + + // if --fix option is true, fix order and prettify + if (options.fix) { + const { + title, + slug, + "page-type": pageType, + status, + "spec-urls": specs, + "browser-compat": bcd, + } = fmObject; + + fmObject = { title, slug }; + + if (pageType) { + fmObject["page-type"] = pageType; + } + + if (status && status?.length) { + fmObject["status"] = status; + } + + if (bcd && bcd?.length) { + if (Array.isArray(bcd) && bcd.length === 1) { + fmObject["browser-compat"] = bcd[0]; + } else { + fmObject["browser-compat"] = bcd; + } + } + + if (specs && specs?.length) { + if (Array.isArray(specs) && specs.length === 1) { + fmObject["spec-urls"] = specs[0]; + } else { + fmObject["spec-urls"] = specs; + } + } + + let yml = YAML.dump(fmObject, { + skipInvalid: true, + lineWidth: options.config.lineWidth, + quotingType: '"', + }); + yml = yml.replace(/[\s\n]+$/g, ""); + content = content.replace(frontMatter, yml); + + fs.writeFile(filePath, content); + } + + return [ + errors.length + ? `Error: ${getRelativePath(filePath)}\n${errors.join("\n")}` + : null, + fixableError, + ]; +} + +async function resolveDirectory(file: string): Promise { + const stats = await fs.lstat(file); + if (stats.isDirectory()) { + const api = new fdir() + .withErrors() + .withFullPaths() + .filter((filePath) => filePath.endsWith("index.md")) + .crawl(file); + return api.withPromise() as Promise; + } else if (stats.isFile() && file.endsWith("index.md")) { + return [file]; + } else { + return []; + } +} + +// create ajv validators for each document type +function compileValidators(config: FMConfig) { + const AJV = _Ajv as unknown as typeof _Ajv.default; + const ajv = new AJV({ allowUnionTypes: true, allErrors: true }); + addFormats.default(ajv); + const validators = {}; + + for (const type of Object.keys(config.allowedPageTypes)) { + const copy = JSON.parse(JSON.stringify(config.schema)); + + copy.properties["page-type"].enum = config.allowedPageTypes[type]; + validators[type] = ajv.compile(copy); + } + return validators; +} + +// lint front matter +export async function runLinter( + filesAndDirectories: string[], + options: LinterOptions +) { + const files = ( + await Promise.all(filesAndDirectories.map(resolveDirectory)) + ).flat(); + + options.config = JSON.parse( + await fs.readFile(options.config as any, "utf-8") + ); + options.validators = compileValidators(options.config); + + const progressBar = new cliProgress.SingleBar({ etaBuffer: 100 }); + progressBar.start(files.length, 0); + + const errors = []; + const fixableErrors = []; + await eachLimit(files, os.cpus().length, async (file) => { + try { + const [error, fixableError] = await checkFrontMatter(file, options); + error && errors.push(error); + fixableError && fixableErrors.push(fixableError); + } catch (err) { + errors.push(err); + } finally { + progressBar.increment(); + } + }); + progressBar.stop(); + console.log(errors.length, fixableErrors.length); + if (errors.length || fixableErrors.length) { + let msg = errors.map((error) => `${error}`).join("\n\n"); + + if (fixableErrors.length) { + msg += + "\n\nFollowing fixable errors can be fixed using '--fix true' option\n"; + msg += fixableErrors.map((error) => `${error}`).join("\n"); + } + throw new Error(msg); + } +} diff --git a/fmlint/types.ts b/fmlint/types.ts new file mode 100644 index 000000000000..76be2aaf5c92 --- /dev/null +++ b/fmlint/types.ts @@ -0,0 +1,31 @@ +import { ActionParameters } from "@caporal/core"; + +export interface CliArgsAndOptions extends ActionParameters { + args: { + files?: string[]; + }; + options: { + cwd?: string; + fix?: boolean; + schemaPath?: string; + }; +} + +export interface LinterOptions { + cwd?: string; + fix?: boolean; + schemaPath?: string; + config?: FMConfig; + validators?: any; +} + +export interface FMConfig { + lineWidth: number; + schema: object; + allowedPageTypes: object; +} + +export interface ValidationError { + message: string; + context: any; +} diff --git a/package.json b/package.json index e3ce8a781c62..e5eb17881ec2 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dev": "yarn build:prepare && nf -j Procfile.dev start", "eslint": "eslint .", "filecheck": "ts-node filecheck/cli.ts", + "fmlint": "ts-node fmlint/cli.ts", "install:all": "find . -mindepth 2 -name 'yarn.lock' ! -wholename '**/node_modules/**' -print0 | xargs -n1 -0 sh -cx 'yarn --cwd $(dirname $0) install'", "jest": "cross-env NODE_OPTIONS=--experimental-vm-modules jest", "m2h": "ts-node markdown/m2h/cli.ts", @@ -55,6 +56,7 @@ "lodash": ">=4.17.15" }, "dependencies": { + "@apideck/better-ajv-errors": "^0.3.6", "@caporal/core": "^2.0.2", "@fast-csv/parse": "^4.3.6", "@mdn/bcd-utils-api": "^0.0.4", @@ -63,6 +65,8 @@ "@use-it/interval": "^1.0.0", "@webref/css": "^5.4.4", "accept-language-parser": "^1.5.0", + "ajv": "^8.12.0", + "ajv-formats": "^2.1.1", "async": "^3.2.4", "chalk": "^5.2.0", "cheerio": "^1.0.0-rc.12", diff --git a/tsconfig.dist.json b/tsconfig.dist.json index 425acf7505c3..b2868937fe25 100644 --- a/tsconfig.dist.json +++ b/tsconfig.dist.json @@ -3,5 +3,13 @@ "compilerOptions": { "noEmit": false }, - "include": ["build", "content", "filecheck", "markdown", "server", "tool"] + "include": [ + "build", + "content", + "filecheck", + "markdown", + "server", + "tool", + "fmlint" + ] } diff --git a/yarn.lock b/yarn.lock index 61493678a83d..87368dd65cd4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,7 +10,7 @@ "@jridgewell/gen-mapping" "^0.1.0" "@jridgewell/trace-mapping" "^0.3.9" -"@apideck/better-ajv-errors@^0.3.1": +"@apideck/better-ajv-errors@^0.3.1", "@apideck/better-ajv-errors@^0.3.6": version "0.3.6" resolved "https://registry.yarnpkg.com/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.6.tgz#957d4c28e886a64a8141f7522783be65733ff097" integrity sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA== @@ -3058,6 +3058,16 @@ ajv@^8.0.0, ajv@^8.0.1, ajv@^8.6.0, ajv@^8.8.0: require-from-string "^2.0.2" uri-js "^4.2.2" +ajv@^8.12.0: + version "8.12.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" + integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + ansi-escapes@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b" From 595e07f3d906331373eaa588210d84664716ded3 Mon Sep 17 00:00:00 2001 From: OnkarRuikar <87750369+OnkarRuikar@users.noreply.github.com> Date: Thu, 23 Feb 2023 21:52:43 +0530 Subject: [PATCH 2/3] add linter as a command to tool cli --- fmlint/cli.ts | 42 ------------------------------------------ fmlint/linter.ts | 2 +- fmlint/types.ts | 2 +- tool/cli.ts | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 44 deletions(-) delete mode 100644 fmlint/cli.ts diff --git a/fmlint/cli.ts b/fmlint/cli.ts deleted file mode 100644 index a1be10ed9b70..000000000000 --- a/fmlint/cli.ts +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env node - -import path from "node:path"; -import { CliArgsAndOptions } from "./types.js"; -import caporal from "@caporal/core"; -import { runLinter } from "./linter.js"; -import { fileURLToPath } from "node:url"; -import { CONTENT_ROOT, CONTENT_TRANSLATED_ROOT } from "../libs/env/index.js"; - -const { program } = caporal; -const appDirectory = new URL("fmlint", import.meta.url); -const resolveApp = (relativePath) => - fileURLToPath(new URL(relativePath, appDirectory)); - -program - .version("0.0.0") - .option("--cwd ", "Explicit current-working-directory", { - validator: program.STRING, - default: process.cwd(), - }) - .option("--config ", "Front matter config file location", { - validator: program.STRING, - default: resolveApp("front-matter-config.json"), - }) - .option("--fix", "Save formatted/corrected output", { - validator: program.BOOLEAN, - default: false, - }) - .argument("[files...]", "list of files and/or directories to check", { - default: [CONTENT_ROOT, CONTENT_TRANSLATED_ROOT].filter(Boolean), - }) - .action(({ args, options, logger }: CliArgsAndOptions) => { - const cwd = options.cwd || process.cwd(); - const files = (args.files || []).map((f) => path.resolve(cwd, f)); - if (!files.length) { - logger.info("No files to lint."); - return; - } - return runLinter(files, options); - }); - -program.run(); diff --git a/fmlint/linter.ts b/fmlint/linter.ts index f6ae9a966165..1acfeeb60333 100644 --- a/fmlint/linter.ts +++ b/fmlint/linter.ts @@ -182,7 +182,7 @@ function compileValidators(config: FMConfig) { } // lint front matter -export async function runLinter( +export async function lintFrontMatter( filesAndDirectories: string[], options: LinterOptions ) { diff --git a/fmlint/types.ts b/fmlint/types.ts index 76be2aaf5c92..1e845ee745b2 100644 --- a/fmlint/types.ts +++ b/fmlint/types.ts @@ -1,6 +1,6 @@ import { ActionParameters } from "@caporal/core"; -export interface CliArgsAndOptions extends ActionParameters { +export interface FMLintArgsAndOptions extends ActionParameters { args: { files?: string[]; }; diff --git a/tool/cli.ts b/tool/cli.ts index fad8ae827d1c..f3c5507e8b83 100644 --- a/tool/cli.ts +++ b/tool/cli.ts @@ -41,6 +41,8 @@ import { MacroInvocationError, MacroRedirectedLinkError, } from "../kumascript/src/errors.js"; +import { FMLintArgsAndOptions } from "../fmlint/types.js"; +import { lintFrontMatter } from "../fmlint/linter.js"; const { program } = caporal; const { prompt } = inquirer; @@ -1220,6 +1222,36 @@ if (Mozilla && !Mozilla.dntEnabled()) { const { deprecatedOnly, format, unusedOnly } = options; return macroUsageReport({ deprecatedOnly, format, unusedOnly }); }) + ) + + .command("fmlint", "Validate and prettify front matter.") + .option("--cwd ", "Explicit current-working-directory", { + validator: program.STRING, + default: process.cwd(), + }) + .option("--config ", "Front matter config file location", { + validator: program.STRING, + default: fileURLToPath( + new URL("../fmlint/front-matter-config.json", import.meta.url) + ), + }) + .option("--fix", "Save formatted/corrected output", { + validator: program.BOOLEAN, + default: false, + }) + .argument("[files...]", "list of files and/or directories to check", { + default: [CONTENT_ROOT, CONTENT_TRANSLATED_ROOT].filter(Boolean), + }) + .action( + tryOrExit(({ args, options, logger }: FMLintArgsAndOptions) => { + const cwd = options.cwd || process.cwd(); + const files = (args.files || []).map((f) => path.resolve(cwd, f)); + if (!files.length) { + logger.info("No files to lint."); + return; + } + return lintFrontMatter(files, options); + }) ); program.run(); From 0f8072b06a67fe919b95b43bc0118db5bb46beb7 Mon Sep 17 00:00:00 2001 From: OnkarRuikar <87750369+OnkarRuikar@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:05:38 +0530 Subject: [PATCH 3/3] support short-title and add remaining page types --- fmlint/front-matter-config.json | 23 +++++++++++++++++++++-- fmlint/linter.ts | 11 ++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/fmlint/front-matter-config.json b/fmlint/front-matter-config.json index d3bf1cb2de78..4413b1d441a2 100644 --- a/fmlint/front-matter-config.json +++ b/fmlint/front-matter-config.json @@ -10,7 +10,13 @@ "title": "Title", "description": "Rendered page title and for SEO", "type": "string", - "maxLength": 150 + "maxLength": 120 + }, + "short-title": { + "title": "Short Title", + "description": "To be used in sidebars", + "type": "string", + "maxLength": 60 }, "slug": { "title": "slug", @@ -19,6 +25,7 @@ }, "page-type": { "title": "Page Type", + "description": "Type of the page", "type": "string" }, "status": { @@ -28,7 +35,7 @@ "uniqueItems": true, "items": { "type": "string", - "enum": ["experimental", "non-standard", "deprecated"] + "enum": ["deprecated", "experimental", "non-standard"] } }, "browser-compat": { @@ -122,6 +129,18 @@ "javascript-static-data-property", "javascript-static-method" ], + "Web/HTTP/Headers/Content-Security-Policy/": ["http-csp-directive"], + "Web/HTTP/Headers/Permissions-Policy/": [ + "http-permissions-policy-directive" + ], + "Web/HTTP/Headers/": ["guide", "http-header"], + "Web/HTTP/Methods/": ["http-method"], + "Web/HTTP/Status/": ["http-status-code"], + "Web/MathML/Element/": ["mathml-element"], + "Web/MathML/Global_attributes/": ["mathml-attribute"], + "Web/Accessibility/ARIA/Attributes/": ["aria-attribute"], + "Web/Accessibility/ARIA/Roles/": ["aria-role"], + "Web/HTTP/CORS/Errors/": ["http-cors-error"], "global": ["guide", "landing-page"] } } diff --git a/fmlint/linter.ts b/fmlint/linter.ts index 1acfeeb60333..24abeab0cac6 100644 --- a/fmlint/linter.ts +++ b/fmlint/linter.ts @@ -15,6 +15,7 @@ import { fdir, PathsOutput } from "fdir"; const ORDER = [ "title", + "short-title", "slug", "page-type", "status", @@ -97,6 +98,7 @@ export async function checkFrontMatter( if (options.fix) { const { title, + "short-title": shortTitle, slug, "page-type": pageType, status, @@ -104,7 +106,13 @@ export async function checkFrontMatter( "browser-compat": bcd, } = fmObject; - fmObject = { title, slug }; + fmObject = { title }; + + if (shortTitle) { + fmObject["short-title"] = shortTitle; + } + + fmObject["slug"] = slug; if (pageType) { fmObject["page-type"] = pageType; @@ -136,6 +144,7 @@ export async function checkFrontMatter( quotingType: '"', }); yml = yml.replace(/[\s\n]+$/g, ""); + yml = yml.replaceAll("$", "$$$"); content = content.replace(frontMatter, yml); fs.writeFile(filePath, content);