diff --git a/package.json b/package.json index b2c8f7c..afd2995 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,10 @@ "dependencies": { "cli-progress": "^3.9.0", "commander": "^7.2.0", + "fdir": "^5.1.0", "minimatch": "^3.0.4", + "picomatch": "^2.3.0", + "tsconfig-paths": "^3.10.1", "typescript": "^4.0.3" }, "devDependencies": { diff --git a/src/core/FdirSourceFileProvider.ts b/src/core/FdirSourceFileProvider.ts new file mode 100644 index 0000000..1395b42 --- /dev/null +++ b/src/core/FdirSourceFileProvider.ts @@ -0,0 +1,224 @@ +import { SourceFileProvider } from './SourceFileProvider'; +import { fdir } from 'fdir'; +import NormalizedPath from '../types/NormalizedPath'; +import { promisify } from 'util'; +import * as fs from 'fs'; +import * as path from 'path'; +const readFile = promisify(fs.readFile); +const stat = promisify(fs.stat); +import { createMatchPathAsync, MatchPathAsync } from 'tsconfig-paths'; +import { getScriptFileExtensions } from '../utils/getScriptFileExtensions'; +import { + getParsedCommandLineOfConfigFile, + JsxEmit, + ParsedCommandLine, + preProcessFile, +} from 'typescript'; + +export class FDirSourceFileProvider implements SourceFileProvider { + parsedCommandLine: ParsedCommandLine; + matchPath: MatchPathAsync; + private sourceFileGlob: string; + private extensionsToCheckDuringImportResolution: string[]; + + constructor(configFileName: NormalizedPath, private rootDirs: string[]) { + // Load the full config file, relying on typescript to recursively walk the "extends" fields, + // while stubbing readDirectory calls to stop the full file walk of the include() patterns. + // + // We do this because we need to access the parsed compilerOptions, but do not care about + // the full file list. + this.parsedCommandLine = getParsedCommandLineOfConfigFile( + configFileName, + {}, // optionsToExtend + { + getCurrentDirectory: process.cwd, + fileExists: fs.existsSync, + useCaseSensitiveFileNames: true, + readFile: path => fs.readFileSync(path, 'utf-8'), + readDirectory: () => { + // this is supposed to be the recursive file walk. + // since we don't care about _actually_ discovering files, + // only about parsing the config's compilerOptions + // (and tracking the "extends": fields across multiple files) + // we short circuit this. + return []; + }, + onUnRecoverableConfigFileDiagnostic: diagnostic => { + console.error(diagnostic); + process.exit(1); + }, + } + ); + + this.sourceFileGlob = `**/*@(${getScriptFileExtensions({ + // Derive these settings from the typescript project itself + allowJs: this.parsedCommandLine.options.allowJs || false, + jsx: this.parsedCommandLine.options.jsx !== JsxEmit.None, + // Since we're trying to find script files that can have imports, + // we explicitly exclude json modules + includeJson: false, + // since definition files are '.d.ts', the extra + // definition extensions here are covered by the glob '*.ts' from + // the above settings. + // + // Here as an optimization we avoid adding these definition files while + // globbing + includeDefinitions: false, + }).join('|')})`; + + // Script extensions to check when looking for imports. + this.extensionsToCheckDuringImportResolution = getScriptFileExtensions({ + // Derive these settings from the typescript project itself + allowJs: this.parsedCommandLine.options.allowJs || false, + jsx: this.parsedCommandLine.options.jsx !== JsxEmit.None, + includeJson: this.parsedCommandLine.options.resolveJsonModule, + // When scanning for imports, we always consider importing + // definition files. + includeDefinitions: true, + }); + + this.matchPath = createMatchPathAsync( + this.parsedCommandLine.options.baseUrl, + this.parsedCommandLine.options.paths + ); + } + + async getSourceFiles(searchRoots?: string[]): Promise { + const allRootsDiscoveredFiles: string[][] = await Promise.all( + (searchRoots || this.rootDirs).map( + (rootDir: string) => + new fdir() + .glob(this.sourceFileGlob) + .withFullPaths() + .crawl(rootDir) + .withPromise() as Promise + ) + ); + + return [...new Set(allRootsDiscoveredFiles.reduce((a, b) => a.concat(b), []))]; + } + + async getImportsForFile(filePath: string): Promise { + const fileInfo = preProcessFile(await readFile(filePath, 'utf-8'), true, true); + return fileInfo.importedFiles.map(importedFile => importedFile.fileName); + } + + async resolveImportFromFile( + importer: string, + importSpecifier: string + ): Promise { + if (importSpecifier.startsWith('.')) { + // resolve relative and check extensions + const directImportResult = await checkExtensions( + path.join(path.dirname(importer), importSpecifier), + [ + ...this.extensionsToCheckDuringImportResolution, + // Also check for no-exension to permit import specifiers that + // already have an extension (e.g. require('foo.js')) + '', + // also check for directory index imports + ...this.extensionsToCheckDuringImportResolution.map(x => '/index' + x), + ] + ); + + if ( + directImportResult && + this.extensionsToCheckDuringImportResolution.some(extension => + directImportResult.endsWith(extension) + ) + ) { + // this is an allowed script file + return directImportResult; + } else { + // this is an asset file + return undefined; + } + } else { + // resolve with tsconfig-paths (use the paths map, then fall back to node-modules) + return await new Promise((resolve, reject) => + this.matchPath( + importSpecifier, + undefined, // readJson + undefined, // fileExists + [...this.extensionsToCheckDuringImportResolution, ''], + async (err: Error, result: string) => { + if (err) { + reject(err); + } else if (!result) { + resolve(undefined); + } else { + if ( + isFile(result) && + this.extensionsToCheckDuringImportResolution.some(extension => + result.endsWith(extension) + ) + ) { + // this is an exact require of a known script extension, resolve + // it up front + resolve(result); + } else { + // tsconfig-paths returns a path without an extension. + // if it resolved to an index file, it returns the path to + // the directory of the index file. + if (await isDirectory(result)) { + resolve( + checkExtensions( + path.join(result, 'index'), + this.extensionsToCheckDuringImportResolution + ) + ); + } else { + resolve( + checkExtensions( + result, + this.extensionsToCheckDuringImportResolution + ) + ); + } + } + } + } + ) + ); + } + } +} + +async function isFile(filePath: string): Promise { + try { + // stat will throw if the file does not exist + const statRes = await stat(filePath); + if (statRes.isFile()) { + return true; + } + } catch { + // file does not exist + return false; + } +} + +async function isDirectory(filePath: string): Promise { + try { + // stat will throw if the file does not exist + const statRes = await stat(filePath); + if (statRes.isDirectory()) { + return true; + } + } catch { + // file does not exist + return false; + } +} + +async function checkExtensions( + filePathNoExt: string, + extensions: string[] +): Promise { + for (let ext of extensions) { + const joinedPath = filePathNoExt + ext; + if (await isFile(joinedPath)) { + return joinedPath; + } + } + return undefined; +} diff --git a/src/core/cli.ts b/src/core/cli.ts index 36a689a..1672ab5 100644 --- a/src/core/cli.ts +++ b/src/core/cli.ts @@ -11,6 +11,14 @@ async function main() { .version(packageVersion) .option('-p, --project ', 'tsconfig.json file') .option('-r, --rootDir ', 'root directories of the project') + .option( + '-x, --looseRootFileDiscovery', + '(UNSTABLE) Check source files under rootDirs instead of instantiating a full typescript program.' + ) + .option( + '-i, --ignoreExternalFences', + 'Whether to ignore external fences (e.g. those from node_modules)' + ) .option( '-j, --maxConcurrentFenceJobs', 'Maximum number of concurrent fence jobs to run. Default 6000' diff --git a/src/core/runner.ts b/src/core/runner.ts index fcce108..1e33780 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -6,6 +6,7 @@ import normalizePath from '../utils/normalizePath'; import { getResult } from './result'; import { validateTagsExist } from '../validation/validateTagsExist'; import { SourceFileProvider } from './SourceFileProvider'; +import { FDirSourceFileProvider } from './FdirSourceFileProvider'; import NormalizedPath from '../types/NormalizedPath'; import { runWithConcurrentLimit } from '../utils/runWithConcurrentLimit'; @@ -23,7 +24,9 @@ export async function run(rawOptions: RawOptions) { setOptions(rawOptions); let options = getOptions(); - let sourceFileProvider: SourceFileProvider = new TypeScriptProgram(options.project); + let sourceFileProvider: SourceFileProvider = options.looseRootFileDiscovery + ? new FDirSourceFileProvider(options.project, options.rootDir) + : new TypeScriptProgram(options.project); // Do some sanity checks on the fences validateTagsExist(); diff --git a/src/types/Options.ts b/src/types/Options.ts index 395bb6c..798d6b1 100644 --- a/src/types/Options.ts +++ b/src/types/Options.ts @@ -4,7 +4,7 @@ export default interface Options { project: NormalizedPath; rootDir: NormalizedPath[]; ignoreExternalFences: boolean; - + looseRootFileDiscovery: boolean; // Maximum number of fence validation jobs that can // be run at the same time. // diff --git a/src/types/RawOptions.ts b/src/types/RawOptions.ts index c342d8b..4f806d5 100644 --- a/src/types/RawOptions.ts +++ b/src/types/RawOptions.ts @@ -2,6 +2,7 @@ export default interface RawOptions { project?: string; rootDir?: string | string[]; ignoreExternalFences?: boolean; + looseRootFileDiscovery?: boolean; maxConcurrentJobs?: number; progressBar?: boolean; } diff --git a/src/utils/getOptions.ts b/src/utils/getOptions.ts index f15d969..d74572d 100644 --- a/src/utils/getOptions.ts +++ b/src/utils/getOptions.ts @@ -27,6 +27,7 @@ export function setOptions(rawOptions: RawOptions) { project, rootDir, ignoreExternalFences: rawOptions.ignoreExternalFences, + looseRootFileDiscovery: rawOptions.looseRootFileDiscovery || false, maxConcurrentFenceJobs: rawOptions.maxConcurrentJobs || 6000, progress: rawOptions.progressBar || false, }; diff --git a/src/utils/getScriptFileExtensions.ts b/src/utils/getScriptFileExtensions.ts new file mode 100644 index 0000000..a3627d3 --- /dev/null +++ b/src/utils/getScriptFileExtensions.ts @@ -0,0 +1,35 @@ +export type PartialConfigOptions = { + allowJs: boolean; + jsx: boolean; + includeJson?: boolean; + includeDefinitions?: boolean; +}; + +export function getScriptFileExtensions(options: PartialConfigOptions): string[] { + const extensions: string[] = ['.ts']; + if (options.allowJs) { + extensions.push('.js'); + if (options.jsx) { + extensions.push('.jsx'); + } + } + + if (options.includeJson) { + extensions.push('.json'); + } + + if (options.jsx) { + extensions.push('.tsx'); + } + + if (options.includeDefinitions) { + extensions.push('.d.ts'); + if (options.jsx) { + // I don't know why this would ever + // be a thing, but it is, so I'm adding it here. + extensions.push('.d.jsx'); + } + } + + return extensions; +} diff --git a/yarn.lock b/yarn.lock index 11a7c0b..d2a3e7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1566,6 +1566,11 @@ fb-watchman@^2.0.0: dependencies: bser "^2.0.0" +fdir@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/fdir/-/fdir-5.1.0.tgz#973e4934e6a3666b59ebdfc56f60bb8e9b16acb8" + integrity sha512-IgTtZwL52tx2wqWeuGDzXYTnNsEjNLahZpJw30hCQDyVnoHXwY5acNDnjGImTTL1R0z1PCyLw20VAbE5qLic3Q== + fill-range@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" @@ -2574,6 +2579,13 @@ json5@^2.1.2: dependencies: minimist "^1.2.5" +json5@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" + integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== + dependencies: + minimist "^1.2.5" + jsprim@^1.2.2: version "1.4.1" resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" @@ -3057,6 +3069,11 @@ picomatch@^2.0.4, picomatch@^2.0.5: resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad" integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg== +picomatch@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" + integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== + pirates@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87" @@ -3601,6 +3618,11 @@ strip-ansi@^6.0.0: dependencies: ansi-regex "^5.0.0" +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM= + strip-bom@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" @@ -3736,6 +3758,15 @@ trim-right@^1.0.1: resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003" integrity sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM= +tsconfig-paths@^3.10.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz#79ae67a68c15289fdf5c51cb74f397522d795ed7" + integrity sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q== + dependencies: + json5 "^2.2.0" + minimist "^1.2.0" + strip-bom "^3.0.0" + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"