Add a file provider that avoids ts program walk

package.json

@@ -17,7 +17,9 @@
     "dependencies": {
         "cli-progress": "^3.9.0",
         "commander": "^7.2.0",
+        "fdir": "^5.1.0",
         "minimatch": "^3.0.4",
+        "tsconfig-paths": "^3.10.1",
         "typescript": "^4.0.3"
     },
     "devDependencies": {

src/core/FdirSourceFileProvider.ts

@@ -0,0 +1,148 @@
+import { SourceFileProvider } from './SourceFileProvider';
+import { fdir } from 'fdir';
+import NormalizedPath from '../types/NormalizedPath';
+import * as ts from 'typescript';
+import { promisify } from 'util';
+import * as fs from 'fs';
+import * as path from 'path';
+const readFile = promisify(fs.readFile);
+const stat = promisify(fs.stat);
+import { createMatchPathAsync, MatchPathAsync } from 'tsconfig-paths';
+
+/**
+ * Extensions to check for when resolving with tsconfig-paths or from relative requires
+ *
+ * TODO: Should this be settable in options / from the CLI when using FdirSourceFileProvider?
+ * Or possibly parsed out of the tsconfig.json?
+ */
+const ALLOWED_EXTENSIONS = ['.ts', '.tsx', '.js', '.jsx', '', '.json'];
+
+export class FDirSourceFileProvider implements SourceFileProvider {
+    parsedCommandLine: ts.ParsedCommandLine;
+    matchPath: MatchPathAsync;
+
+    constructor(configFileName: NormalizedPath, private rootDirs: string[]) {
+        // Load the full config file, relying on typescript to recursively walk the "extends" fields,
+        // while stubbing readDirectory calls to stop the full file walk of the include() patterns.
+        //
+        // We do this because we need to access the parsed compilerOptions, but do not care about
+        // the full file list.
+        this.parsedCommandLine = ts.getParsedCommandLineOfConfigFile(
+            configFileName,
+            {}, // optionsToExtend
+            {
+                getCurrentDirectory: process.cwd,
+                fileExists: fs.existsSync,
+                useCaseSensitiveFileNames: true,
+                readFile: path => fs.readFileSync(path, 'utf-8'),
+                readDirectory: () => {
+                    // this is supposed to be the recursive file walk.
+                    // since we don't care about _actually_ discovering files,
+                    // only about parsing the config's compilerOptions
+                    // (and tracking the "extends": fields across multiple files)
+                    // we short circuit this.
+                    return [];
+                },
+                onUnRecoverableConfigFileDiagnostic: diagnostic => {
+                    console.error(diagnostic);
+                    process.exit(1);
+                },
+            }
+        );
+
+        this.matchPath = createMatchPathAsync(
+            this.parsedCommandLine.options.baseUrl,
+            this.parsedCommandLine.options.paths
+        );
+    }
+
+    async getSourceFiles(searchRoots?: string[]): Promise<string[]> {
+        const allRootsDiscoveredFiles: string[][] = await Promise.all(
+            (searchRoots || this.rootDirs).map(
+                (rootDir: string) =>
+                    new fdir()
+                        .glob(
+                            this.parsedCommandLine.options.allowJs
+                                ? `**/!(.d)*@(.js|.ts${
+                                      this.parsedCommandLine.options.jsx ? '|.jsx|.tsx' : ''
+                                  })`
+                                : `**/*!(.d)*.ts${
+                                      this.parsedCommandLine.options.jsx ? '|.tsx' : ''
+                                  }`
+                        )
+                        .withFullPaths()
+                        .crawl(rootDir)
+                        .withPromise() as Promise<string[]>
+            )
+        );
+
+        return [...new Set<string>(allRootsDiscoveredFiles.reduce((a, b) => a.concat(b), []))];
+    }
+
+    async getImportsForFile(filePath: string): Promise<string[]> {
+        const fileInfo = ts.preProcessFile(await readFile(filePath, 'utf-8'), true, true);
+        return fileInfo.importedFiles.map(importedFile => importedFile.fileName);
+    }
+
+    async resolveImportFromFile(
+        importer: string,
+        importSpecifier: string
+    ): Promise<string | undefined> {
+        if (importSpecifier.startsWith('.')) {
+            // resolve relative and check extensions
+            return await checkExtensions(
+                path.join(path.dirname(importer), importSpecifier),
+                ALLOWED_EXTENSIONS
+            );
+        } else {
+            // resolve with tsconfig-paths (use the paths map, then fall back to node-modules)
+            return await new Promise((res, rej) =>
+                this.matchPath(
+                    importSpecifier,
+                    undefined, // readJson
+                    undefined, // fileExists
+                    ALLOWED_EXTENSIONS,
+                    async (err: Error, result: string) => {
+                        if (err) {
+                            rej(err);
+                        } else if (!result) {
+                            res(undefined);
+                        } else {
+                            // tsconfig-paths returns a path without an extension, and if it resolved to
+                            // an index file, it returns the path to the directory of the index file.
+                            const withoutIndex = await checkExtensions(result, ALLOWED_EXTENSIONS);
+                            if (withoutIndex) {
+                                res(withoutIndex);
+                            } else {
+                                // fallback -- check if tsconfig-paths resolved to a
+                                // folder index file
+                                res(
+                                    checkExtensions(path.join(result, 'index'), ALLOWED_EXTENSIONS)
+                                );
+                            }
+                        }
+                    }
+                )
+            );
+        }
+    }
+}
+
+async function checkExtensions(
+    filePathNoExt: string,
+    extensions: string[]
+): Promise<string | undefined> {
+    for (let ext of extensions) {
+        const joinedPath = filePathNoExt + ext;
+        try {
+            // stat will throw if the file does no~t exist
+            const statRes = await stat(joinedPath);
+            if (statRes.isFile()) {
+                return joinedPath;
+            }
+        } catch {
+            // file does not exist, smother the ENOENT
+        }
+    }
+    return undefined;
+}

src/core/cli.ts

@@ -11,6 +11,14 @@ async function main() {
         .version(packageVersion)
         .option('-p, --project <string> ', 'tsconfig.json file')
         .option('-r, --rootDir <string...>', 'root directories of the project')
+        .option(
+            '-x, --looseRootFileDiscovery',
+            '(UNSTABLE) Check source files under rootDirs instead of instantiating a full typescript program.'
+        )
+        .option(
+            '-i, --ignoreExternalFences',
+            'Whether to ignore external fences (e.g. those from node_modules)'
+        )
         .option(
             '-j, --maxConcurrentFenceJobs',
             'Maximum number of concurrent fence jobs to run. Default 6000'

src/core/runner.ts

@@ -6,6 +6,7 @@ import normalizePath from '../utils/normalizePath';
 import { getResult } from './result';
 import { validateTagsExist } from '../validation/validateTagsExist';
 import { SourceFileProvider } from './SourceFileProvider';
+import { FDirSourceFileProvider } from './FdirSourceFileProvider';
 import NormalizedPath from '../types/NormalizedPath';
 import { runWithConcurrentLimit } from '../utils/runWithConcurrentLimit';
 
@@ -23,7 +24,9 @@ export async function run(rawOptions: RawOptions) {
     setOptions(rawOptions);
     let options = getOptions();
 
-    let sourceFileProvider: SourceFileProvider = new TypeScriptProgram(options.project);
+    let sourceFileProvider: SourceFileProvider = options.looseRootFileDiscovery
+        ? new FDirSourceFileProvider(options.project, options.rootDir)
+        : new TypeScriptProgram(options.project);
 
     // Do some sanity checks on the fences
     validateTagsExist();

src/types/Options.ts

@@ -4,7 +4,7 @@ export default interface Options {
     project: NormalizedPath;
     rootDir: NormalizedPath[];
     ignoreExternalFences: boolean;
-
+    looseRootFileDiscovery: boolean;
     // Maximum number of fence validation jobs that can
     // be run at the same time.
     //

src/types/RawOptions.ts

@@ -2,6 +2,7 @@ export default interface RawOptions {
     project?: string;
     rootDir?: string | string[];
     ignoreExternalFences?: boolean;
+    looseRootFileDiscovery?: boolean;
     maxConcurrentJobs?: number;
-    progressBar?: boolean;
+    progress?: boolean;
 }

src/utils/getOptions.ts

@@ -27,7 +27,8 @@ export function setOptions(rawOptions: RawOptions) {
         project,
         rootDir,
         ignoreExternalFences: rawOptions.ignoreExternalFences,
+        looseRootFileDiscovery: rawOptions.looseRootFileDiscovery || false,
         maxConcurrentFenceJobs: rawOptions.maxConcurrentJobs || 6000,
-        progress: rawOptions.progressBar || false,
+        progress: rawOptions.progress || false,
     };
 }

yarn.lock

@@ -1566,6 +1566,11 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "^2.0.0"
 
+fdir@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/fdir/-/fdir-5.1.0.tgz#973e4934e6a3666b59ebdfc56f60bb8e9b16acb8"
+  integrity sha512-IgTtZwL52tx2wqWeuGDzXYTnNsEjNLahZpJw30hCQDyVnoHXwY5acNDnjGImTTL1R0z1PCyLw20VAbE5qLic3Q==
+
 fill-range@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7"
@@ -2574,6 +2579,13 @@ json5@^2.1.2:
   dependencies:
     minimist "^1.2.5"
 
+json5@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3"
+  integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA==
+  dependencies:
+    minimist "^1.2.5"
+
 jsprim@^1.2.2:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@@ -3601,6 +3613,11 @@ strip-ansi@^6.0.0:
   dependencies:
     ansi-regex "^5.0.0"
 
+strip-bom@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3"
+  integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=
+
 strip-bom@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878"
@@ -3736,6 +3753,15 @@ trim-right@^1.0.1:
   resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003"
   integrity sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=
 
+tsconfig-paths@^3.10.1:
+  version "3.10.1"
+  resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.10.1.tgz#79ae67a68c15289fdf5c51cb74f397522d795ed7"
+  integrity sha512-rETidPDgCpltxF7MjBZlAFPUHv5aHH2MymyPvh+vEyWAED4Eb/WeMbsnD/JDr4OKPOA1TssDHgIcpTN5Kh0p6Q==
+  dependencies:
+    json5 "^2.2.0"
+    minimist "^1.2.0"
+    strip-bom "^3.0.0"
+
 tunnel-agent@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"