Lazy load fences (#99)

* lazy load configs

* Update getConfigManager.ts

* Address comments in #99

Co-authored-by: Maxwell Huang-Hobbs <mahuangh@microsoft.com>
Co-authored-by: Scott Mikula <mikula@gmail.com>

src/utils/getConfigManager.ts

@@ -0,0 +1,104 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import ConfigSet from '../types/ConfigSet';
+import getOptions from './getOptions';
+import loadConfig from './loadConfig';
+import { getConfigPathCandidatesForFile } from './getConfigPathCandidatesForFile';
+import NormalizedPath from '../types/NormalizedPath';
+
+class ConfigManager {
+    private fullConfigSet: ConfigSet = null;
+    // The subset of configs that has been loaded
+    private partialDiscoveredConfigs: ConfigSet = {};
+
+    // The set of paths we have checked for configs in the filesystem
+    private discoveredPaths: Set<string> = new Set();
+
+    public getAllConfigs(): ConfigSet {
+        if (this.fullConfigSet === null) {
+            this._getAllConfigs();
+        }
+        return this.fullConfigSet;
+    }
+
+    public getPartialConfigSetForPath(configSourcePath: NormalizedPath): ConfigSet {
+        const partialSet: ConfigSet = {};
+
+        const configCandidatesForFile = getConfigPathCandidatesForFile(configSourcePath);
+
+        if (this.fullConfigSet) {
+            // If the full config set has been initialized (e.g. by calling cfgManager.getAllConfigs)
+            // then instead of doing redundant fs access, construct the result from the full config
+            // set
+            for (let configPathCandidate of configCandidatesForFile) {
+                if (this.fullConfigSet[configPathCandidate]) {
+                    partialSet[configPathCandidate] = this.fullConfigSet[configPathCandidate];
+                }
+            }
+        } else {
+            // If the full config set has not been initialized, go to disk to find configs in the
+            // candidate set.
+            //
+            // As we scan paths, we add them to our partial configs and our set of checked paths
+            // so we can avoid redudnant fs access for this same fence and path in the future.
+            for (let configPathCandidate of configCandidatesForFile) {
+                const configPathCandidateFull = path.join(configPathCandidate, 'fence.json');
+                if (this.discoveredPaths.has(configPathCandidateFull)) {
+                    const discoveredConfig = this.partialDiscoveredConfigs[configPathCandidate];
+                    if (discoveredConfig) {
+                        partialSet[configPathCandidateFull] = discoveredConfig;
+                    }
+                } else {
+                    try {
+                        const stat = fs.statSync(configPathCandidateFull);
+                        if (stat?.isFile()) {
+                            loadConfig(configPathCandidateFull, partialSet);
+                        }
+                    } catch {
+                        // pass e.g. for ENOENT
+                    }
+                    this.discoveredPaths.add(configPathCandidateFull);
+                }
+            }
+            Object.assign(this.partialDiscoveredConfigs, partialSet);
+        }
+
+        return partialSet;
+    }
+
+    private _getAllConfigs() {
+        this.fullConfigSet = {};
+
+        let files: string[] = [];
+        for (let rootDir of getOptions().rootDir) {
+            accumulateFences(rootDir, files, getOptions().ignoreExternalFences);
+        }
+
+        files.forEach(file => {
+            loadConfig(file, this.fullConfigSet);
+        });
+    }
+}
+
+let configManager: ConfigManager | null = null;
+export default function getConfigManager(): ConfigManager {
+    if (!configManager) {
+        configManager = new ConfigManager();
+    }
+    return configManager;
+}
+
+function accumulateFences(dir: string, files: string[], ignoreExternalFences: boolean) {
+    const directoryEntries: fs.Dirent[] = fs.readdirSync(dir, { withFileTypes: true });
+    for (const directoryEntry of directoryEntries) {
+        const fullPath = path.join(dir, directoryEntry.name);
+        if (directoryEntry.name == 'fence.json') {
+            files.push(fullPath);
+        } else if (
+            directoryEntry.isDirectory() &&
+            !(ignoreExternalFences && directoryEntry.name == 'node_modules')
+        ) {
+            accumulateFences(fullPath, files, ignoreExternalFences);
+        }
+    }
+}

src/utils/getConfigPathCandidatesForFile.ts

@@ -0,0 +1,16 @@
+import * as path from 'path';
+import NormalizedPath from '../types/NormalizedPath';
+import normalizePath from './normalizePath';
+
+export function getConfigPathCandidatesForFile(filePath: NormalizedPath): string[] {
+    const candidates: string[] = [];
+
+    let pathSegments = normalizePath(path.dirname(filePath)).split(path.sep);
+    while (pathSegments.length) {
+        let dirPath = pathSegments.join(path.sep);
+        candidates.push(dirPath);
+        pathSegments.pop();
+    }
+
+    return candidates;
+}

src/utils/getConfigsForFile.ts

@@ -1,23 +1,10 @@
-import * as path from 'path';
 import Config from '../types/config/Config';
 import NormalizedPath from '../types/NormalizedPath';
-import normalizePath from './normalizePath';
-import getAllConfigs from './getAllConfigs';
+import getConfigManager from './getConfigManager';
 
 // Returns an array of all the configs that apply to a given file
 export default function getConfigsForFile(filePath: NormalizedPath): Config[] {
-    let allConfigs = getAllConfigs();
-    let configsForFile: Config[] = [];
+    const partialFenceSet = getConfigManager().getPartialConfigSetForPath(filePath);
 
-    let pathSegments = normalizePath(path.dirname(filePath)).split(path.sep);
-    while (pathSegments.length) {
-        let dirPath = pathSegments.join(path.sep);
-        if (allConfigs[dirPath]) {
-            configsForFile.push(allConfigs[dirPath]);
-        }
-
-        pathSegments.pop();
-    }
-
-    return configsForFile;
+    return Object.entries(partialFenceSet).map(([_configPath, config]) => config);
 }

src/utils/loadConfig.ts

@@ -8,23 +8,34 @@ import RawExportRule from '../types/rawConfig/RawExportRule';
 import ConfigSet from '../types/ConfigSet';
 import ExportRule from '../types/config/ExportRule';
 import validateRawConfig from '../validation/validateRawConfig';
+import NormalizedPath from '../types/NormalizedPath';
 
-export default function loadConfig(file: string, configSet: ConfigSet) {
+function loadConfigFromString(configPath: NormalizedPath, fileContent: string): Config | null {
     // Load the raw config
-    let rawConfig: RawConfig = JSON.parse(fs.readFileSync(file).toString());
+    let rawConfig: RawConfig = JSON.parse(fileContent);
 
     // Validate it
-    const configPath = normalizePath(path.dirname(file));
     if (validateRawConfig(rawConfig, configPath)) {
         // Normalize it
-        const config: Config = {
+        return {
             path: configPath,
             tags: rawConfig.tags,
             exports: normalizeExportRules(rawConfig.exports),
             dependencies: normalizeDependencyRules(rawConfig.dependencies),
             imports: rawConfig.imports,
         };
+    }
+
+    return null;
+}
+
+export default function loadConfig(file: string, configSet: ConfigSet) {
+    // Load the raw config
+    const configPath = normalizePath(path.dirname(file));
 
+    // Validate and normalize it
+    const config = loadConfigFromString(configPath, fs.readFileSync(file, 'utf-8'));
+    if (config) {
         // Add it to the config set
         configSet[config.path] = config;
     }

src/validation/validateTagsExist.ts

@@ -1,9 +1,9 @@
 import Config from '../types/config/Config';
-import getAllConfigs from '../utils/getAllConfigs';
+import getConfigManager from '../utils/getConfigManager';
 import { reportWarning } from '../core/result';
 
 export function validateTagsExist() {
-    const allConfigs = getAllConfigs();
+    const allConfigs = getConfigManager().getAllConfigs();
     const allTags = new Set<string>();
 
     // Accumulate all tags that are defined

test/validation/validateTagsExistTests.ts

@@ -1,5 +1,5 @@
 import * as result from '../../src/core/result';
-import * as getAllConfigs from '../../src/utils/getAllConfigs';
+import * as getConfigManager from '../../src/utils/getConfigManager';
 import ConfigSet from '../../src/types/ConfigSet';
 import { validateTagsExist } from '../../src/validation/validateTagsExist';
 
@@ -8,7 +8,9 @@ describe('validateTagsExist', () => {
 
     beforeEach(() => {
         spyOn(result, 'reportWarning');
-        spyOn(getAllConfigs, 'default').and.callFake(() => allConfigs);
+        spyOn(getConfigManager, 'default').and.callFake(() => ({
+            getAllConfigs: () => allConfigs,
+        }));
     });
 
     it('passes for an empty config', () => {