Merge pull request #31 from ersel/scan-parallelism

feat(parallelism): scan files in chunks

cli/dependencies/index.js

@@ -23,6 +23,7 @@ const dependenciesAction = (args, opts) => {
 			includedPatterns: opts.incl,
 			excludedPatterns: opts.excl,
 			extensions: opts.ext,
+			fileScanParallelism: opts.fileScanParallelism,
 			aliases
 		},
 		opts.rescan

cli/dependencies/index.test.js

@@ -94,6 +94,7 @@ describe('dependencies command', () => {
 			incl: ['**/*.js'],
 			excl: ['node_modules/**'],
 			alias: 'some-alias:/path/to/some-alias',
+			fileScanParallelism: 50,
 			rescan: true
 		};
 
@@ -107,6 +108,7 @@ describe('dependencies command', () => {
 					aliases: { 'some-alias': '/path/to/some-alias' },
 					excludedPatterns: ['node_modules/**'],
 					extensions: ['js'],
+					fileScanParallelism: 50,
 					includedPatterns: ['**/*.js'],
 					projectRootPath: process.cwd(),
 					targetDirectory: '.'
@@ -125,6 +127,7 @@ describe('dependencies command', () => {
 					aliases: {},
 					excludedPatterns: ['node_modules/**'],
 					extensions: ['js'],
+					fileScanParallelism: 50,
 					includedPatterns: ['**/*.js'],
 					projectRootPath: process.cwd(),
 					targetDirectory: '.'

cli/dependent/index.js

@@ -31,6 +31,7 @@ const dependentAction = (args, opts) => {
 			includedPatterns: opts.incl,
 			excludedPatterns: opts.excl,
 			extensions: opts.ext,
+			fileScanParallelism: opts.fileScanParallelism,
 			aliases
 		},
 		opts.rescan

cli/dependent/index.test.js

@@ -62,6 +62,7 @@ describe('dependent command', () => {
 			incl: ['**/*.js'],
 			excl: ['node_modules/**'],
 			alias: 'some-alias:/path/to/some-alias',
+			fileScanParallelism: 50,
 			rescan: true
 		};
 	});
@@ -73,6 +74,7 @@ describe('dependent command', () => {
 						aliases: { 'some-alias': '/path/to/some-alias' },
 						excludedPatterns: ['node_modules/**'],
 						extensions: ['js'],
+						fileScanParallelism: 50,
 						includedPatterns: ['**/*.js'],
 						projectRootPath: testRootPath,
 						targetDirectory: '.'
@@ -98,6 +100,7 @@ describe('dependent command', () => {
 					aliases: {},
 					excludedPatterns: ['node_modules/**'],
 					extensions: ['js'],
+					fileScanParallelism: 50,
 					includedPatterns: ['**/*.js'],
 					projectRootPath: testRootPath,
 					targetDirectory: '.'

cli/naylias/index.js

@@ -23,6 +23,7 @@ const nayliasAction = (args, opts) => {
 			includedPatterns: opts.incl,
 			excludedPatterns: opts.excl,
 			extensions: opts.ext,
+			fileScanParallelism: opts.fileScanParallelism,
 			aliases
 		},
 		opts.rescan

cli/naylias/index.test.js

@@ -88,6 +88,7 @@ describe('naylias command', () => {
 			incl: ['**/*.js'],
 			excl: ['node_modules/**'],
 			alias: `my-utils:${PROJECT_ROOT}/src/utils`,
+			fileScanParallelism: 50,
 			rescan: true
 		};
 	});
@@ -99,6 +100,7 @@ describe('naylias command', () => {
 					aliases: { 'my-utils': `${PROJECT_ROOT}/src/utils` },
 					excludedPatterns: ['node_modules/**'],
 					extensions: ['js'],
+					fileScanParallelism: 50,
 					includedPatterns: ['**/*.js'],
 					projectRootPath: PROJECT_ROOT,
 					targetDirectory: '.'

cli/surface/index.js

@@ -36,6 +36,7 @@ const surfaceAction = (args, opts) => {
 			includedPatterns: opts.incl,
 			excludedPatterns: opts.excl,
 			extensions: opts.ext,
+			fileScanParallelism: opts.fileScanParallelism,
 			aliases
 		},
 		opts.rescan

cli/surface/index.test.js

@@ -65,6 +65,7 @@ describe('surface command', () => {
 			incl: ['**/*.js'],
 			excl: ['node_modules/**'],
 			alias: 'some-alias:/path/to/some-alias',
+			fileScanParallelism: 50,
 			rescan: true
 		};
 
@@ -78,6 +79,7 @@ describe('surface command', () => {
 					aliases: { 'some-alias': '/path/to/some-alias' },
 					excludedPatterns: ['node_modules/**'],
 					extensions: ['js'],
+					fileScanParallelism: 50,
 					includedPatterns: ['**/*.js'],
 					projectRootPath: process.cwd(),
 					targetDirectory: '.'
@@ -96,6 +98,7 @@ describe('surface command', () => {
 					aliases: {},
 					excludedPatterns: ['node_modules/**'],
 					extensions: ['js'],
+					fileScanParallelism: 50,
 					includedPatterns: ['**/*.js'],
 					projectRootPath: process.cwd(),
 					targetDirectory: '.'

lib/scanner/index.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const path = require('path');
 const util = require('util');
 const glob = require('../../utils/glob');
+const sequence = require('../../utils/sequence');
 const resolveES6Imports = require('../../utils/es6-import-resolver');
 const resolveCJSImports = require('../../utils/static-require-resolver');
 const resolvePath = require('../../utils/filepath-resolver');
@@ -25,27 +26,26 @@ async function scanner({
 	excludedPatterns,
 	extensions,
 	aliases,
-	projectRootPath
+	projectRootPath,
+	fileScanParallelism = 50
 }) {
 	const filesToAnalyse = glob(
 		path.join(projectRootPath, targetDirectory),
 		includedPatterns,
 		excludedPatterns
 	);
 
-	const chunksOfFiles = ramda.splitEvery(2, filesToAnalyse);
-	return Promise.all(
-		chunksOfFiles.map(async files => {
-			const modules = await readModules(files);
-			return modules.map(({ sourceFile, fileContents }) => ({
-				sourceFile,
-				importedModules: [
-					...resolveES6Imports(fileContents, sourceFile),
-					...resolveCJSImports(fileContents, sourceFile)
-				]
-			}));
-		})
-	)
+	const chunksOfFiles = ramda.splitEvery(fileScanParallelism, filesToAnalyse);
+	return sequence(chunksOfFiles, async files => {
+		const modules = await readModules(files);
+		return modules.map(({ sourceFile, fileContents }) => ({
+			sourceFile,
+			importedModules: [
+				...resolveES6Imports(fileContents, sourceFile),
+				...resolveCJSImports(fileContents, sourceFile)
+			]
+		}));
+	})
 		.then(results => ramda.flatten(results))
 		.then(results =>
 			results.map(({ sourceFile, importedModules }) => ({

lib/scanner/index.test.js

@@ -17,7 +17,8 @@ describe('Import Scanner ES6', () => {
 			includedPatterns: testIncludePatterns,
 			extensions: testExtensions,
 			aliases: testAliases,
-			projectRootPath: testRootPath
+			projectRootPath: testRootPath,
+			fileScanParallelism: 50
 		}).then(results => {
 			expect(results.length).toEqual(4);
 			expect(results).toEqual([

prog.js

@@ -43,6 +43,12 @@ const GLOBAL_OPTIONS = [
 		prog.BOOL,
 		false,
 		false
+	],
+	[
+		'--file-scan-parallelism',
+		'Number of files to scan at once',
+		prog.INTEGER,
+		50
 	]
 ];
 

utils/sequence/index.js

@@ -0,0 +1,7 @@
+const sequence = (tasks, fn) =>
+	tasks.reduce(
+		(promise, task) => promise.then(() => fn(task)),
+		Promise.resolve()
+	);
+
+module.exports = sequence;