Merge pull request #50 from yamadashy/feature/parallel

Implement parallel processing

package.json

@@ -59,6 +59,7 @@
     "istextorbinary": "^9.5.0",
     "jschardet": "^3.1.3",
     "log-update": "^6.1.0",
+    "p-map": "^7.0.2",
     "picocolors": "^1.0.1",
     "strip-comments": "^2.0.1",
     "tiktoken": "^1.0.15"

src/core/file/fileCollector.ts

@@ -1,23 +1,30 @@
 import * as fs from 'node:fs/promises';
+import os from 'node:os';
 import path from 'node:path';
 import { isBinary } from 'istextorbinary';
 import jschardet from 'jschardet';
 import iconv from 'iconv-lite';
+import pMap from 'p-map';
 import { logger } from '../../shared/logger.js';
 import { RawFile } from './fileTypes.js';
 
 export const collectFiles = async (filePaths: string[], rootDir: string): Promise<RawFile[]> => {
-  const rawFiles: RawFile[] = [];
-
-  for (const filePath of filePaths) {
-    const fullPath = path.resolve(rootDir, filePath);
-    const content = await readRawFile(fullPath);
-    if (content) {
-      rawFiles.push({ path: filePath, content });
-    }
-  }
+  const rawFiles = await pMap(
+    filePaths,
+    async (filePath) => {
+      const fullPath = path.resolve(rootDir, filePath);
+      const content = await readRawFile(fullPath);
+      if (content) {
+        return { path: filePath, content };
+      }
+      return null;
+    },
+    {
+      concurrency: os.cpus().length,
+    },
+  );
 
-  return rawFiles;
+  return rawFiles.filter((file): file is RawFile => file != null);
 };
 
 const readRawFile = async (filePath: string): Promise<string | null> => {

src/core/file/fileProcessor.ts

@@ -1,15 +1,27 @@
+import os from 'node:os';
+import pMap from 'p-map';
 import { RepopackConfigMerged } from '../../config/configTypes.js';
 import { getFileManipulator } from './fileManipulater.js';
 import { ProcessedFile, RawFile } from './fileTypes.js';
 
-export const processFiles = (rawFiles: RawFile[], config: RepopackConfigMerged): ProcessedFile[] => {
-  return rawFiles.map((rawFile) => ({
-    path: rawFile.path,
-    content: processContent(rawFile.content, rawFile.path, config),
-  }));
+export const processFiles = async (rawFiles: RawFile[], config: RepopackConfigMerged): Promise<ProcessedFile[]> => {
+  return pMap(
+    rawFiles,
+    async (rawFile) => ({
+      path: rawFile.path,
+      content: await processContent(rawFile.content, rawFile.path, config),
+    }),
+    {
+      concurrency: os.cpus().length,
+    },
+  );
 };
 
-export const processContent = (content: string, filePath: string, config: RepopackConfigMerged): string => {
+export const processContent = async (
+  content: string,
+  filePath: string,
+  config: RepopackConfigMerged,
+): Promise<string> => {
   let processedContent = content;
   const manipulator = getFileManipulator(filePath);
 

src/core/packager.ts

@@ -1,5 +1,7 @@
+import os from 'node:os';
 import * as fs from 'node:fs/promises';
 import path from 'node:path';
+import pMap from 'p-map';
 import { RepopackConfigMerged } from '../config/configTypes.js';
 import { generateOutput as defaultGenerateOutput } from './output/outputGenerator.js';
 import { SuspiciousFileResult, runSecurityCheck as defaultRunSecurityCheck } from './security/securityCheckRunner.js';
@@ -50,7 +52,7 @@ export const pack = async (
   const safeFilePaths = safeRawFiles.map((file) => file.path);
 
   // Process files (remove comments, etc.)
-  const processedFiles = deps.processFiles(safeRawFiles, config);
+  const processedFiles = await deps.processFiles(safeRawFiles, config);
 
   // Generate output
   const output = await deps.generateOutput(config, processedFiles, safeFilePaths);
@@ -63,16 +65,30 @@ export const pack = async (
   const tokenCounter = new TokenCounter();
 
   // Metrics
+  const fileMetrics = await pMap(
+    processedFiles,
+    async (file) => {
+      const charCount = file.content.length;
+      const tokenCount = tokenCounter.countTokens(file.content);
+      return { path: file.path, charCount, tokenCount };
+    },
+    {
+      concurrency: os.cpus().length,
+    },
+  );
+
+  tokenCounter.free();
+
   const totalFiles = processedFiles.length;
-  const totalCharacters = processedFiles.reduce((sum, file) => sum + file.content.length, 0);
-  const totalTokens = processedFiles.reduce((sum, file) => sum + tokenCounter.countTokens(file.content), 0);
+  const totalCharacters = fileMetrics.reduce((sum, fileMetric) => sum + fileMetric.charCount, 0);
+  const totalTokens = fileMetrics.reduce((sum, fileMetric) => sum + fileMetric.tokenCount, 0);
+
   const fileCharCounts: Record<string, number> = {};
   const fileTokenCounts: Record<string, number> = {};
-  processedFiles.forEach((file) => {
-    fileCharCounts[file.path] = file.content.length;
-    fileTokenCounts[file.path] = tokenCounter.countTokens(file.content);
+  fileMetrics.forEach((file) => {
+    fileCharCounts[file.path] = file.charCount;
+    fileTokenCounts[file.path] = file.tokenCount;
   });
-  tokenCounter.free();
 
   return {
     totalFiles,

src/core/security/securityCheckRunner.ts

@@ -1,6 +1,8 @@
+import os from 'node:os';
 import type { SecretLintCoreConfig, SecretLintCoreResult } from '@secretlint/types';
 import { lintSource } from '@secretlint/core';
 import { creator } from '@secretlint/secretlint-rule-preset-recommend';
+import pMap from 'p-map';
 import { logger } from '../../shared/logger.js';
 import { RawFile } from '../file/fileTypes.js';
 
@@ -11,20 +13,25 @@ export interface SuspiciousFileResult {
 
 export const runSecurityCheck = async (rawFiles: RawFile[]): Promise<SuspiciousFileResult[]> => {
   const secretLintConfig = createSecretLintConfig();
-  const suspiciousFilesResults: SuspiciousFileResult[] = [];
-
-  for (const rawFile of rawFiles) {
-    const secretLintResult = await runSecretLint(rawFile.path, rawFile.content, secretLintConfig);
-    const isSuspicious = secretLintResult.messages.length > 0;
-    if (isSuspicious) {
-      suspiciousFilesResults.push({
-        filePath: rawFile.path,
-        messages: secretLintResult.messages.map((message) => message.message),
-      });
-    }
-  }
 
-  return suspiciousFilesResults;
+  const results = await pMap(
+    rawFiles,
+    async (rawFile) => {
+      const secretLintResult = await runSecretLint(rawFile.path, rawFile.content, secretLintConfig);
+      if (secretLintResult.messages.length > 0) {
+        return {
+          filePath: rawFile.path,
+          messages: secretLintResult.messages.map((message) => message.message),
+        };
+      }
+      return null;
+    },
+    {
+      concurrency: os.cpus().length,
+    },
+  );
+
+  return results.filter((result): result is SuspiciousFileResult => result != null);
 };
 
 export const runSecretLint = async (

tests/core/file/fileProcessor.test.ts

@@ -8,7 +8,7 @@ vi.mock('../../../src/core/file/fileManipulater');
 
 describe('fileProcessor', () => {
   describe('processFiles', () => {
-    it('should process multiple files', () => {
+    it('should process multiple files', async () => {
       const mockRawFiles: RawFile[] = [
         { path: 'file1.js', content: '// comment\nconst a = 1;' },
         { path: 'file2.js', content: '/* comment */\nconst b = 2;' },
@@ -25,7 +25,7 @@ describe('fileProcessor', () => {
         removeEmptyLines: (content: string) => content.replace(/^\s*[\r\n]/gm, ''),
       });
 
-      const result = processFiles(mockRawFiles, mockConfig);
+      const result = await processFiles(mockRawFiles, mockConfig);
 
       expect(result).toEqual([
         { path: 'file1.js', content: 'const a = 1;' },
@@ -35,7 +35,7 @@ describe('fileProcessor', () => {
   });
 
   describe('processContent', () => {
-    it('should remove comments and empty lines when configured', () => {
+    it('should remove comments and empty lines when configured', async () => {
       const content = '// comment\nconst a = 1;\n\n/* multi-line\ncomment */\nconst b = 2;';
       const filePath = 'test.js';
       const config: RepopackConfigMerged = {
@@ -50,12 +50,12 @@ describe('fileProcessor', () => {
         removeEmptyLines: (content: string) => content.replace(/^\s*[\r\n]/gm, ''),
       });
 
-      const result = processContent(content, filePath, config);
+      const result = await processContent(content, filePath, config);
 
       expect(result).toBe('const a = 1;\nconst b = 2;');
     });
 
-    it('should not remove comments or empty lines when not configured', () => {
+    it('should not remove comments or empty lines when not configured', async () => {
       const content = '// comment\nconst a = 1;\n\n/* multi-line\ncomment */\nconst b = 2;';
       const filePath = 'test.js';
       const config: RepopackConfigMerged = {
@@ -65,12 +65,12 @@ describe('fileProcessor', () => {
         },
       } as RepopackConfigMerged;
 
-      const result = processContent(content, filePath, config);
+      const result = await processContent(content, filePath, config);
 
       expect(result).toBe(content.trim());
     });
 
-    it('should handle files without a manipulator', () => {
+    it('should handle files without a manipulator', async () => {
       const content = 'Some content';
       const filePath = 'unknown.ext';
       const config: RepopackConfigMerged = {
@@ -82,7 +82,7 @@ describe('fileProcessor', () => {
 
       vi.mocked(getFileManipulator).mockReturnValue(null);
 
-      const result = processContent(content, filePath, config);
+      const result = await processContent(content, filePath, config);
 
       expect(result).toBe(content);
     });