Enhance performance and user experience with improved progress reporting

src/cli/actions/defaultActionRunner.ts

@@ -63,7 +63,9 @@ export const runDefaultAction = async (
   let packResult: PackResult;
 
   try {
-    packResult = await pack(targetPath, config);
+    packResult = await pack(targetPath, config, (message) => {
+      spinner.update(message);
+    });
   } catch (error) {
     spinner.fail('Error during packing');
     throw error;

src/cli/actions/remoteActionRunner.ts

@@ -26,6 +26,8 @@ export const runRemoteAction = async (repoUrl: string, options: CliOptions): Pro
     spinner.start();
     await cloneRepository(formattedUrl, tempDir);
     spinner.succeed('Repository cloned successfully!');
+    logger.log('');
+
     const result = await runDefaultAction(tempDir, tempDir, options);
     await copyOutputToCurrentDirectory(tempDir, process.cwd(), result.config.output.filePath);
   } finally {

src/cli/cliSpinner.ts

@@ -13,13 +13,19 @@ class Spinner {
   }
 
   start(): void {
+    const frames = this.spinner.frames;
+    const framesLength = frames.length;
     this.interval = setInterval(() => {
-      const frame = this.spinner.frames[this.currentFrame];
+      this.currentFrame++;
+      const frame = frames[this.currentFrame % framesLength];
       logUpdate(`${pc.cyan(frame)} ${this.message}`);
-      this.currentFrame = (this.currentFrame + 1) % this.spinner.frames.length;
     }, this.spinner.interval);
   }
 
+  update(message: string): void {
+    this.message = message;
+  }
+
   stop(finalMessage: string): void {
     if (this.interval) {
       clearInterval(this.interval);

src/config/defaultIgnore.ts

@@ -118,4 +118,15 @@ export const defaultIgnoreList = [
 
   // repopack output
   'repopack-output.txt',
+
+  // Essential Python-related entries
+  '**/__pycache__/**',
+  '**/*.py[cod]',
+  'venv/**',
+  '.venv/**',
+  '.pytest_cache/**',
+  '.mypy_cache/**',
+  '.ipynb_checkpoints/**',
+  'Pipfile.lock',
+  'poetry.lock',
 ];

src/core/packager.ts

@@ -1,9 +1,11 @@
-import * as fs from 'node:fs/promises';
+import fs from 'node:fs/promises';
 import path from 'node:path';
 import pMap from 'p-map';
 import type { RepopackConfigMerged } from '../config/configTypes.js';
 import { logger } from '../shared/logger.js';
 import { getProcessConcurrency } from '../shared/processConcurrency.js';
+import { sleep } from '../shared/sleep.js';
+import type { RepopackProgressCallback } from '../shared/types.js';
 import { collectFiles as defaultCollectFiles } from './file/fileCollector.js';
 import { processFiles as defaultProcessFiles } from './file/fileProcessor.js';
 import { searchFiles as defaultSearchFiles } from './file/fileSearcher.js';
@@ -34,6 +36,7 @@ export interface PackResult {
 export const pack = async (
   rootDir: string,
   config: RepopackConfigMerged,
+  progressCallback: RepopackProgressCallback = () => {},
   deps: PackDependencies = {
     searchFiles: defaultSearchFiles,
     collectFiles: defaultCollectFiles,
@@ -43,25 +46,32 @@ export const pack = async (
   },
 ): Promise<PackResult> => {
   // Get all file paths considering the config
+  progressCallback('Searching for files...');
   const filePaths = await deps.searchFiles(rootDir, config);
 
   // Collect raw files
+  progressCallback('Collecting files...');
   const rawFiles = await deps.collectFiles(filePaths, rootDir);
 
   // Perform security check and filter out suspicious files
-  const suspiciousFilesResults = await deps.runSecurityCheck(rawFiles);
+  progressCallback('Running security check...');
+  const suspiciousFilesResults = await deps.runSecurityCheck(rawFiles, progressCallback);
   const safeRawFiles = rawFiles.filter(
     (rawFile) => !suspiciousFilesResults.some((result) => result.filePath === rawFile.path),
   );
   const safeFilePaths = safeRawFiles.map((file) => file.path);
+  logger.trace('Safe files count:', safeRawFiles.length);
 
   // Process files (remove comments, etc.)
+  progressCallback('Processing files...');
   const processedFiles = await deps.processFiles(safeRawFiles, config);
 
   // Generate output
+  progressCallback('Generating output...');
   const output = await deps.generateOutput(config, processedFiles, safeFilePaths);
 
   // Write output to file. path is relative to the cwd
+  progressCallback('Writing output file...');
   const outputPath = path.resolve(config.cwd, config.output.filePath);
   logger.trace(`Writing output to: ${outputPath}`);
   await fs.writeFile(outputPath, output);
@@ -70,11 +80,18 @@ export const pack = async (
   const tokenCounter = new TokenCounter();
 
   // Metrics
+  progressCallback('Calculating metrics...');
   const fileMetrics = await pMap(
     processedFiles,
-    async (file) => {
+    async (file, index) => {
       const charCount = file.content.length;
       const tokenCount = tokenCounter.countTokens(file.content);
+
+      progressCallback(`Calculating metrics... (${index + 1}/${processedFiles.length})`);
+
+      // Sleep for a short time to prevent blocking the event loop
+      await sleep(1);
+
       return { path: file.path, charCount, tokenCount };
     },
     {

src/core/security/securityCheckRunner.ts

@@ -4,26 +4,38 @@ import type { SecretLintCoreConfig, SecretLintCoreResult } from '@secretlint/typ
 import pMap from 'p-map';
 import { logger } from '../../shared/logger.js';
 import { getProcessConcurrency } from '../../shared/processConcurrency.js';
+import { sleep } from '../../shared/sleep.js';
+import type { RepopackProgressCallback } from '../../shared/types.js';
 import type { RawFile } from '../file/fileTypes.js';
 
 export interface SuspiciousFileResult {
   filePath: string;
   messages: string[];
 }
 
-export const runSecurityCheck = async (rawFiles: RawFile[]): Promise<SuspiciousFileResult[]> => {
+export const runSecurityCheck = async (
+  rawFiles: RawFile[],
+  progressCallback: RepopackProgressCallback = () => {},
+): Promise<SuspiciousFileResult[]> => {
   const secretLintConfig = createSecretLintConfig();
 
   const results = await pMap(
     rawFiles,
-    async (rawFile) => {
+    async (rawFile, index) => {
       const secretLintResult = await runSecretLint(rawFile.path, rawFile.content, secretLintConfig);
+
       if (secretLintResult.messages.length > 0) {
         return {
           filePath: rawFile.path,
           messages: secretLintResult.messages.map((message) => message.message),
         };
       }
+
+      progressCallback(`Running security check... (${index + 1}/${rawFiles.length})`);
+
+      // Sleep for a short time to prevent blocking the event loop
+      await sleep(1);
+
       return null;
     },
     {

src/shared/sleep.ts

@@ -0,0 +1 @@
+export const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));

src/shared/types.ts

@@ -0,0 +1 @@
+export type RepopackProgressCallback = (message: string) => void;

tests/core/packager.test.ts

@@ -36,7 +36,7 @@ describe('packager', () => {
   test('pack should process files and generate output', async () => {
     const mockConfig = createMockConfig();
 
-    const result = await pack('root', mockConfig, mockDeps);
+    const result = await pack('root', mockConfig, () => {}, mockDeps);
 
     const file2Path = path.join('dir1', 'file2.txt');
 
@@ -79,7 +79,7 @@ describe('packager', () => {
       },
     ]);
 
-    const result = await pack('root', mockConfig, mockDeps);
+    const result = await pack('root', mockConfig, () => {}, mockDeps);
 
     expect(mockDeps.searchFiles).toHaveBeenCalledWith('root', mockConfig);
     expect(mockDeps.processFiles).toHaveBeenCalledWith(