Merge pull request #4 from yamadashy/feature/comment-remove-option

feat: Add comment remove feature

Author: yamadashy

README.md

@@ -6,13 +6,17 @@
 
 Repopack is a powerful tool that packs your entire repository into a single, AI-friendly file. Perfect for when you need to feed your codebase to Large Language Models (LLMs) or other AI tools.
 
+
+
 ## 🚀 Features
 
 - **AI-Optimized**: Formats your codebase in a way that's easy for AI to understand and process.
 - **Simple to Use**: Just one command to pack your entire repository.
 - **Customizable**: Easily configure what to include or exclude.
 - **Git-Aware**: Automatically respects your .gitignore files.
 
+
+
 ## 🛠 Installation
 
 You can install Repopack globally using npm:
@@ -33,6 +37,8 @@ Alternatively, you can use npx to run Repopack without installing it:
 npx repopack
 ```
 
+
+
 ## 📊 Usage
 
 To pack your entire repository:
@@ -81,6 +87,8 @@ repopack -c ./custom-config.json
 npx repopack src
 ```
 
+
+
 ## ⚙️ Configuration
 
 Create a `repopack.config.json` file in your project root for custom configurations. Here's an explanation of the configuration options:
@@ -89,6 +97,7 @@ Create a `repopack.config.json` file in your project root for custom configurati
 |--------|-------------|---------|
 |`output.filePath`| The name of the output file | `"repopack-output.txt"` |
 |`output.headerText`| Custom text to include in the file header |`null`|
+|`output.removeComments`| Whether to remove comments from supported file types. Suppurts python  | `false` |
 |`ignore.useDefaultPatterns`| Whether to use default ignore patterns |`true`|
 |`ignore.customPatterns`| Additional patterns to ignore |`[]`|
 
@@ -97,8 +106,9 @@ Example configuration:
 ```json
 {
   "output": {
-    "filePath": "custom-output.txt",
-    "headerText": "Custom header information for the packed file."
+    "filePath": "repopack-output.txt",
+    "headerText": "Custom header information for the packed file.",
+    "removeComments": true
   },
   "ignore": {
     "useDefaultPatterns": true,
@@ -119,6 +129,36 @@ Repopack automatically ignores certain files and directories by default:
 
 This ensures that only relevant source code is included in the packed file. You can add additional ignore patterns using the `ignore.customPatterns` configuration option or the `-i` command line flag.
 
+### Comment Removal
+
+When `output.removeComments` is set to `true`, Repopack will attempt to remove comments from supported file types. This feature can help reduce the size of the output file and focus on the essential code content.
+
+Currently supported file types for comment removal:
+
+- HTML (.html)
+- CSS (.css, .scss, .sass)
+- JavaScript, React (.js, .jsx)
+- TypeScript (.ts, .tsx)
+- Vue (.vue)
+- Svelte (.svelte)
+- Python (.py)
+- PHP (.php)
+- Ruby (.rb)
+- C (.c)
+- C# (.cs)
+- Java (.java)
+- Go (.go)
+- Rust (.rs)
+- Swift (.swift)
+- Kotlin (.kt)
+- Dart (.dart)
+- Shell (.sh)
+- YAML (.yml, .yaml)
+
+Note: The comment removal process is designed to be conservative to avoid accidentally removing code. In some complex cases, especially with nested comments or language-specific peculiarities, some comments might be retained.
+
+
+
 ## 📄 Output Format
 
 Repopack generates a single file with clear separators between different parts of your codebase:
@@ -146,5 +186,7 @@ File: src/utils.js
 
 This format ensures that AI tools can easily distinguish between different files in your codebase.
 
+
+
 ## 📜 License
 MIT

package-lock.json

@@ -56,14 +56,16 @@
     "is-binary-path": "^2.1.0",
     "jschardet": "^3.1.3",
     "log-update": "^6.0.0",
-    "picocolors": "^1.0.1"
+    "picocolors": "^1.0.1",
+    "strip-comments": "^2.0.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.7.0",
     "@types/eslint": "~8.56.10",
     "@types/eslint__js": "~8.42.3",
     "@types/eslint-config-prettier": "~6.11.3",
     "@types/node": "^20.14.10",
+    "@types/strip-comments": "^2.0.4",
     "@typescript-eslint/eslint-plugin": "^7.16.0",
     "@typescript-eslint/parser": "^7.16.0",
     "@vitest/coverage-v8": "^2.0.2",

package.json

@@ -1,7 +1,8 @@
 {
   "output": {
     "filePath": "repopack-output.txt",
-    "headerText": "This repository contains the source code for the Repopack tool.\nRepopack is designed to pack repository contents into a single file,\nmaking it easier for AI systems to analyze and process the codebase.\n\nKey Features:\n- Configurable ignore patterns\n- Custom header text support\n- Efficient file processing and packing\n\nPlease refer to the README.md file for more detailed information on usage and configuration.\n"
+    "headerText": "This repository contains the source code for the Repopack tool.\nRepopack is designed to pack repository contents into a single file,\nmaking it easier for AI systems to analyze and process the codebase.\n\nKey Features:\n- Configurable ignore patterns\n- Custom header text support\n- Efficient file processing and packing\n\nPlease refer to the README.md file for more detailed information on usage and configuration.\n",
+    "removeComments": false
   },
   "ignore": {
     "useDefaultPatterns": true,

repopack.config.json

@@ -3,6 +3,7 @@ import { RepopackConfigDefault } from '../types/index.js';
 export const defaultConfig: RepopackConfigDefault = {
   output: {
     filePath: 'repopack-output.txt',
+    removeComments: false,
   },
   ignore: {
     useDefaultPatterns: true,

src/config/defaultConfig.ts

@@ -68,6 +68,7 @@ Notes:
 - Some files may have been excluded based on .gitignore rules and Repopack's
   configuration.
 - Binary files are not included in this packed representation.
+${config.output.removeComments ? '- Code comments have been removed.\n' : ''}
 
 For more information about Repopack, visit: https://github.com/yamadashy/repopack
 `;

src/core/outputGenerator.ts

@@ -81,7 +81,7 @@ async function packDirectory(
       const subDirFiles = await packDirectory(fullPath, entryRelativePath, config, ignoreFilter, deps);
       packedFiles.push(...subDirFiles);
     } else {
-      const content = await deps.processFile(fullPath);
+      const content = await deps.processFile(fullPath, config);
       if (content) {
         packedFiles.push({ path: entryRelativePath, content });
       }

src/core/packager.ts

@@ -2,6 +2,7 @@ interface RepopackConfigBase {
   output?: {
     filePath?: string;
     headerText?: string;
+    removeComments?: boolean;
   };
   ignore?: {
     useDefaultPatterns?: boolean;
@@ -13,6 +14,7 @@ export type RepopackConfigDefault = RepopackConfigBase & {
   output: {
     filePath: string;
     headerText?: string;
+    removeComments?: boolean;
   };
   ignore: {
     useDefaultPatterns: boolean;
@@ -24,6 +26,7 @@ export type RepopackConfigFile = RepopackConfigBase & {
   output?: {
     filePath?: string;
     headerText?: string;
+    removeComments?: boolean;
   };
   ignore?: {
     useDefaultPatterns?: boolean;
@@ -35,6 +38,7 @@ export type RepopackConfigCli = RepopackConfigBase & {
   output?: {
     filePath?: string;
     headerText?: string;
+    removeComments?: boolean;
   };
   ignore?: {
     useDefaultPatterns?: boolean;

src/types/index.ts

@@ -2,8 +2,14 @@ import * as fs from 'fs/promises';
 import isBinaryPath from 'is-binary-path';
 import jschardet from 'jschardet';
 import iconv from 'iconv-lite';
+import { RepopackConfigMerged } from '../types/index.js';
+import { getFileManipulator } from './fileManipulator.js';
 
-export async function processFile(filePath: string, fsModule = fs): Promise<string | null> {
+export async function processFile(
+  filePath: string,
+  config: RepopackConfigMerged,
+  fsModule = fs,
+): Promise<string | null> {
   // Skip binary files
   if (isBinaryPath(filePath)) {
     return null;
@@ -12,13 +18,22 @@ export async function processFile(filePath: string, fsModule = fs): Promise<stri
   try {
     const buffer = await fsModule.readFile(filePath);
     const encoding = jschardet.detect(buffer).encoding || 'utf-8';
-    const content = iconv.decode(buffer, encoding);
+    let content = iconv.decode(buffer, encoding);
 
     if (!isValidTextContent(content)) {
       return null;
     }
 
-    return preprocessContent(content);
+    content = preprocessContent(content);
+
+    if (config.output.removeComments) {
+      const manipulator = getFileManipulator(filePath);
+      if (manipulator) {
+        content = manipulator.removeComments(content);
+      }
+    }
+
+    return content;
   } catch (error) {
     console.warn(`Error processing file ${filePath}:`, error);
     return null;

src/utils/fileHandler.ts

@@ -0,0 +1,93 @@
+import path from 'path';
+import strip from 'strip-comments';
+
+interface FileManipulator {
+  removeComments(content: string): string;
+}
+
+function rtrimLines(content: string): string {
+  return content.replace(/\s+$/gm, '');
+}
+
+class StripCommentsManipulator implements FileManipulator {
+  private language: string;
+
+  constructor(language: string) {
+    this.language = language;
+  }
+
+  removeComments(content: string): string {
+    let result = strip(content, { language: this.language, preserveNewlines: true });
+    return rtrimLines(result);
+  }
+}
+
+class PythonManipulator implements FileManipulator {
+  removeComments(content: string): string {
+    // First, use strip-comments to remove standard comments
+    let result = strip(content, { language: 'python', preserveNewlines: true });
+
+    // Then, remove triple-quote comments
+    result = result.replace(/'''[\s\S]*?'''/g, '');
+    result = result.replace(/"""[\s\S]*?"""/g, '');
+
+    return rtrimLines(result);
+  }
+}
+
+class CompositeManipulator implements FileManipulator {
+  private manipulators: FileManipulator[];
+
+  constructor(...manipulators: FileManipulator[]) {
+    this.manipulators = manipulators;
+  }
+
+  removeComments(content: string): string {
+    return this.manipulators.reduce((acc, manipulator) => manipulator.removeComments(acc), content);
+  }
+}
+
+const manipulators: Record<string, FileManipulator> = {
+  '.c': new StripCommentsManipulator('c'),
+  '.cs': new StripCommentsManipulator('csharp'),
+  '.css': new StripCommentsManipulator('css'),
+  '.dart': new StripCommentsManipulator('c'),
+  '.go': new StripCommentsManipulator('c'),
+  '.html': new StripCommentsManipulator('html'),
+  '.java': new StripCommentsManipulator('java'),
+  '.js': new StripCommentsManipulator('javascript'),
+  '.jsx': new StripCommentsManipulator('javascript'),
+  '.kt': new StripCommentsManipulator('c'),
+  '.less': new StripCommentsManipulator('less'),
+  '.php': new StripCommentsManipulator('php'),
+  '.rb': new StripCommentsManipulator('ruby'),
+  '.rs': new StripCommentsManipulator('c'),
+  '.sass': new StripCommentsManipulator('sass'),
+  '.scss': new StripCommentsManipulator('sass'),
+  '.sh': new StripCommentsManipulator('perl'),
+  '.sql': new StripCommentsManipulator('sql'),
+  '.swift': new StripCommentsManipulator('swift'),
+  '.ts': new StripCommentsManipulator('javascript'),
+  '.tsx': new StripCommentsManipulator('javascript'),
+  '.xml': new StripCommentsManipulator('xml'),
+  '.yaml': new StripCommentsManipulator('perl'),
+  '.yml': new StripCommentsManipulator('perl'),
+
+  '.py': new PythonManipulator(),
+
+  '.vue': new CompositeManipulator(
+    new StripCommentsManipulator('html'),
+    new StripCommentsManipulator('css'),
+    new StripCommentsManipulator('javascript'),
+  ),
+  '.svelte': new CompositeManipulator(
+    new StripCommentsManipulator('html'),
+    new StripCommentsManipulator('css'),
+    new StripCommentsManipulator('javascript'),
+  ),
+};
+
+export function getFileManipulator(filePath: string): FileManipulator | null {
+  const ext = path.extname(filePath);
+  return manipulators[ext] || null;
+}

src/utils/fileManipulator.ts

@@ -1,14 +1,15 @@
 import * as fs from 'fs/promises';
 import path from 'path';
 import ignore from 'ignore';
+import { logger } from './logger.js';
 
 export async function getGitignorePatterns(rootDir: string, fsModule = fs): Promise<string[]> {
   const gitignorePath = path.join(rootDir, '.gitignore');
   try {
     const gitignoreContent = await fsModule.readFile(gitignorePath, 'utf-8');
     return parseGitignoreContent(gitignoreContent);
   } catch (error) {
-    console.warn('No .gitignore file found or unable to read it.');
+    logger.warn('No .gitignore file found or unable to read it.');
     return [];
   }
 }