Add feature split files by maxTokenSize per file

src/cli/actions/defaultAction.ts

@@ -56,8 +56,8 @@ export const runDefaultAction = async (
   spinner.succeed('Packing completed successfully!');
   logger.log('');
 
-  if (config.output.topFilesLength > 0) {
-    printTopFiles(packResult.fileCharCounts, packResult.fileTokenCounts, config.output.topFilesLength);
+  if (config.output.topFilesLength && config.output.topFilesLength > 0) {
+    printTopFiles(packResult.fileCharCounts, packResult.fileTokenCounts, config.output.topFilesLength ?? 0);
     logger.log('');
   }
 
@@ -68,7 +68,7 @@ export const runDefaultAction = async (
     packResult.totalFiles,
     packResult.totalCharacters,
     packResult.totalTokens,
-    config.output.filePath,
+    config.output.filePath ?? 'No output file specified',
     packResult.suspiciousFilesResults,
     config,
   );
@@ -103,6 +103,9 @@ const buildCliConfig = (options: CliOptions): RepopackConfigCli => {
   if (options.style) {
     cliConfig.output = { ...cliConfig.output, style: options.style.toLowerCase() as RepopackOutputStyle };
   }
+  if (options.maxTokens !== undefined) {
+    cliConfig.output = { ...cliConfig.output, maxTokensPerFile: options.maxTokens };
+  }
 
   return cliConfig;
 };

src/cli/actions/remoteAction.ts

@@ -29,7 +29,7 @@ export const runRemoteAction = async (repoUrl: string, options: CliOptions): Pro
     logger.log('');
 
     const result = await runDefaultAction(tempDir, tempDir, options);
-    await copyOutputToCurrentDirectory(tempDir, process.cwd(), result.config.output.filePath);
+    await copyOutputToCurrentDirectory(tempDir, process.cwd(), result.config.output.filePath ?? 'repopack-output.txt');
   } finally {
     // Clean up the temporary directory
     await cleanupTempDirectory(tempDir);

src/cli/cliRun.ts

@@ -23,6 +23,7 @@ export interface CliOptions extends OptionValues {
   init?: boolean;
   global?: boolean;
   remote?: string;
+  maxTokens?: number; // Add the maxTokens option
 }
 
 export async function run() {
@@ -44,6 +45,7 @@ export async function run() {
       .option('--init', 'initialize a new repopack.config.json file')
       .option('--global', 'use global configuration (only applicable with --init)')
       .option('--remote <url>', 'process a remote Git repository')
+      .option('--max-tokens <number>', 'maximum number of tokens per output file', Number.parseInt) // Add the maxTokens option
       .action((directory = '.', options: CliOptions = {}) => executeAction(directory, process.cwd(), options));
 
     await program.parseAsync(process.argv);

src/config/configLoad.ts

@@ -85,7 +85,7 @@ export const mergeConfigs = (
   // If the output file path is not provided in the config file or CLI, use the default file path for the style
   if (cliConfig.output?.filePath == null && fileConfig.output?.filePath == null) {
     const style = cliConfig.output?.style || fileConfig.output?.style || defaultConfig.output.style;
-    defaultConfig.output.filePath = defaultFilePathMap[style];
+    defaultConfig.output.filePath = defaultFilePathMap[style ?? 'plain'];
   }
 
   return {

src/config/configTypes.ts

@@ -1,16 +1,20 @@
 export type RepopackOutputStyle = 'plain' | 'xml' | 'markdown';
 
+export interface RepopackOutputConfig {
+  filePath?: string;
+  style?: RepopackOutputStyle;
+  headerText?: string;
+  instructionFilePath?: string;
+  removeComments?: boolean;
+  removeEmptyLines?: boolean;
+  topFilesLength?: number;
+  showLineNumbers?: boolean;
+  maxTokensPerFile?: number; // Added maxTokensPerFile
+  onlyShowPartFilesInRepoStructure?: boolean;
+}
+
 interface RepopackConfigBase {
-  output?: {
-    filePath?: string;
-    style?: RepopackOutputStyle;
-    headerText?: string;
-    instructionFilePath?: string;
-    removeComments?: boolean;
-    removeEmptyLines?: boolean;
-    topFilesLength?: number;
-    showLineNumbers?: boolean;
-  };
+  output?: RepopackOutputConfig;
   include?: string[];
   ignore?: {
     useGitignore?: boolean;
@@ -23,16 +27,7 @@ interface RepopackConfigBase {
 }
 
 export type RepopackConfigDefault = RepopackConfigBase & {
-  output: {
-    filePath: string;
-    style: RepopackOutputStyle;
-    headerText?: string;
-    instructionFilePath?: string;
-    removeComments: boolean;
-    removeEmptyLines: boolean;
-    topFilesLength: number;
-    showLineNumbers: boolean;
-  };
+  output: RepopackOutputConfig;
   include: string[];
   ignore: {
     useGitignore: boolean;

src/config/defaultConfig.ts

@@ -14,6 +14,7 @@ export const defaultConfig: RepopackConfigDefault = {
     removeEmptyLines: false,
     topFilesLength: 5,
     showLineNumbers: false,
+    onlyShowPartFilesInRepoStructure: false
   },
   include: [],
   ignore: {

src/core/output/outputGenerate.ts

@@ -8,52 +8,88 @@ import type { OutputGeneratorContext } from './outputGeneratorTypes.js';
 import { generateMarkdownStyle } from './outputStyles/markdownStyle.js';
 import { generatePlainStyle } from './outputStyles/plainStyle.js';
 import { generateXmlStyle } from './outputStyles/xmlStyle.js';
+import { splitOutput, type OutputSplit } from './outputSplitter.js';
 
 export const generateOutput = async (
   rootDir: string,
   config: RepopackConfigMerged,
   processedFiles: ProcessedFile[],
   allFilePaths: string[],
-): Promise<string> => {
-  const outputGeneratorContext = await buildOutputGeneratorContext(rootDir, config, allFilePaths, processedFiles);
-
-  let output: string;
-  switch (config.output.style) {
-    case 'xml':
-      output = generateXmlStyle(outputGeneratorContext);
-      break;
-    case 'markdown':
-      output = generateMarkdownStyle(outputGeneratorContext);
-      break;
-    default:
-      output = generatePlainStyle(outputGeneratorContext);
-  }
+): Promise<string[]> => {
+  const maxTokensPerFile = config.output.maxTokensPerFile ?? Infinity; // Use Infinity if no limit is set
+
+  const outputSplits: OutputSplit[] =
+    maxTokensPerFile < Infinity
+      ? splitOutput(
+          processedFiles,
+          maxTokensPerFile
+        )
+      : [{ partNumber: 1, tokenCount: 0, includedFiles: processedFiles }];
+
+  const outputs = await Promise.all(
+    outputSplits.map(async (outputSplit) => {
+      const outputGeneratorContext = await buildOutputGeneratorContext(
+        rootDir,
+        config,
+        outputSplit.includedFiles,  
+        config.output.onlyShowPartFilesInRepoStructure ? outputSplit.includedFiles.map(f => f.path) : allFilePaths,
+        processedFiles.length,
+        outputSplits.length,
+        outputSplit.partNumber,
+      )
 
-  return output;
+      let output: string;
+      switch (config.output.style) {
+        case 'xml':
+          output = generateXmlStyle(outputGeneratorContext);
+          break;
+        case 'markdown':
+          output = generateMarkdownStyle(outputGeneratorContext);
+          break;
+        default:
+          output = generatePlainStyle(outputGeneratorContext);
+      }
+      return output;
+    }),
+  );
+
+  return outputs;
 };
 
 export const buildOutputGeneratorContext = async (
   rootDir: string,
   config: RepopackConfigMerged,
-  allFilePaths: string[],
-  processedFiles: ProcessedFile[],
+  includedFiles: ProcessedFile[] = [], // Add includedFiles parameter
+  repositoryStructure: string[] = [],
+  totalFiles: number = 1,
+  totalParts: number = 1,
+  partNumber: number = 1
 ): Promise<OutputGeneratorContext> => {
   let repositoryInstruction = '';
 
   if (config.output.instructionFilePath) {
-    const instructionPath = path.resolve(rootDir, config.output.instructionFilePath);
+    const instructionPath = path.resolve(
+      rootDir,
+      config.output.instructionFilePath,
+    );
     try {
       repositoryInstruction = await fs.readFile(instructionPath, 'utf-8');
     } catch {
-      throw new RepopackError(`Instruction file not found at ${instructionPath}`);
+      throw new RepopackError(
+        `Instruction file not found at ${instructionPath}`,
+      );
     }
   }
 
   return {
     generationDate: new Date().toISOString(),
-    treeString: generateTreeString(allFilePaths),
-    processedFiles,
+    treeString: generateTreeString(repositoryStructure), // Use includedFiles for treeString
     config,
     instruction: repositoryInstruction,
+    content: '',
+    includedFiles, 
+    totalFiles,
+    totalParts,
+    partNumber
   };
 };

src/core/output/outputGeneratorTypes.ts

@@ -4,7 +4,11 @@ import type { ProcessedFile } from '../file/fileTypes.js';
 export interface OutputGeneratorContext {
   generationDate: string;
   treeString: string;
-  processedFiles: ProcessedFile[];
   config: RepopackConfigMerged;
   instruction: string;
+  content: string;
+  includedFiles: ProcessedFile[]; // Add the includedFiles property
+  totalFiles: number,
+  partNumber: number,
+  totalParts: number
 }

src/core/output/outputSplitter.ts

@@ -0,0 +1,53 @@
+import type { ProcessedFile } from '../file/fileTypes.js';
+import { TokenCounter } from '../tokenCount/tokenCount.js';
+
+export interface OutputSplit {
+  partNumber: number;
+  tokenCount: number;
+  includedFiles: ProcessedFile[]; // Add includedFiles property
+}
+
+export const splitOutput = (
+  processedFiles: ProcessedFile[],
+  maxTokensPerFile: number,
+): OutputSplit[] => {
+  const tokenCounter = new TokenCounter();
+  const outputSplits: OutputSplit[] = [];
+  let currentTokenCount = 0;
+  let currentOutput = '';
+  let currentIncludedFiles: ProcessedFile[] = []; // Initialize currentIncludedFiles
+
+  for (const file of processedFiles) {
+    const fileTokenCount = tokenCounter.countTokens(file.content, file.path);
+
+    if (currentTokenCount + fileTokenCount > maxTokensPerFile) {
+      // Start a new part
+      outputSplits.push({
+        partNumber: outputSplits.length+1,
+        tokenCount: currentTokenCount,
+        includedFiles: currentIncludedFiles, // Add includedFiles to the outputSplit
+      });
+
+      currentTokenCount = 0;
+      currentOutput = '';
+      currentIncludedFiles = []; // Reset currentIncludedFiles
+    }
+
+    currentOutput += file.content;
+    currentTokenCount += fileTokenCount;
+    currentIncludedFiles.push(file); // Add file path to currentIncludedFiles
+
+  }
+
+  if (currentIncludedFiles.length) {
+    // Add the last part
+    outputSplits.push({
+      partNumber: outputSplits.length+1,
+      tokenCount: currentTokenCount,
+      includedFiles: currentIncludedFiles, // Add includedFiles to the outputSplit
+    });
+  }
+  tokenCounter.free();
+
+  return outputSplits;
+};

src/core/output/outputStyles/markdownStyle.ts

@@ -25,7 +25,11 @@ export const generateMarkdownStyle = (outputGeneratorContext: OutputGeneratorCon
     headerText: outputGeneratorContext.config.output.headerText,
     instruction: outputGeneratorContext.instruction,
     treeString: outputGeneratorContext.treeString,
-    processedFiles: outputGeneratorContext.processedFiles,
+    includedFiles: outputGeneratorContext.includedFiles,
+    partNumber: outputGeneratorContext.partNumber,
+    totalParts: outputGeneratorContext.totalParts,
+    totalPartFiles: outputGeneratorContext.includedFiles.length,
+    totalFiles: outputGeneratorContext.totalFiles
   };
 
   return `${template(renderContext).trim()}\n`;
@@ -48,6 +52,10 @@ const markdownTemplate = /* md */ `
 ## Usage Guidelines
 {{{summaryUsageGuidelines}}}
 
+## Repository Size
+This file is part {{{partNumber}}} of {{{totalParts}}} of a split representation of the entire codebase.
+This file contains {{{totalPartFiles}}} out of a total of {{{totalFiles}}} files.
+
 ## Notes
 {{{summaryNotes}}}
 

src/core/output/outputStyles/plainStyle.ts

@@ -25,7 +25,11 @@ export const generatePlainStyle = (outputGeneratorContext: OutputGeneratorContex
     headerText: outputGeneratorContext.config.output.headerText,
     instruction: outputGeneratorContext.instruction,
     treeString: outputGeneratorContext.treeString,
-    processedFiles: outputGeneratorContext.processedFiles,
+    includedFiles: outputGeneratorContext.includedFiles,
+    partNumber: outputGeneratorContext.partNumber,
+    totalParts: outputGeneratorContext.totalParts,
+    totalPartFiles: outputGeneratorContext.includedFiles.length,
+    totalFiles: outputGeneratorContext.totalFiles
   };
 
   return `${template(renderContext).trim()}\n`;
@@ -59,6 +63,11 @@ Usage Guidelines:
 -----------------
 {{{summaryUsageGuidelines}}}
 
+Repository Size:
+-----------------
+This file is part {{{partNumber}}} of {{{totalParts}}} of a split representation of the entire codebase.
+This file contains {{{totalPartFiles}}} out of a total of {{{totalFiles}}} files.
+
 Notes:
 ------
 {{{summaryNotes}}}

src/core/output/outputStyles/xmlStyle.ts

@@ -25,7 +25,11 @@ export const generateXmlStyle = (outputGeneratorContext: OutputGeneratorContext)
     headerText: outputGeneratorContext.config.output.headerText,
     instruction: outputGeneratorContext.instruction,
     treeString: outputGeneratorContext.treeString,
-    processedFiles: outputGeneratorContext.processedFiles,
+    includedFiles: outputGeneratorContext.includedFiles,
+    partNumber: outputGeneratorContext.partNumber,
+    totalParts: outputGeneratorContext.totalParts,
+    totalPartFiles: outputGeneratorContext.includedFiles.length,
+    totalFiles: outputGeneratorContext.totalFiles
   };
 
   return `${template(renderContext).trim()}\n`;
@@ -52,6 +56,12 @@ This section contains a summary of this file.
 {{{summaryUsageGuidelines}}}
 </usage_guidelines>
 
+<repository_size>
+This file is part {{{partNumber}}} of {{{totalParts}}} of a split representation of the entire codebase.
+This file contains {{{totalPartFiles}}} out of a total of {{{totalFiles}}} files.
+
+</repository_size>
+
 <notes>
 {{{summaryNotes}}}
 </notes>
@@ -75,7 +85,7 @@ This section contains a summary of this file.
 <repository_files>
 This section contains the contents of the repository's files.
 
-{{#each processedFiles}}
+{{#each includedFiles}}
 <file path="{{{this.path}}}">
 {{{this.content}}}
 </file>