Skip empty files in indexing, add more partitions (#964)

* Skip empty files and show empty and unindexed files in list files command
* Add more partitions

Author: logancyang

src/main.ts

@@ -343,18 +343,49 @@ export default class CopilotPlugin extends Plugin {
       callback: async () => {
         try {
           const indexedFiles = await this.vectorStoreManager.getIndexedFiles();
-          if (indexedFiles.length === 0) {
-            new Notice("No indexed files found.");
+          const indexedFilePaths = new Set(indexedFiles);
+          const allMarkdownFiles = this.app.vault.getMarkdownFiles();
+          const emptyFiles = new Set<string>();
+          const unindexedFiles = new Set<string>();
+
+          // Categorize files
+          for (const file of allMarkdownFiles) {
+            const content = await this.app.vault.cachedRead(file);
+            if (!content || content.trim().length === 0) {
+              emptyFiles.add(file.path);
+            } else if (!indexedFilePaths.has(file.path)) {
+              unindexedFiles.add(file.path);
+            }
+          }
+
+          if (indexedFiles.length === 0 && emptyFiles.size === 0 && unindexedFiles.size === 0) {
+            new Notice("No files found to list.");
             return;
           }
 
           // Create content for the file
           const content = [
-            "# Copilot Indexed Files",
-            `Total files indexed: ${indexedFiles.length}`,
+            "# Copilot Files Status",
+            `- Indexed files: ${indexedFiles.length}`,
+            `- Unindexed files: ${unindexedFiles.size}`,
+            `- Empty files: ${emptyFiles.size}`,
             "",
-            "## Files",
+            "## Indexed Files",
             ...indexedFiles.map((file) => `- [[${file}]]`),
+            "",
+            "## Unindexed Files",
+            ...(unindexedFiles.size > 0
+              ? Array.from(unindexedFiles)
+                  .sort()
+                  .map((file) => `- [[${file}]]`)
+              : ["No unindexed files found."]),
+            "",
+            "## Empty Files",
+            ...(emptyFiles.size > 0
+              ? Array.from(emptyFiles)
+                  .sort()
+                  .map((file) => `- [[${file}]]`)
+              : ["No empty files found."]),
           ].join("\n");
 
           // Create or update the file in the vault

src/search/indexOperations.ts

@@ -249,28 +249,33 @@ export class IndexOperations {
     // Get all markdown files that should be indexed under current rules
     const allMarkdownFiles = this.app.vault.getMarkdownFiles();
     const filesToIndex = new Set<TFile>();
+    const emptyFiles = new Set<string>();
 
     for (const file of allMarkdownFiles) {
-      // Always skip excluded files
+      // Skip excluded files
       if (excludedFiles.has(file.path)) {
         continue;
       }
 
+      // Check actual content
+      const content = await this.app.vault.cachedRead(file);
+      if (!content || content.trim().length === 0) {
+        emptyFiles.add(file.path);
+        continue;
+      }
+
       const shouldBeIndexed = includedFiles.size === 0 || includedFiles.has(file.path);
+      const isIndexed = indexedFilePaths.has(file.path);
 
-      if (shouldBeIndexed) {
-        // Add file if:
-        // 1. It's not currently indexed but should be (newly included)
-        // 2. It's indexed but has been modified since last index
-        if (!indexedFilePaths.has(file.path) || file.stat.mtime > latestMtime) {
-          filesToIndex.add(file);
-        }
+      if (shouldBeIndexed && (!isIndexed || file.stat.mtime > latestMtime)) {
+        filesToIndex.add(file);
       }
     }
 
     if (getSettings().debug) {
       console.log(`Files to index: ${filesToIndex.size}`);
       console.log(`Previously indexed: ${indexedFilePaths.size}`);
+      console.log(`Empty files skipped: ${emptyFiles.size}`);
     }
 
     return Array.from(filesToIndex);

src/settings/components/QASettings.tsx

@@ -139,7 +139,24 @@ const QASettings: React.FC<QASettingsProps> = ({ vectorStoreManager }) => {
         description="Number of partitions for Copilot index. Default is 1. Increase if you have issues indexing large vaults. Warning: Changes require clearing and rebuilding the index!"
         value={settings.numPartitions.toString()}
         onChange={handlePartitionsChange}
-        options={["1", "2", "3", "4", "5", "6", "7", "8"]}
+        options={[
+          "1",
+          "2",
+          "3",
+          "4",
+          "5",
+          "6",
+          "7",
+          "8",
+          "12",
+          "16",
+          "20",
+          "24",
+          "28",
+          "32",
+          "36",
+          "40",
+        ]}
       />
       <TextAreaComponent
         name="Exclusions"