Merge pull request #6963 from continuedev/jacob/con-3141

feat: cache model responses for near-instant suggestions

Author: tomasz-stefaniak

core/core.ts

@@ -580,7 +580,7 @@ export class Core {
       const outcome = await this.nextEditProvider.provideInlineCompletionItems(
         msg.data,
         undefined,
-        { withChain: false },
+        { withChain: false, usingFullFileDiff: true },
       );
       return outcome ? [outcome.completion, outcome.originalEditableRange] : [];
     });

core/nextEdit/NextEditEditableRegionCalculator.ts

@@ -2,9 +2,14 @@ import Parser from "web-tree-sitter";
 import { Chunk, IDE, ILLM, Position, Range, RangeInFile } from "..";
 import { getAst } from "../autocomplete/util/ast";
 import { DocumentHistoryTracker } from "./DocumentHistoryTracker";
+import {
+  NEXT_EDIT_EDITABLE_REGION_BOTTOM_MARGIN,
+  NEXT_EDIT_EDITABLE_REGION_TOP_MARGIN,
+} from "./constants";
 
 export enum EditableRegionStrategy {
   Naive = "naive",
+  Sliding = "sliding",
   Rerank = "rerank",
   StaticRerank = "staticRerank",
   Static = "static",
@@ -17,6 +22,8 @@ export async function getNextEditableRegion(
   switch (strategy) {
     case EditableRegionStrategy.Naive:
       return naiveJump(ctx);
+    case EditableRegionStrategy.Sliding:
+      return slidingJump(ctx);
     case EditableRegionStrategy.Rerank:
       return await rerankJump(ctx);
     case EditableRegionStrategy.StaticRerank:
@@ -51,6 +58,63 @@ function naiveJump(ctx: any): RangeInFile[] | null {
   ];
 }
 
+// Sliding splits the file using into sliding window.
+function slidingJump(ctx: any): RangeInFile[] | null {
+  const { fileLines, filepath } = ctx;
+  if (!fileLines || !filepath) {
+    console.warn("Missing required context for naive jump");
+    return null;
+  }
+
+  const windowSize =
+    NEXT_EDIT_EDITABLE_REGION_TOP_MARGIN +
+    NEXT_EDIT_EDITABLE_REGION_BOTTOM_MARGIN +
+    1; // 1 for current line;
+
+  if (fileLines.length <= windowSize) {
+    return [
+      {
+        filepath,
+        range: {
+          start: { line: 0, character: 0 },
+          end: {
+            line: fileLines.length - 1,
+            character: fileLines[fileLines.length - 1].length,
+          },
+        },
+      },
+    ];
+  }
+
+  const slidingStep = Math.max(1, Math.floor(windowSize / 2));
+  const ranges: RangeInFile[] = [];
+
+  for (
+    let startLine = 0;
+    startLine < fileLines.length;
+    startLine += slidingStep
+  ) {
+    const endLine = Math.min(startLine + windowSize - 1, fileLines.length - 1);
+
+    ranges.push({
+      filepath,
+      range: {
+        start: { line: startLine, character: 0 },
+        end: {
+          line: endLine,
+          character: fileLines[endLine].length,
+        },
+      },
+    });
+
+    if (endLine >= fileLines.length - 1) {
+      break;
+    }
+  }
+
+  return ranges;
+}
+
 // A rerank jump splits the current file into chunks.
 // Then it uses a rerank model to get the most relevant chunks and their positions.
 async function rerankJump(ctx: {

core/nextEdit/NextEditPrefetchQueue.ts

@@ -0,0 +1,121 @@
+import { RangeInFile } from "..";
+import { NextEditProvider } from "./NextEditProvider";
+import { NextEditOutcome } from "./types";
+
+interface ProcessedItem {
+  location: RangeInFile;
+  outcome: NextEditOutcome; // Result from the model
+}
+
+export class PrefetchQueue {
+  private static instance: PrefetchQueue | null = null;
+
+  private unprocessedQueue: RangeInFile[] = [];
+  private processedQueue: ProcessedItem[] = [];
+  private prefetchLimit: number;
+  private abortController: AbortController;
+
+  private usingFullFileDiff: boolean = true;
+
+  private constructor(prefetchLimit: number = 3) {
+    this.prefetchLimit = prefetchLimit;
+    this.abortController = new AbortController();
+  }
+
+  public static getInstance(prefetchLimit: number = 3): PrefetchQueue {
+    if (!PrefetchQueue.instance) {
+      PrefetchQueue.instance = new PrefetchQueue(prefetchLimit);
+    }
+
+    return PrefetchQueue.instance;
+  }
+
+  initialize(usingFullFileDiff: boolean) {
+    this.usingFullFileDiff = usingFullFileDiff;
+  }
+
+  // Queue management methods
+  enqueueUnprocessed(location: RangeInFile): void {
+    this.unprocessedQueue.push(location);
+  }
+
+  private dequeueUnprocessed(): RangeInFile | undefined {
+    return this.unprocessedQueue.shift();
+  }
+
+  enqueueProcessed(item: ProcessedItem): void {
+    this.processedQueue.push(item);
+  }
+
+  dequeueProcessed(): ProcessedItem | undefined {
+    return this.processedQueue.shift();
+  }
+
+  // Process items from unprocessed queue
+  async process(ctx: any): Promise<void> {
+    while (
+      this.unprocessedQueue.length > 0 &&
+      this.processedQueue.length < this.prefetchLimit &&
+      !this.abortController.signal.aborted
+    ) {
+      const location = this.dequeueUnprocessed();
+      if (!location) break;
+
+      try {
+        const outcome =
+          await NextEditProvider.getInstance().provideInlineCompletionItemsWithChain(
+            ctx,
+            location,
+            this.abortController.signal,
+            this.usingFullFileDiff,
+          );
+
+        if (!outcome) continue;
+
+        this.enqueueProcessed({
+          location,
+          outcome,
+        });
+      } catch (error) {
+        if (!this.abortController.signal.aborted) {
+          // Handle error
+          console.error("Error processing item:", error);
+        }
+        // If aborted, we just stop processing
+        break;
+      }
+    }
+  }
+
+  // Abort all operations
+  abort(): void {
+    this.abortController.abort();
+    this.clear();
+
+    // Create a new AbortController for future operations
+    this.abortController = new AbortController();
+  }
+
+  // Clear all queues
+  clear(): void {
+    this.unprocessedQueue = [];
+    this.processedQueue = [];
+  }
+
+  // Additional helper methods
+  get unprocessedCount(): number {
+    return this.unprocessedQueue.length;
+  }
+
+  get processedCount(): number {
+    return this.processedQueue.length;
+  }
+
+  peekProcessed(): ProcessedItem | undefined {
+    return this.processedQueue[0];
+  }
+
+  setPreetchLimit(limit: number): void {
+    this.prefetchLimit = limit;
+  }
+}