feat(tabby-agent): add recent opened files frontend to code completion requests

clients/tabby-agent/src/codeCompletion/contexts.ts

@@ -22,6 +22,7 @@ export type CompletionRequest = {
   };
   declarations?: Declaration[];
   relevantSnippetsFromChangedFiles?: CodeSnippet[];
+  relevantSnippetsFromOpenedFiles?: CodeSnippet[];
 };
 
 export type Declaration = {
@@ -68,7 +69,7 @@ export class CompletionContext {
 
   declarations?: Declaration[];
   relevantSnippetsFromChangedFiles?: CodeSnippet[];
-
+  snippetsFromOpenedFiles?: CodeSnippet[];
   // "default": the cursor is at the end of the line
   // "fill-in-line": the cursor is not at the end of the line, except auto closed characters
   //   In this case, we assume the completion should be a single line, so multiple lines completion will be dropped.
@@ -96,6 +97,7 @@ export class CompletionContext {
 
     this.declarations = request.declarations;
     this.relevantSnippetsFromChangedFiles = request.relevantSnippetsFromChangedFiles;
+    this.snippetsFromOpenedFiles = request.relevantSnippetsFromOpenedFiles;
 
     const lineEnd = isAtLineEndExcludingAutoClosedChar(this.currentLineSuffix);
     this.mode = lineEnd ? "default" : "fill-in-line";
@@ -203,6 +205,17 @@ export class CompletionContext {
       })
       .sort((a, b) => b.score - a.score);
 
+    //FIXME(Sma1lboy): deduplicate in next few PR
+    const snippetsOpenedFiles = this.snippetsFromOpenedFiles
+      ?.map((snippet) => {
+        return {
+          filepath: snippet.filepath,
+          body: snippet.text,
+          score: snippet.score,
+        };
+      })
+      .sort((a, b) => b.score - a.score);
+
     // clipboard
     let clipboard = undefined;
     if (this.clipboard.length >= config.clipboard.minChars && this.clipboard.length <= config.clipboard.maxChars) {
@@ -215,6 +228,7 @@ export class CompletionContext {
       git_url: gitUrl,
       declarations,
       relevant_snippets_from_changed_files: relevantSnippetsFromChangedFiles,
+      relevant_snippets_from_recently_opened_files: snippetsOpenedFiles,
       clipboard,
     };
   }

clients/tabby-agent/src/codeCompletion/index.ts

@@ -57,6 +57,7 @@ import { abortSignalFromAnyOf } from "../utils/signal";
 import { splitLines, extractNonReservedWordList } from "../utils/string";
 import { MutexAbortError, isCanceledError } from "../utils/error";
 import { isPositionInRange, intersectionRange } from "../utils/range";
+import { FileTracker } from "../codeSearch/fileTracker";
 
 export class CompletionProvider implements Feature {
   private readonly logger = getLogger("CompletionProvider");
@@ -80,6 +81,7 @@ export class CompletionProvider implements Feature {
     private readonly anonymousUsageLogger: AnonymousUsageLogger,
     private readonly gitContextProvider: GitContextProvider,
     private readonly recentlyChangedCodeSearch: RecentlyChangedCodeSearch,
+    private readonly fileTracker: FileTracker,
   ) {}
 
   initialize(connection: Connection, clientCapabilities: ClientCapabilities): ServerCapabilities {
@@ -565,7 +567,7 @@ export class CompletionProvider implements Feature {
       request.declarations = await this.collectDeclarationSnippets(connection, document, position, token);
     }
     request.relevantSnippetsFromChangedFiles = await this.collectSnippetsFromRecentlyChangedFiles(document, position);
-
+    request.relevantSnippetsFromOpenedFiles = await this.collectSnippetsFromOpenedFiles();
     this.logger.trace("Completed completion context:", { request });
     return { request, additionalPrefixLength: additionalContext?.prefix.length };
   }
@@ -838,6 +840,82 @@ export class CompletionProvider implements Feature {
     return snippets;
   }
 
+  //get all recently opened files from the file tracker
+  private async collectSnippetsFromOpenedFiles(): Promise<
+    { filepath: string; offset: number; text: string; score: number }[] | undefined
+  > {
+    const config = this.configurations.getMergedConfig();
+    if (!config.completion.prompt.collectSnippetsFromRecentOpenedFiles.enabled) {
+      return undefined;
+    }
+    this.logger.debug("Starting collecting snippets from opened files.");
+    const recentlyOpenedFiles = this.fileTracker.getAllFilesWithoutActive();
+    const codeSnippets: { filepath: string; offset: number; text: string; score: number }[] = [];
+    const chunkSize = config.completion.prompt.collectSnippetsFromRecentOpenedFiles.maxCharsPerOpenedFiles;
+    recentlyOpenedFiles.forEach((file) => {
+      const doc = this.documents.get(file.uri);
+      if (doc) {
+        file.lastVisibleRange.forEach((range: Range) => {
+          this.logger.info(
+            `Original range: start(${range.start.line},${range.start.character}), end(${range.end.line},${range.end.character})`,
+          );
+
+          const startOffset = doc.offsetAt(range.start);
+          const endOffset = doc.offsetAt(range.end);
+          const middleOffset = Math.floor((startOffset + endOffset) / 2);
+          const halfChunkSize = Math.floor(chunkSize / 2);
+
+          const upwardChunkSize = Math.min(halfChunkSize, middleOffset);
+          const newStartOffset = middleOffset - upwardChunkSize;
+
+          const downwardChunkSize = Math.min(chunkSize - upwardChunkSize, doc.getText().length - middleOffset);
+          let newEndOffset = middleOffset + downwardChunkSize;
+
+          if (newEndOffset - newStartOffset > chunkSize) {
+            const excess = newEndOffset - newStartOffset - chunkSize;
+            newEndOffset -= excess;
+          }
+
+          let newStart = doc.positionAt(newStartOffset);
+          let newEnd = doc.positionAt(newEndOffset);
+
+          newStart = { line: newStart.line, character: 0 };
+          newEnd = {
+            line: newEnd.line,
+            character: doc.getText({
+              start: { line: newEnd.line, character: 0 },
+              end: { line: newEnd.line + 1, character: 0 },
+            }).length,
+          };
+
+          this.logger.info(
+            `New range: start(${newStart.line},${newStart.character}), end(${newEnd.line},${newEnd.character})`,
+          );
+
+          const newRange = { start: newStart, end: newEnd };
+          let text = doc.getText(newRange);
+
+          if (text.length > chunkSize) {
+            text = text.substring(0, chunkSize);
+          }
+
+          this.logger.info(`Text length: ${text.length}`);
+          this.logger.info(`Upward chunk size: ${upwardChunkSize}, Downward chunk size: ${downwardChunkSize}`);
+
+          codeSnippets.push({
+            filepath: file.uri,
+            offset: newStartOffset,
+            text: text,
+            score: file.invisible ? 0.98 : 1,
+          });
+        });
+      }
+    });
+
+    this.logger.debug("Completed collecting snippets from opened files.");
+    return codeSnippets;
+  }
+
   private async submitStats() {
     const stats = this.completionStats.stats();
     if (stats["completion_request"]["count"] > 0) {

clients/tabby-agent/src/codeSearch/fileTracker.ts

@@ -0,0 +1,118 @@
+import { Connection, Range } from "vscode-languageserver";
+import { Feature } from "../feature";
+import { DidChangeActiveEditorNotification, DidChangeActiveEditorParams, ServerCapabilities } from "../protocol";
+import { Configurations } from "../config";
+import { LRUCache } from "lru-cache";
+import { isRangeEqual } from "../utils/range";
+
+interface OpenedFile {
+  uri: string;
+  //order by range, the left most is the most recent one
+  lastVisibleRange: Range[];
+  invisible: boolean;
+  isActive: boolean;
+}
+
+export class FileTracker implements Feature {
+  private fileList = new LRUCache<string, OpenedFile>({
+    max: this.configurations.getMergedConfig().completion.prompt.collectSnippetsFromRecentOpenedFiles.maxOpenedFiles,
+  });
+
+  constructor(private readonly configurations: Configurations) {}
+  initialize(connection: Connection): ServerCapabilities | Promise<ServerCapabilities> {
+    connection.onNotification(DidChangeActiveEditorNotification.type, (param: DidChangeActiveEditorParams) => {
+      this.resolveChangedFile(param);
+    });
+    return {};
+  }
+
+  resolveChangedFile(param: DidChangeActiveEditorParams) {
+    const { activeEditor, visibleEditors } = param;
+
+    const visitedPaths = new Set<string>();
+
+    //get all visible editors
+    if (visibleEditors) {
+      visibleEditors.forEach((editor) => {
+        const visibleFile = this.fileList.get(editor.uri);
+        if (visibleFile) {
+          visibleFile.lastVisibleRange = [];
+        }
+      });
+
+      visibleEditors.forEach((editor) => {
+        let visibleFile = this.fileList.get(editor.uri);
+        if (!visibleFile) {
+          visibleFile = {
+            uri: editor.uri,
+            lastVisibleRange: [editor.range],
+            invisible: false,
+            isActive: false,
+          };
+          this.fileList.set(editor.uri, visibleFile);
+        } else {
+          if (visitedPaths.has(visibleFile.uri)) {
+            const idx = visibleFile.lastVisibleRange.findIndex((range) => isRangeEqual(range, editor.range));
+            if (idx === -1) {
+              visibleFile.lastVisibleRange = [editor.range, ...visibleFile.lastVisibleRange];
+            }
+            visibleFile.invisible = false;
+          } else {
+            visibleFile.invisible = false;
+            visibleFile.lastVisibleRange = [editor.range];
+          }
+        }
+        visitedPaths.add(visibleFile.uri);
+      });
+    }
+
+    // //get active editor
+    let file = this.fileList.get(activeEditor.uri);
+    if (!file) {
+      file = {
+        uri: activeEditor.uri,
+        lastVisibleRange: [activeEditor.range],
+        invisible: false,
+        isActive: true,
+      };
+      this.fileList.set(activeEditor.uri, file);
+    } else {
+      if (visitedPaths.has(file.uri)) {
+        const idx = file.lastVisibleRange.findIndex((range) => isRangeEqual(range, activeEditor.range));
+        if (idx === -1) {
+          file.lastVisibleRange = [activeEditor.range, ...file.lastVisibleRange];
+        }
+      } else {
+        file.lastVisibleRange = [activeEditor.range];
+      }
+      file.invisible = false;
+      file.isActive = true;
+    }
+    visitedPaths.add(file.uri);
+
+    //set invisible flag for all files that are not in the current file list
+    Array.from(this.fileList.values())
+      .filter(this.isOpenedFile)
+      .forEach((file) => {
+        if (!visitedPaths.has(file.uri)) {
+          file.invisible = true;
+        }
+        if (file.uri !== activeEditor.uri) {
+          file.isActive = false;
+        }
+      });
+  }
+  private isOpenedFile(file: unknown): file is OpenedFile {
+    return (file as OpenedFile).uri !== undefined;
+  }
+
+  /**
+   * Return All recently opened files by order. [recently opened, ..., oldest] without active file
+   * @returns return all recently opened files by order
+   */
+  getAllFilesWithoutActive(): OpenedFile[] {
+    return Array.from(this.fileList.values())
+      .filter(this.isOpenedFile)
+      .filter((f) => !f.isActive);
+  }
+}

clients/tabby-agent/src/config/default.ts

@@ -38,6 +38,11 @@ export const defaultConfigData: ConfigData = {
           overlapLines: 1,
         },
       },
+      collectSnippetsFromRecentOpenedFiles: {
+        enabled: true,
+        maxOpenedFiles: 5,
+        maxCharsPerOpenedFiles: 500,
+      },
       clipboard: {
         minChars: 3,
         maxChars: 2000,

clients/tabby-agent/src/config/type.d.ts

@@ -44,6 +44,13 @@ export type ConfigData = {
           overlapLines: number;
         };
       };
+      collectSnippetsFromRecentOpenedFiles: {
+        enabled: boolean;
+        //max number of opened files
+        maxOpenedFiles: number;
+        //chars size per each opened file
+        maxCharsPerOpenedFiles: number;
+      };
       clipboard: {
         minChars: number;
         maxChars: number;

clients/tabby-agent/src/protocol.ts

@@ -526,6 +526,26 @@ export type ChatEditResolveCommand = LspCommand & {
   arguments: [ChatEditResolveParams];
 };
 
+/**
+ * [Tabby] Did Change Active Editor Notification(➡️)
+ *
+ * This method is sent from the client to server when the active editor changed.
+ *
+ *
+ * - method: `tabby/editors/didChangeActiveEditor`
+ * - params: {@link OpenedFileParams}
+ * - result: void
+ */
+export namespace DidChangeActiveEditorNotification {
+  export const method = "tabby/editors/didChangeActiveEditor";
+  export const messageDirection = MessageDirection.clientToServer;
+  export const type = new ProtocolNotificationType<DidChangeActiveEditorParams, void>(method);
+}
+export type DidChangeActiveEditorParams = {
+  activeEditor: Location;
+  visibleEditors: Location[] | undefined;
+};
+
 /**
  * [Tabby] GenerateCommitMessage Request(↩️)
  *

clients/tabby-agent/src/server.ts

@@ -47,6 +47,7 @@ import { StatusProvider } from "./status";
 import { CommandProvider } from "./command";
 import { name as serverName, version as serverVersion } from "../package.json";
 import "./utils/array";
+import { FileTracker } from "./codeSearch/fileTracker";
 
 export class Server {
   private readonly logger = getLogger("TabbyLSP");
@@ -66,6 +67,7 @@ export class Server {
 
   private readonly gitContextProvider = new GitContextProvider();
   private readonly recentlyChangedCodeSearch = new RecentlyChangedCodeSearch(this.configurations, this.documents);
+  private readonly fileTracker = new FileTracker(this.configurations);
 
   private readonly codeLensProvider = new CodeLensProvider(this.documents);
   private readonly completionProvider = new CompletionProvider(
@@ -76,6 +78,7 @@ export class Server {
     this.anonymousUsageLogger,
     this.gitContextProvider,
     this.recentlyChangedCodeSearch,
+    this.fileTracker,
   );
   private readonly chatFeature = new ChatFeature(this.tabbyApiClient);
   private readonly chatEditProvider = new ChatEditProvider(this.configurations, this.tabbyApiClient, this.documents);
@@ -188,6 +191,7 @@ export class Server {
       this.commitMessageGenerator,
       this.statusProvider,
       this.commandProvider,
+      this.fileTracker,
     ].mapAsync((feature: Feature) => {
       return feature.initialize(this.connection, clientCapabilities, clientProvidedConfig);
     });

clients/tabby-openapi/lib/tabby.d.ts

@@ -257,6 +257,8 @@ export interface components {
              *
              *     Sorted in descending order of [Snippet::score]. */
             relevant_snippets_from_changed_files?: components["schemas"]["Snippet"][] | null;
+            /** @description The relevant code snippets extracted from recently opened files. These snippets are selected from candidates found within code chunks based on the last visited location. Current Active file is excluded from the search candidates. When provided with [Segments::relevant_snippets_from_changed_files], the snippets have already been deduplicated to ensure no duplication with entries in [Segments::relevant_snippets_from_changed_files]. */
+            relevant_snippets_from_recently_opened_files?: components["schemas"]["Snippet"][] | null;
             /** @description Clipboard content when requesting code completion. */
             clipboard?: string | null;
         };