diff --git a/clients/tabby-agent/src/codeCompletion/contexts.ts b/clients/tabby-agent/src/codeCompletion/contexts.ts index 01255b5f9bfc..751d9b04f554 100644 --- a/clients/tabby-agent/src/codeCompletion/contexts.ts +++ b/clients/tabby-agent/src/codeCompletion/contexts.ts @@ -22,6 +22,7 @@ export type CompletionRequest = { }; declarations?: Declaration[]; relevantSnippetsFromChangedFiles?: CodeSnippet[]; + relevantSnippetsFromOpenedFiles?: CodeSnippet[]; }; export type Declaration = { @@ -68,7 +69,7 @@ export class CompletionContext { declarations?: Declaration[]; relevantSnippetsFromChangedFiles?: CodeSnippet[]; - + snippetsFromOpenedFiles?: CodeSnippet[]; // "default": the cursor is at the end of the line // "fill-in-line": the cursor is not at the end of the line, except auto closed characters // In this case, we assume the completion should be a single line, so multiple lines completion will be dropped. @@ -96,6 +97,7 @@ export class CompletionContext { this.declarations = request.declarations; this.relevantSnippetsFromChangedFiles = request.relevantSnippetsFromChangedFiles; + this.snippetsFromOpenedFiles = request.relevantSnippetsFromOpenedFiles; const lineEnd = isAtLineEndExcludingAutoClosedChar(this.currentLineSuffix); this.mode = lineEnd ? "default" : "fill-in-line"; @@ -203,6 +205,17 @@ export class CompletionContext { }) .sort((a, b) => b.score - a.score); + //FIXME(Sma1lboy): deduplicate in next few PR + const snippetsOpenedFiles = this.snippetsFromOpenedFiles + ?.map((snippet) => { + return { + filepath: snippet.filepath, + body: snippet.text, + score: snippet.score, + }; + }) + .sort((a, b) => b.score - a.score); + // clipboard let clipboard = undefined; if (this.clipboard.length >= config.clipboard.minChars && this.clipboard.length <= config.clipboard.maxChars) { @@ -215,6 +228,7 @@ export class CompletionContext { git_url: gitUrl, declarations, relevant_snippets_from_changed_files: relevantSnippetsFromChangedFiles, + relevant_snippets_from_recently_opened_files: snippetsOpenedFiles, clipboard, }; } diff --git a/clients/tabby-agent/src/codeCompletion/index.ts b/clients/tabby-agent/src/codeCompletion/index.ts index 238738076b4d..3c203ec913ff 100644 --- a/clients/tabby-agent/src/codeCompletion/index.ts +++ b/clients/tabby-agent/src/codeCompletion/index.ts @@ -57,6 +57,7 @@ import { abortSignalFromAnyOf } from "../utils/signal"; import { splitLines, extractNonReservedWordList } from "../utils/string"; import { MutexAbortError, isCanceledError } from "../utils/error"; import { isPositionInRange, intersectionRange } from "../utils/range"; +import { FileTracker } from "../codeSearch/fileTracker"; export class CompletionProvider implements Feature { private readonly logger = getLogger("CompletionProvider"); @@ -80,6 +81,7 @@ export class CompletionProvider implements Feature { private readonly anonymousUsageLogger: AnonymousUsageLogger, private readonly gitContextProvider: GitContextProvider, private readonly recentlyChangedCodeSearch: RecentlyChangedCodeSearch, + private readonly fileTracker: FileTracker, ) {} initialize(connection: Connection, clientCapabilities: ClientCapabilities): ServerCapabilities { @@ -565,7 +567,7 @@ export class CompletionProvider implements Feature { request.declarations = await this.collectDeclarationSnippets(connection, document, position, token); } request.relevantSnippetsFromChangedFiles = await this.collectSnippetsFromRecentlyChangedFiles(document, position); - + request.relevantSnippetsFromOpenedFiles = await this.collectSnippetsFromOpenedFiles(); this.logger.trace("Completed completion context:", { request }); return { request, additionalPrefixLength: additionalContext?.prefix.length }; } @@ -838,6 +840,82 @@ export class CompletionProvider implements Feature { return snippets; } + //get all recently opened files from the file tracker + private async collectSnippetsFromOpenedFiles(): Promise< + { filepath: string; offset: number; text: string; score: number }[] | undefined + > { + const config = this.configurations.getMergedConfig(); + if (!config.completion.prompt.collectSnippetsFromRecentOpenedFiles.enabled) { + return undefined; + } + this.logger.debug("Starting collecting snippets from opened files."); + const recentlyOpenedFiles = this.fileTracker.getAllFilesWithoutActive(); + const codeSnippets: { filepath: string; offset: number; text: string; score: number }[] = []; + const chunkSize = config.completion.prompt.collectSnippetsFromRecentOpenedFiles.maxCharsPerOpenedFiles; + recentlyOpenedFiles.forEach((file) => { + const doc = this.documents.get(file.uri); + if (doc) { + file.lastVisibleRange.forEach((range: Range) => { + this.logger.info( + `Original range: start(${range.start.line},${range.start.character}), end(${range.end.line},${range.end.character})`, + ); + + const startOffset = doc.offsetAt(range.start); + const endOffset = doc.offsetAt(range.end); + const middleOffset = Math.floor((startOffset + endOffset) / 2); + const halfChunkSize = Math.floor(chunkSize / 2); + + const upwardChunkSize = Math.min(halfChunkSize, middleOffset); + const newStartOffset = middleOffset - upwardChunkSize; + + const downwardChunkSize = Math.min(chunkSize - upwardChunkSize, doc.getText().length - middleOffset); + let newEndOffset = middleOffset + downwardChunkSize; + + if (newEndOffset - newStartOffset > chunkSize) { + const excess = newEndOffset - newStartOffset - chunkSize; + newEndOffset -= excess; + } + + let newStart = doc.positionAt(newStartOffset); + let newEnd = doc.positionAt(newEndOffset); + + newStart = { line: newStart.line, character: 0 }; + newEnd = { + line: newEnd.line, + character: doc.getText({ + start: { line: newEnd.line, character: 0 }, + end: { line: newEnd.line + 1, character: 0 }, + }).length, + }; + + this.logger.info( + `New range: start(${newStart.line},${newStart.character}), end(${newEnd.line},${newEnd.character})`, + ); + + const newRange = { start: newStart, end: newEnd }; + let text = doc.getText(newRange); + + if (text.length > chunkSize) { + text = text.substring(0, chunkSize); + } + + this.logger.info(`Text length: ${text.length}`); + this.logger.info(`Upward chunk size: ${upwardChunkSize}, Downward chunk size: ${downwardChunkSize}`); + + codeSnippets.push({ + filepath: file.uri, + offset: newStartOffset, + text: text, + score: file.invisible ? 0.98 : 1, + }); + }); + } + }); + + this.logger.debug("Completed collecting snippets from opened files."); + return codeSnippets; + } + private async submitStats() { const stats = this.completionStats.stats(); if (stats["completion_request"]["count"] > 0) { diff --git a/clients/tabby-agent/src/codeSearch/fileTracker.ts b/clients/tabby-agent/src/codeSearch/fileTracker.ts new file mode 100644 index 000000000000..0d2a5248315c --- /dev/null +++ b/clients/tabby-agent/src/codeSearch/fileTracker.ts @@ -0,0 +1,118 @@ +import { Connection, Range } from "vscode-languageserver"; +import { Feature } from "../feature"; +import { DidChangeActiveEditorNotification, DidChangeActiveEditorParams, ServerCapabilities } from "../protocol"; +import { Configurations } from "../config"; +import { LRUCache } from "lru-cache"; +import { isRangeEqual } from "../utils/range"; + +interface OpenedFile { + uri: string; + //order by range, the left most is the most recent one + lastVisibleRange: Range[]; + invisible: boolean; + isActive: boolean; +} + +export class FileTracker implements Feature { + private fileList = new LRUCache({ + max: this.configurations.getMergedConfig().completion.prompt.collectSnippetsFromRecentOpenedFiles.maxOpenedFiles, + }); + + constructor(private readonly configurations: Configurations) {} + initialize(connection: Connection): ServerCapabilities | Promise { + connection.onNotification(DidChangeActiveEditorNotification.type, (param: DidChangeActiveEditorParams) => { + this.resolveChangedFile(param); + }); + return {}; + } + + resolveChangedFile(param: DidChangeActiveEditorParams) { + const { activeEditor, visibleEditors } = param; + + const visitedPaths = new Set(); + + //get all visible editors + if (visibleEditors) { + visibleEditors.forEach((editor) => { + const visibleFile = this.fileList.get(editor.uri); + if (visibleFile) { + visibleFile.lastVisibleRange = []; + } + }); + + visibleEditors.forEach((editor) => { + let visibleFile = this.fileList.get(editor.uri); + if (!visibleFile) { + visibleFile = { + uri: editor.uri, + lastVisibleRange: [editor.range], + invisible: false, + isActive: false, + }; + this.fileList.set(editor.uri, visibleFile); + } else { + if (visitedPaths.has(visibleFile.uri)) { + const idx = visibleFile.lastVisibleRange.findIndex((range) => isRangeEqual(range, editor.range)); + if (idx === -1) { + visibleFile.lastVisibleRange = [editor.range, ...visibleFile.lastVisibleRange]; + } + visibleFile.invisible = false; + } else { + visibleFile.invisible = false; + visibleFile.lastVisibleRange = [editor.range]; + } + } + visitedPaths.add(visibleFile.uri); + }); + } + + // //get active editor + let file = this.fileList.get(activeEditor.uri); + if (!file) { + file = { + uri: activeEditor.uri, + lastVisibleRange: [activeEditor.range], + invisible: false, + isActive: true, + }; + this.fileList.set(activeEditor.uri, file); + } else { + if (visitedPaths.has(file.uri)) { + const idx = file.lastVisibleRange.findIndex((range) => isRangeEqual(range, activeEditor.range)); + if (idx === -1) { + file.lastVisibleRange = [activeEditor.range, ...file.lastVisibleRange]; + } + } else { + file.lastVisibleRange = [activeEditor.range]; + } + file.invisible = false; + file.isActive = true; + } + visitedPaths.add(file.uri); + + //set invisible flag for all files that are not in the current file list + Array.from(this.fileList.values()) + .filter(this.isOpenedFile) + .forEach((file) => { + if (!visitedPaths.has(file.uri)) { + file.invisible = true; + } + if (file.uri !== activeEditor.uri) { + file.isActive = false; + } + }); + } + private isOpenedFile(file: unknown): file is OpenedFile { + return (file as OpenedFile).uri !== undefined; + } + + /** + * Return All recently opened files by order. [recently opened, ..., oldest] without active file + * @returns return all recently opened files by order + */ + getAllFilesWithoutActive(): OpenedFile[] { + return Array.from(this.fileList.values()) + .filter(this.isOpenedFile) + .filter((f) => !f.isActive); + } +} diff --git a/clients/tabby-agent/src/config/default.ts b/clients/tabby-agent/src/config/default.ts index 885cbd9e5a95..0f17b22ea83e 100644 --- a/clients/tabby-agent/src/config/default.ts +++ b/clients/tabby-agent/src/config/default.ts @@ -38,6 +38,11 @@ export const defaultConfigData: ConfigData = { overlapLines: 1, }, }, + collectSnippetsFromRecentOpenedFiles: { + enabled: true, + maxOpenedFiles: 5, + maxCharsPerOpenedFiles: 500, + }, clipboard: { minChars: 3, maxChars: 2000, diff --git a/clients/tabby-agent/src/config/type.d.ts b/clients/tabby-agent/src/config/type.d.ts index 0526182a9e75..7bf1712df0fa 100644 --- a/clients/tabby-agent/src/config/type.d.ts +++ b/clients/tabby-agent/src/config/type.d.ts @@ -44,6 +44,13 @@ export type ConfigData = { overlapLines: number; }; }; + collectSnippetsFromRecentOpenedFiles: { + enabled: boolean; + //max number of opened files + maxOpenedFiles: number; + //chars size per each opened file + maxCharsPerOpenedFiles: number; + }; clipboard: { minChars: number; maxChars: number; diff --git a/clients/tabby-agent/src/protocol.ts b/clients/tabby-agent/src/protocol.ts index 8a56ba261b02..2bebfa3f61fa 100644 --- a/clients/tabby-agent/src/protocol.ts +++ b/clients/tabby-agent/src/protocol.ts @@ -526,6 +526,26 @@ export type ChatEditResolveCommand = LspCommand & { arguments: [ChatEditResolveParams]; }; +/** + * [Tabby] Did Change Active Editor Notification(➡️) + * + * This method is sent from the client to server when the active editor changed. + * + * + * - method: `tabby/editors/didChangeActiveEditor` + * - params: {@link OpenedFileParams} + * - result: void + */ +export namespace DidChangeActiveEditorNotification { + export const method = "tabby/editors/didChangeActiveEditor"; + export const messageDirection = MessageDirection.clientToServer; + export const type = new ProtocolNotificationType(method); +} +export type DidChangeActiveEditorParams = { + activeEditor: Location; + visibleEditors: Location[] | undefined; +}; + /** * [Tabby] GenerateCommitMessage Request(↩️) * diff --git a/clients/tabby-agent/src/server.ts b/clients/tabby-agent/src/server.ts index b6b5be713d41..4410938b3e0c 100644 --- a/clients/tabby-agent/src/server.ts +++ b/clients/tabby-agent/src/server.ts @@ -47,6 +47,7 @@ import { StatusProvider } from "./status"; import { CommandProvider } from "./command"; import { name as serverName, version as serverVersion } from "../package.json"; import "./utils/array"; +import { FileTracker } from "./codeSearch/fileTracker"; export class Server { private readonly logger = getLogger("TabbyLSP"); @@ -66,6 +67,7 @@ export class Server { private readonly gitContextProvider = new GitContextProvider(); private readonly recentlyChangedCodeSearch = new RecentlyChangedCodeSearch(this.configurations, this.documents); + private readonly fileTracker = new FileTracker(this.configurations); private readonly codeLensProvider = new CodeLensProvider(this.documents); private readonly completionProvider = new CompletionProvider( @@ -76,6 +78,7 @@ export class Server { this.anonymousUsageLogger, this.gitContextProvider, this.recentlyChangedCodeSearch, + this.fileTracker, ); private readonly chatFeature = new ChatFeature(this.tabbyApiClient); private readonly chatEditProvider = new ChatEditProvider(this.configurations, this.tabbyApiClient, this.documents); @@ -188,6 +191,7 @@ export class Server { this.commitMessageGenerator, this.statusProvider, this.commandProvider, + this.fileTracker, ].mapAsync((feature: Feature) => { return feature.initialize(this.connection, clientCapabilities, clientProvidedConfig); }); diff --git a/clients/vscode/src/InlineCompletionProvider.ts b/clients/vscode/src/InlineCompletionProvider.ts index eefd6bce1e6a..9b2bd7ecf259 100644 --- a/clients/vscode/src/InlineCompletionProvider.ts +++ b/clients/vscode/src/InlineCompletionProvider.ts @@ -96,6 +96,7 @@ export class InlineCompletionProvider extends EventEmitter implements InlineComp }; let request: Promise | undefined = undefined; try { + this.client.fileTrack.addingChangeEditor(window.activeTextEditor); request = this.client.languageClient.sendRequest(InlineCompletionRequest.method, params, token); this.ongoing = request; this.emit("didChangeLoading", true); diff --git a/clients/vscode/src/lsp/Client.ts b/clients/vscode/src/lsp/Client.ts index 6e44e6de54b9..d22f60b7912a 100644 --- a/clients/vscode/src/lsp/Client.ts +++ b/clients/vscode/src/lsp/Client.ts @@ -17,12 +17,14 @@ import { Config } from "../Config"; import { InlineCompletionProvider } from "../InlineCompletionProvider"; import { GitProvider } from "../git/GitProvider"; import { getLogger } from "../logger"; +import { FileTrackerFeature } from "./FileTrackFeature"; export class Client { private readonly logger = getLogger(""); readonly agent: AgentFeature; readonly chat: ChatFeature; readonly telemetry: TelemetryFeature; + readonly fileTrack: FileTrackerFeature; constructor( private readonly context: ExtensionContext, readonly languageClient: BaseLanguageClient, @@ -30,9 +32,11 @@ export class Client { this.agent = new AgentFeature(this.languageClient); this.chat = new ChatFeature(this.languageClient); this.telemetry = new TelemetryFeature(this.languageClient); + this.fileTrack = new FileTrackerFeature(this, this.context); this.languageClient.registerFeature(this.agent); this.languageClient.registerFeature(this.chat); this.languageClient.registerFeature(this.telemetry); + this.languageClient.registerFeature(this.fileTrack); this.languageClient.registerFeature(new DataStoreFeature(this.context, this.languageClient)); this.languageClient.registerFeature(new EditorOptionsFeature(this.languageClient)); this.languageClient.registerFeature(new LanguageSupportFeature(this.languageClient)); diff --git a/clients/vscode/src/lsp/FileTrackFeature.ts b/clients/vscode/src/lsp/FileTrackFeature.ts new file mode 100644 index 000000000000..a376d1c446d2 --- /dev/null +++ b/clients/vscode/src/lsp/FileTrackFeature.ts @@ -0,0 +1,59 @@ +import { DidChangeActiveEditorNotification, DidChangeActiveEditorParams } from "tabby-agent"; +import { Client } from "./Client"; +import { ExtensionContext, TextEditor, window } from "vscode"; +import { + DocumentSelector, + FeatureState, + InitializeParams, + ServerCapabilities, + StaticFeature, +} from "vscode-languageclient"; +import EventEmitter from "events"; +import { collectVisibleEditors } from "../windowUtils"; + +export class FileTrackerFeature extends EventEmitter implements StaticFeature { + constructor( + private readonly client: Client, + private readonly context: ExtensionContext, + ) { + super(); + } + fillInitializeParams?: ((params: InitializeParams) => void) | undefined; + fillClientCapabilities(): void { + //nothing + } + preInitialize?: + | ((capabilities: ServerCapabilities, documentSelector: DocumentSelector | undefined) => void) + | undefined; + initialize(): void { + this.context.subscriptions.push( + //when active text editor changes + window.onDidChangeActiveTextEditor(async (editor) => { + await this.addingChangeEditor(editor); + }), + ); + } + getState(): FeatureState { + throw new Error("Method not implemented."); + } + clear(): void { + throw new Error("Method not implemented."); + } + + async addingChangeEditor(editor: TextEditor | undefined) { + if (editor && editor.visibleRanges[0] && editor.document.fileName.startsWith("/")) { + const editorRange = editor.visibleRanges[0]; + const params: DidChangeActiveEditorParams = { + activeEditor: { + uri: editor.document.uri.toString(), + range: { + start: { line: editorRange.start.line, character: editorRange.start.character }, + end: { line: editorRange.end.line, character: editorRange.end.character }, + }, + }, + visibleEditors: collectVisibleEditors(true, editor), + }; + await this.client.languageClient.sendNotification(DidChangeActiveEditorNotification.method, params); + } + } +} diff --git a/clients/vscode/src/windowUtils.ts b/clients/vscode/src/windowUtils.ts new file mode 100644 index 000000000000..72721fc30706 --- /dev/null +++ b/clients/vscode/src/windowUtils.ts @@ -0,0 +1,67 @@ +import { TextEditor, window } from "vscode"; +import { Location } from "vscode-languageclient"; + +export function collectVisibleEditors(exceptActiveEditor = false, activeEditor?: TextEditor): Location[] { + let editors = window.visibleTextEditors + .filter((e) => e.document.fileName.startsWith("/")) + .map((editor) => { + if (!editor.visibleRanges[0]) { + return null; + } + return { + uri: editor.document.uri.toString(), + range: { + start: { + line: editor.visibleRanges[0].start.line, + character: editor.visibleRanges[0].start.character, + }, + end: { + line: editor.visibleRanges[0].end.line, + character: editor.visibleRanges[0].end.character, + }, + }, + } as Location; + }) + .filter((e): e is Location => e !== null) + .sort((a, b) => + a.uri === window.activeTextEditor?.document.uri.toString() + ? -1 + : b.uri === window.activeTextEditor?.document.uri.toString() + ? 1 + : 0, + ); + if (exceptActiveEditor) { + if (activeEditor && activeEditor.visibleRanges[0]) { + const range = activeEditor.visibleRanges[0]; + editors = editors.filter( + (e) => + e.uri !== activeEditor.document.uri.toString() || + e.range.start.line !== range.start.line || + e.range.start.character !== range.start.character || + e.range.end.line !== range.end.line || + e.range.end.character !== range.end.character, + ); + } + } + return editors; +} +export function collectActiveEditor(): Location | undefined { + const activeEditor = window.activeTextEditor; + //only return TextDocument editor + if (!activeEditor || !activeEditor.visibleRanges[0] || !activeEditor.document.fileName.startsWith("/")) { + return undefined; + } + return { + uri: activeEditor.document.uri.toString(), + range: { + start: { + line: activeEditor.visibleRanges[0].start.line, + character: activeEditor.visibleRanges[0].start.character, + }, + end: { + line: activeEditor.visibleRanges[0].end.line, + character: activeEditor.visibleRanges[0].end.character, + }, + }, + }; +}