From f1752876413251b957cc07f1c4768b7673953bdf Mon Sep 17 00:00:00 2001 From: Jackson Chen <541898146chen@gmail.com> Date: Fri, 15 Nov 2024 11:29:33 -0600 Subject: [PATCH] chore: remove duplicate suffix lines in postprocess filter --- .../src/codeCompletion/postprocess/index.ts | 4 +- .../removeDuplicateSuffixLines.test.ts | 246 ++++++++---------- .../postprocess/removeDuplicateSuffixLines.ts | 28 +- 3 files changed, 113 insertions(+), 165 deletions(-) diff --git a/clients/tabby-agent/src/codeCompletion/postprocess/index.ts b/clients/tabby-agent/src/codeCompletion/postprocess/index.ts index 43e7058f4e0d..0c7163962ce1 100644 --- a/clients/tabby-agent/src/codeCompletion/postprocess/index.ts +++ b/clients/tabby-agent/src/codeCompletion/postprocess/index.ts @@ -54,7 +54,7 @@ export async function postCacheProcess( .then(applyFilter(formatIndentation)) .then(applyFilter(dropDuplicated)) .then(applyFilter(trimSpace)) + .then(applyFilter(removeDuplicateSuffixLines)) .then(applyFilter(dropMinimum)) - .then(applyFilter(calculateReplaceRange)) - .then(applyFilter(removeDuplicateSuffixLines)); + .then(applyFilter(calculateReplaceRange)); } diff --git a/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts index 157c46c88d9a..f855eb17814e 100644 --- a/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts +++ b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts @@ -1,152 +1,120 @@ import { documentContext, inline, assertFilterResult } from "./testUtils"; import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines"; -describe("postprocess", () => { - describe("removeDuplicateSuffixLines", () => { - const filter = removeDuplicateSuffixLines(); +describe("removeDuplicateSuffixLines", () => { + const filter = removeDuplicateSuffixLines(); - it("should remove duplicated suffix lines", async () => { - const context = documentContext` - function example() { - const items = [ - ║ - ]; - } - `; - context.language = "javascript"; - const completion = inline` - ├1, - 2, - 3, - 4,┤ - `; - context.suffix = ` - 4, - 5, - 6 - ]; - } - `; - const expected = inline` - ├1, - 2, - 3,┤ - `; - await assertFilterResult(filter, context, completion, expected); - }); - - it("should handle empty suffix", async () => { - const context = documentContext` - const value = ║ - `; - context.language = "javascript"; - const completion = inline` - ├42;┤ - `; - context.suffix = ""; - const expected = completion; - await assertFilterResult(filter, context, completion, expected); - }); + it("should remove duplicated array item", async () => { + const context = documentContext` + const items = [ + ║ + { id: 2 }, + { id: 3 } + ]; + `; + const completion = inline` + ├{ id: 1 }, + { id: 2 },┤ + `; + const expected = inline` + ├{ id: 1 },┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); - it("should handle multiple line matches", async () => { - const context = documentContext` - class Example { - constructor() { - ║ - } - } - `; - context.language = "javascript"; - const completion = inline` - ├this.value = 1; - this.name = "test"; - this.items = []; - this.setup();┤ - `; - context.suffix = ` - this.setup(); - this.init(); - } - } - `; - const expected = inline` - ├this.value = 1; - this.name = "test"; - this.items = [];┤ - `; - await assertFilterResult(filter, context, completion, expected); - }); + it("should handle empty content after cursor", async () => { + const context = documentContext` + const a = ║ + `; + const completion = inline` + ├42;┤ + `; + const expected = completion; + await assertFilterResult(filter, context, completion, expected); + }); - it("should handle partial line matches without trimming", async () => { - const context = documentContext` - const config = { + it("should remove duplicated comma items", async () => { + const context = documentContext` + function example() { + const items = [ ║ - }; - `; - context.language = "javascript"; - const completion = inline` - ├name: "test", - value: 42, - items: [], - enabled: true,┤ - `; - context.suffix = ` - enabled: true, - debug: false - }; - `; - const expected = inline` - ├name: "test", - value: 42, - items: [],┤ - `; - await assertFilterResult(filter, context, completion, expected); - }); + 4, + 5, + 6 + ]; + } + `; + const completion = inline` + ├1, + 2, + 3, + 4,┤ + `; + const expected = inline` + ├1, + 2, + 3,┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); - it("should not modify when no matches found", async () => { - const context = documentContext` - function process() { + it("should remove duplicate method calls", async () => { + const context = documentContext` + class Example { + constructor() { ║ + this.setup(); + this.init() } - `; - context.language = "javascript"; - const completion = inline` - ├console.log("processing"); - return true;┤ - `; - context.suffix = ` - console.log("done"); - } - `; - const expected = completion; - await assertFilterResult(filter, context, completion, expected); - }); + } + `; + const completion = inline` + ├this.value = 1; + this.name = "test"; + this.items = []; + this.setup();┤ + `; + const expected = inline` + ├this.value = 1; + this.name = "test"; + this.items = [];┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); - it("should handle whitespace differences", async () => { - const context = documentContext` - const arr = [ - ║ - ]; - `; - context.language = "javascript"; - const completion = inline` - ├1, - 2, - 3, - 4,┤ - `; - context.suffix = ` - 4, - 5, - 6 - ]; - `; - const expected = inline` - ├1, - 2, - 3,┤ - `; - await assertFilterResult(filter, context, completion, expected); - }); + it("should remove duplicate object properties", async () => { + const context = documentContext` + const config = { + ║ + enabled: true, + debug: false + }; + `; + const completion = inline` + ├name: "test", + value: 42, + items: [], + enabled: true,┤ + `; + const expected = inline` + ├name: "test", + value: 42, + items: [],┤ + `; + await assertFilterResult(filter, context, completion, expected); + }); + + it("should keep content when no matches", async () => { + const context = documentContext` + function process() { + ║ + console.log("done"); + } + `; + const completion = inline` + ├console.log("processing");┤ + `; + const expected = completion; + await assertFilterResult(filter, context, completion, expected); }); }); diff --git a/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts index 37cac94d355c..0b13878f45f9 100644 --- a/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts +++ b/clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts @@ -1,13 +1,9 @@ import { PostprocessFilter } from "./base"; import { CompletionItem } from "../solution"; import { isBlank } from "../../utils/string"; -import { getLogger } from "../../logger"; export function removeDuplicateSuffixLines(): PostprocessFilter { return (item: CompletionItem): CompletionItem => { - const log = getLogger("removeDuplicateSuffixLines"); - log.info("Processing item" + JSON.stringify(item?.text || "")); - const text = item?.text; const suffix = item?.context?.suffix; @@ -16,7 +12,6 @@ export function removeDuplicateSuffixLines(): PostprocessFilter { } const originalLines = text.split("\n").map((line) => line || ""); - const trimmedLines = originalLines.map((line) => (line || "").trim()); const suffixLines = (suffix || "") .split("\n") @@ -30,36 +25,21 @@ export function removeDuplicateSuffixLines(): PostprocessFilter { const firstSuffixLine = suffixLines[0] || ""; // iterate through lines from end to find potential match - for (let i = trimmedLines.length - 1; i >= 0; i--) { - const currentLine = trimmedLines[i] || ""; + for (let i = originalLines.length - 1; i >= 0; i--) { + const currentLine = originalLines[i] || ""; if (!isBlank(currentLine) && currentLine === firstSuffixLine) { // check if subsequent lines also match with suffix let isFullMatch = true; - for (let j = 0; j < suffixLines.length && i + j < trimmedLines.length; j++) { + for (let j = 0; j < suffixLines.length && i + j < originalLines.length; j++) { const suffixLine = suffixLines[j] || ""; - const textLine = trimmedLines[i + j] || ""; - + const textLine = originalLines[i + j] || ""; if (suffixLine !== textLine) { isFullMatch = false; break; } } - - // if all checked lines match, check for code structure if (isFullMatch) { const remainingLines = originalLines.slice(0, i); - const lastLine = remainingLines[remainingLines.length - 1] || ""; - - // skip empty last lines - if (isBlank(lastLine.trim())) { - return item; - } - - // preserve code block structure - if (lastLine.includes("{") || currentLine.includes("}")) { - return item; - } - return item.withText(remainingLines.join("\n")); } }