feat: add removeDuplicateSuffixLines postprocess filter

clients/tabby-agent/src/codeCompletion/postprocess/index.ts

@@ -13,6 +13,7 @@ import { trimMultiLineInSingleLineMode } from "./trimMultiLineInSingleLineMode";
 import { dropDuplicated } from "./dropDuplicated";
 import { dropMinimum } from "./dropMinimum";
 import { calculateReplaceRange } from "./calculateReplaceRange";
+import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines";
 
 type ItemListFilter = (items: CompletionItem[]) => Promise<CompletionItem[]>;
 
@@ -54,5 +55,6 @@ export async function postCacheProcess(
     .then(applyFilter(dropDuplicated))
     .then(applyFilter(trimSpace))
     .then(applyFilter(dropMinimum))
-    .then(applyFilter(calculateReplaceRange));
+    .then(applyFilter(calculateReplaceRange))
+    .then(applyFilter(removeDuplicateSuffixLines));
 }

clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts

@@ -0,0 +1,152 @@
+import { documentContext, inline, assertFilterResult } from "./testUtils";
+import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines";
+
+describe("postprocess", () => {
+  describe("removeDuplicateSuffixLines", () => {
+    const filter = removeDuplicateSuffixLines();
+
+    it("should remove duplicated suffix lines", async () => {
+      const context = documentContext`
+        function example() {
+          const items = [
+            ║
+          ];
+        }
+      `;
+      context.language = "javascript";
+      const completion = inline`
+            ├1,
+            2,
+            3,
+            4,┤
+      `;
+      context.suffix = `
+            4,
+            5,
+            6
+          ];
+        }
+      `;
+      const expected = inline`
+            ├1,
+            2,
+            3,┤
+      `;
+      await assertFilterResult(filter, context, completion, expected);
+    });
+
+    it("should handle empty suffix", async () => {
+      const context = documentContext`
+        const value = ║
+      `;
+      context.language = "javascript";
+      const completion = inline`
+                      ├42;┤
+      `;
+      context.suffix = "";
+      const expected = completion;
+      await assertFilterResult(filter, context, completion, expected);
+    });
+
+    it("should handle multiple line matches", async () => {
+      const context = documentContext`
+        class Example {
+          constructor() {
+            ║
+          }
+        }
+      `;
+      context.language = "javascript";
+      const completion = inline`
+            ├this.value = 1;
+            this.name = "test";
+            this.items = [];
+            this.setup();┤
+      `;
+      context.suffix = `
+            this.setup();
+            this.init();
+          }
+        }
+      `;
+      const expected = inline`
+            ├this.value = 1;
+            this.name = "test";
+            this.items = [];┤
+      `;
+      await assertFilterResult(filter, context, completion, expected);
+    });
+
+    it("should handle partial line matches without trimming", async () => {
+      const context = documentContext`
+        const config = {
+          ║
+        };
+      `;
+      context.language = "javascript";
+      const completion = inline`
+          ├name: "test",
+          value: 42,
+          items: [],
+          enabled: true,┤
+      `;
+      context.suffix = `
+          enabled: true,
+          debug: false
+        };
+      `;
+      const expected = inline`
+          ├name: "test",
+          value: 42,
+          items: [],┤
+      `;
+      await assertFilterResult(filter, context, completion, expected);
+    });
+
+    it("should not modify when no matches found", async () => {
+      const context = documentContext`
+        function process() {
+          ║
+        }
+      `;
+      context.language = "javascript";
+      const completion = inline`
+          ├console.log("processing");
+          return true;┤
+      `;
+      context.suffix = `
+          console.log("done");
+        }
+      `;
+      const expected = completion;
+      await assertFilterResult(filter, context, completion, expected);
+    });
+
+    it("should handle whitespace differences", async () => {
+      const context = documentContext`
+        const arr = [
+          ║
+        ];
+      `;
+      context.language = "javascript";
+      const completion = inline`
+          ├1,
+             2,
+               3,
+                 4,┤
+      `;
+      context.suffix = `
+                 4,
+                 5,
+                 6
+        ];
+      `;
+      const expected = inline`
+          ├1,
+             2,
+               3,┤
+      `;
+      await assertFilterResult(filter, context, completion, expected);
+    });
+  });
+});

clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts

@@ -0,0 +1,70 @@
+import { PostprocessFilter } from "./base";
+import { CompletionItem } from "../solution";
+import { isBlank } from "../../utils/string";
+import { getLogger } from "../../logger";
+
+export function removeDuplicateSuffixLines(): PostprocessFilter {
+  return (item: CompletionItem): CompletionItem => {
+    const log = getLogger("removeDuplicateSuffixLines");
+    log.info("Processing item" + JSON.stringify(item?.text || ""));
+
+    const text = item?.text;
+    const suffix = item?.context?.suffix;
+
+    if (text == null || suffix == null) {
+      return item;
+    }
+
+    const originalLines = text.split("\n").map((line) => line || "");
+    const trimmedLines = originalLines.map((line) => (line || "").trim());
+
+    const suffixLines = (suffix || "")
+      .split("\n")
+      .map((line) => (line || "").trim())
+      .filter((line) => !isBlank(line));
+
+    if (suffixLines.length === 0) {
+      return item;
+    }
+
+    const firstSuffixLine = suffixLines[0] || "";
+
+    // iterate through lines from end to find potential match
+    for (let i = trimmedLines.length - 1; i >= 0; i--) {
+      const currentLine = trimmedLines[i] || "";
+      if (!isBlank(currentLine) && currentLine === firstSuffixLine) {
+        // check if subsequent lines also match with suffix
+        let isFullMatch = true;
+        for (let j = 0; j < suffixLines.length && i + j < trimmedLines.length; j++) {
+          const suffixLine = suffixLines[j] || "";
+          const textLine = trimmedLines[i + j] || "";
+
+          if (suffixLine !== textLine) {
+            isFullMatch = false;
+            break;
+          }
+        }
+
+        // if all checked lines match, check for code structure
+        if (isFullMatch) {
+          const remainingLines = originalLines.slice(0, i);
+          const lastLine = remainingLines[remainingLines.length - 1] || "";
+
+          // skip empty last lines
+          if (isBlank(lastLine.trim())) {
+            return item;
+          }
+
+          // preserve code block structure
+          if (lastLine.includes("{") || currentLine.includes("}")) {
+            return item;
+          }
+
+          return item.withText(remainingLines.join("\n"));
+        }
+      }
+    }
+
+    return item;
+  };
+}