Skip to content

Commit

Permalink
chore: remove duplicate suffix lines in postprocess filter
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sma1lboy
Sma1lboy committed Nov 15, 2024
1 parent 68ee0a6 commit f175287
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 165 deletions.
4 changes: 2 additions & 2 deletions clients/tabby-agent/src/codeCompletion/postprocess/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ export async function postCacheProcess(
.then(applyFilter(formatIndentation))
.then(applyFilter(dropDuplicated))
.then(applyFilter(trimSpace))
.then(applyFilter(removeDuplicateSuffixLines))
.then(applyFilter(dropMinimum))
.then(applyFilter(calculateReplaceRange))
.then(applyFilter(removeDuplicateSuffixLines));
.then(applyFilter(calculateReplaceRange));
}
246 changes: 107 additions & 139 deletions clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,152 +1,120 @@
import { documentContext, inline, assertFilterResult } from "./testUtils";
import { removeDuplicateSuffixLines } from "./removeDuplicateSuffixLines";

describe("postprocess", () => {
describe("removeDuplicateSuffixLines", () => {
const filter = removeDuplicateSuffixLines();
describe("removeDuplicateSuffixLines", () => {
const filter = removeDuplicateSuffixLines();

it("should remove duplicated suffix lines", async () => {
const context = documentContext`
function example() {
const items = [
║
];
}
`;
context.language = "javascript";
const completion = inline`
├1,
2,
3,
4,┤
`;
context.suffix = `
4,
5,
6
];
}
`;
const expected = inline`
├1,
2,
3,┤
`;
await assertFilterResult(filter, context, completion, expected);
});

it("should handle empty suffix", async () => {
const context = documentContext`
const value = ║
`;
context.language = "javascript";
const completion = inline`
├42;┤
`;
context.suffix = "";
const expected = completion;
await assertFilterResult(filter, context, completion, expected);
});
it("should remove duplicated array item", async () => {
const context = documentContext`
const items = [
║
{ id: 2 },
{ id: 3 }
];
`;
const completion = inline`
├{ id: 1 },
{ id: 2 },┤
`;
const expected = inline`
├{ id: 1 },┤
`;
await assertFilterResult(filter, context, completion, expected);
});

it("should handle multiple line matches", async () => {
const context = documentContext`
class Example {
constructor() {
║
}
}
`;
context.language = "javascript";
const completion = inline`
├this.value = 1;
this.name = "test";
this.items = [];
this.setup();┤
`;
context.suffix = `
this.setup();
this.init();
}
}
`;
const expected = inline`
├this.value = 1;
this.name = "test";
this.items = [];┤
`;
await assertFilterResult(filter, context, completion, expected);
});
it("should handle empty content after cursor", async () => {
const context = documentContext`
const a = ║
`;
const completion = inline`
├42;┤
`;
const expected = completion;
await assertFilterResult(filter, context, completion, expected);
});

it("should handle partial line matches without trimming", async () => {
const context = documentContext`
const config = {
it("should remove duplicated comma items", async () => {
const context = documentContext`
function example() {
const items = [
║
};
`;
context.language = "javascript";
const completion = inline`
├name: "test",
value: 42,
items: [],
enabled: true,┤
`;
context.suffix = `
enabled: true,
debug: false
};
`;
const expected = inline`
├name: "test",
value: 42,
items: [],┤
`;
await assertFilterResult(filter, context, completion, expected);
});
4,
5,
6
];
}
`;
const completion = inline`
├1,
2,
3,
4,┤
`;
const expected = inline`
├1,
2,
3,┤
`;
await assertFilterResult(filter, context, completion, expected);
});

it("should not modify when no matches found", async () => {
const context = documentContext`
function process() {
it("should remove duplicate method calls", async () => {
const context = documentContext`
class Example {
constructor() {
║
this.setup();
this.init()
}
`;
context.language = "javascript";
const completion = inline`
├console.log("processing");
return true;┤
`;
context.suffix = `
console.log("done");
}
`;
const expected = completion;
await assertFilterResult(filter, context, completion, expected);
});
}
`;
const completion = inline`
├this.value = 1;
this.name = "test";
this.items = [];
this.setup();┤
`;
const expected = inline`
├this.value = 1;
this.name = "test";
this.items = [];┤
`;
await assertFilterResult(filter, context, completion, expected);
});

it("should handle whitespace differences", async () => {
const context = documentContext`
const arr = [
║
];
`;
context.language = "javascript";
const completion = inline`
├1,
2,
3,
4,┤
`;
context.suffix = `
4,
5,
6
];
`;
const expected = inline`
├1,
2,
3,┤
`;
await assertFilterResult(filter, context, completion, expected);
});
it("should remove duplicate object properties", async () => {
const context = documentContext`
const config = {
║
enabled: true,
debug: false
};
`;
const completion = inline`
├name: "test",
value: 42,
items: [],
enabled: true,┤
`;
const expected = inline`
├name: "test",
value: 42,
items: [],┤
`;
await assertFilterResult(filter, context, completion, expected);
});

it("should keep content when no matches", async () => {
const context = documentContext`
function process() {
║
console.log("done");
}
`;
const completion = inline`
├console.log("processing");┤
`;
const expected = completion;
await assertFilterResult(filter, context, completion, expected);
});
});
28 changes: 4 additions & 24 deletions clients/tabby-agent/src/codeCompletion/postprocess/removeDuplicateSuffixLines.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
import { PostprocessFilter } from "./base";
import { CompletionItem } from "../solution";
import { isBlank } from "../../utils/string";
import { getLogger } from "../../logger";

export function removeDuplicateSuffixLines(): PostprocessFilter {
return (item: CompletionItem): CompletionItem => {
const log = getLogger("removeDuplicateSuffixLines");
log.info("Processing item" + JSON.stringify(item?.text || ""));

const text = item?.text;
const suffix = item?.context?.suffix;

Expand All @@ -16,7 +12,6 @@ export function removeDuplicateSuffixLines(): PostprocessFilter {
}

const originalLines = text.split("\n").map((line) => line || "");
const trimmedLines = originalLines.map((line) => (line || "").trim());

const suffixLines = (suffix || "")
.split("\n")
Expand All @@ -30,36 +25,21 @@ export function removeDuplicateSuffixLines(): PostprocessFilter {
const firstSuffixLine = suffixLines[0] || "";

// iterate through lines from end to find potential match
for (let i = trimmedLines.length - 1; i >= 0; i--) {
const currentLine = trimmedLines[i] || "";
for (let i = originalLines.length - 1; i >= 0; i--) {
const currentLine = originalLines[i] || "";
if (!isBlank(currentLine) && currentLine === firstSuffixLine) {
// check if subsequent lines also match with suffix
let isFullMatch = true;
for (let j = 0; j < suffixLines.length && i + j < trimmedLines.length; j++) {
for (let j = 0; j < suffixLines.length && i + j < originalLines.length; j++) {
const suffixLine = suffixLines[j] || "";
const textLine = trimmedLines[i + j] || "";

const textLine = originalLines[i + j] || "";
if (suffixLine !== textLine) {
isFullMatch = false;
break;
}
}

// if all checked lines match, check for code structure
if (isFullMatch) {
const remainingLines = originalLines.slice(0, i);
const lastLine = remainingLines[remainingLines.length - 1] || "";

// skip empty last lines
if (isBlank(lastLine.trim())) {
return item;
}

// preserve code block structure
if (lastLine.includes("{") || currentLine.includes("}")) {
return item;
}

return item.withText(remainingLines.join("\n"));
}
}
Expand Down

0 comments on commit f175287

Please sign in to comment.