feat(agent): add experimental postprocess limit scope by syntax. (#758)

* feat(agent): add tree-sitter parser.

* feat(agent): make parser updating tree cache optional.

* feat(agent): add experimental limit scopy by syntax.

* test(agent): update unit test for limitScopeBySyntax.

.github/workflows/intellij-test.yml

@@ -18,6 +18,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: Setup JDK
         uses: actions/setup-java@v3
         with:

.github/workflows/tabby-agent-test.yml

@@ -22,6 +22,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:

.github/workflows/vscode-test.yml

@@ -24,6 +24,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:

clients/tabby-agent/.gitattributes

@@ -0,0 +1 @@
+*.wasm filter=lfs diff=lfs merge=lfs -text

clients/tabby-agent/package.json

@@ -23,11 +23,11 @@
     "@types/node": "^18.12.0",
     "chai": "^4.3.7",
     "dedent": "^0.7.0",
+    "esbuild-plugin-copy": "^2.1.1",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "mocha": "^10.2.0",
     "openapi-typescript": "^6.6.1",
     "prettier": "^3.0.0",
-    "rimraf": "^5.0.1",
     "ts-node": "^10.9.1",
     "tsup": "^7.1.0",
     "typescript": "^5.0.3"
@@ -47,6 +47,7 @@
     "rotating-file-stream": "^3.1.0",
     "stats-logscale": "^1.0.7",
     "toml": "^3.0.0",
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "web-tree-sitter": "^0.20.8"
   }
 }

clients/tabby-agent/src/AgentConfig.ts

@@ -21,11 +21,15 @@ export type AgentConfig = {
     timeout: number;
   };
   postprocess: {
-    limitScopeByIndentation: {
-      // When completion is continuing the current line, limit the scope to:
-      // false(default): the line scope, meaning use the next indent level as the limit.
-      // true: the block scope, meaning use the current indent level as the limit.
-      experimentalKeepBlockScopeWhenCompletingLine: boolean;
+    limitScope: {
+      // Prefer to use syntax parser than indentation
+      experimentalSyntax: boolean;
+      indentation: {
+        // When completion is continuing the current line, limit the scope to:
+        // false(default): the line scope, meaning use the next indent level as the limit.
+        // true: the block scope, meaning use the current indent level as the limit.
+        experimentalKeepBlockScopeWhenCompletingLine: boolean;
+      };
     };
   };
   logs: {
@@ -66,8 +70,11 @@ export const defaultAgentConfig: AgentConfig = {
     timeout: 4000, // ms
   },
   postprocess: {
-    limitScopeByIndentation: {
-      experimentalKeepBlockScopeWhenCompletingLine: false,
+    limitScope: {
+      experimentalSyntax: false,
+      indentation: {
+        experimentalKeepBlockScopeWhenCompletingLine: false,
+      },
     },
   },
   logs: {

clients/tabby-agent/src/TabbyAgent.ts

@@ -529,7 +529,7 @@ export class TabbyAgent extends EventEmitter implements Agent {
             throw options.signal.reason;
           }
           // Build cache
-          this.completionCache.buildCache(context, completionResponse);
+          this.completionCache.buildCache(context, JSON.parse(JSON.stringify(completionResponse)));
         }
       }
       // Postprocess (post-cache)

clients/tabby-agent/src/postprocess/index.ts

@@ -4,7 +4,7 @@ import { applyFilter } from "./base";
 import { removeRepetitiveBlocks } from "./removeRepetitiveBlocks";
 import { removeRepetitiveLines } from "./removeRepetitiveLines";
 import { removeLineEndsWithRepetition } from "./removeLineEndsWithRepetition";
-import { limitScopeByIndentation } from "./limitScopeByIndentation";
+import { limitScope } from "./limitScope";
 import { trimSpace } from "./trimSpace";
 import { dropDuplicated } from "./dropDuplicated";
 import { dropBlank } from "./dropBlank";
@@ -30,7 +30,7 @@ export async function postCacheProcess(
   return Promise.resolve(response)
     .then(applyFilter(removeRepetitiveBlocks(context), context))
     .then(applyFilter(removeRepetitiveLines(context), context))
-    .then(applyFilter(limitScopeByIndentation(context, config["limitScopeByIndentation"]), context))
+    .then(applyFilter(limitScope(context, config["limitScope"]), context))
     .then(applyFilter(dropDuplicated(context), context))
     .then(applyFilter(trimSpace(context), context))
     .then(applyFilter(dropBlank(), context));

clients/tabby-agent/src/postprocess/limitScope.ts

@@ -0,0 +1,24 @@
+import { CompletionContext } from "../Agent";
+import { AgentConfig } from "../AgentConfig";
+import { isBrowser } from "../env";
+import { PostprocessFilter } from "./base";
+import { limitScopeByIndentation } from "./limitScopeByIndentation";
+import { limitScopeBySyntax, supportedLanguages } from "./limitScopeBySyntax";
+
+export function limitScope(
+  context: CompletionContext,
+  config: AgentConfig["postprocess"]["limitScope"],
+): PostprocessFilter {
+  return isBrowser
+    ? (input) => {
+        // syntax parser is not supported in browser yet
+        return limitScopeByIndentation(context, config["indentation"])(input);
+      }
+    : (input) => {
+        if (config.experimentalSyntax && supportedLanguages.indexOf(context.language) >= 0) {
+          return limitScopeBySyntax(context)(input);
+        } else {
+          return limitScopeByIndentation(context, config["indentation"])(input);
+        }
+      };
+}

clients/tabby-agent/src/postprocess/limitScopeByIndentation.ts

@@ -18,7 +18,7 @@ function processContext(
   lines: string[],
   prefixLines: string[],
   suffixLines: string[],
-  config: AgentConfig["postprocess"]["limitScopeByIndentation"],
+  config: AgentConfig["postprocess"]["limitScope"]["indentation"],
 ): { indentLevelLimit: number; allowClosingLine: (closingLine: string) => boolean } {
   let allowClosingLine = false;
   let result = { indentLevelLimit: 0, allowClosingLine: (closingLine: string) => allowClosingLine };
@@ -103,14 +103,14 @@ function processContext(
 
 export function limitScopeByIndentation(
   context: CompletionContext,
-  config: AgentConfig["postprocess"]["limitScopeByIndentation"],
+  config: AgentConfig["postprocess"]["limitScope"]["indentation"],
 ): PostprocessFilter {
   return (input) => {
-    const { prefix, suffix, prefixLines, suffixLines } = context;
+    const { prefixLines, suffixLines } = context;
     const inputLines = splitLines(input);
     if (context.mode === "fill-in-line") {
       if (inputLines.length > 1) {
-        logger.debug({ input, prefix, suffix }, "Drop content with multiple lines");
+        logger.debug({ inputLines, prefixLines, suffixLines }, "Drop content with multiple lines");
         return null;
       }
     }
@@ -139,7 +139,10 @@ export function limitScopeByIndentation(
       }
     }
     if (index < inputLines.length) {
-      logger.debug({ input, prefix, suffix, scopeEndAt: index }, "Remove content out of scope");
+      logger.debug(
+        { inputLines, prefixLines, suffixLines, scopeLineCount: index },
+        "Remove content out of indent scope",
+      );
       return inputLines.slice(0, index).join("").trimEnd();
     }
     return input;