Upgrade web-tree-sitter to 0.23.0…

…and update our API usages.

Tree-sitter harmonized the API differences between `web-tree-sitter` and
`node-tree-sitter` in version 0.22.0. This is the first time we’ve had to deal
with that. Luckily, the changes were mostly palatable.

The biggest API difference is in `Query#captures`; two positional arguments for
defining the extent of the query have been moved to keyword arguments. We’ve
updated our internal usages, but any community packages that relied on the old
function signature would break if we didn’t do anything about it. So we’ve
wrapped the `Query#captures` method in one of our own; it detects usages that
expect the old signature and rearranges their arguments, issuing a deprecation
warning in the process. Hopefully this generates enough noise that any such
packages understand what’s going on and can update.

Other API changes are more obscure — which is good, because we can’t wrap them
the way we wrapped `Query#captures`. They involve conversion of functions to
getters (`node.hasErrors` instead of `node.hasErrors()`), and there’s no good
way to make both usages work… short of wrapping nodes in `Proxy` objects, and
that’s not on the table.

Since lots has changed in `tree-sitter` since we last upgraded
`web-tree-sitter`, I updated our documentation about building a custom version
of `web-tree-sitter`.

packages/symbol-provider-tree-sitter/lib/tree-sitter-provider.js

@@ -77,8 +77,7 @@ class TreeSitterProvider {
       let tagsQuery = layer.queries?.tagsQuery ?? layer.tagsQuery;
       let captures = tagsQuery.captures(
         layer.tree.rootNode,
-        extent.start,
-        extent.end
+        { startPosition: extent.start, endPosition: extent.end }
       );
 
       results.push(

spec/scope-resolver-spec.js

@@ -38,7 +38,7 @@ async function getAllCapturesWithScopeResolver(grammar, languageMode, scopeResol
   let { start, end } = languageMode.buffer.getRange();
   let { tree } = layer;
   return {
-    captures: query.captures(tree.rootNode, start, end),
+    captures: query.captures(tree.rootNode, { startPosition: start, endPosition: end }),
     scopeResolver
   };
 }
@@ -142,7 +142,7 @@ describe('ScopeResolver', () => {
 
     let tokens = [];
     const original = grammar.idForScope.bind(grammar);
-    grammar.idForScope = function(scope, text) {
+    grammar.idForScope = function (scope, text) {
       if (text) {
         tokens.push(text);
       }
@@ -972,7 +972,7 @@ describe('ScopeResolver', () => {
 
       expect(matched.length).toBe(1);
       expect(matched.every(cap => {
-        return cap.name === 'messed-up-statement-block' && cap.node.hasError();
+        return cap.name === 'messed-up-statement-block' && cap.node.hasError;
       })).toBe(true);
     });
 

src/scope-resolver.js

@@ -641,7 +641,7 @@ ScopeResolver.TESTS = {
 
   // Passes only if the node contains any descendant ERROR nodes.
   hasError(node) {
-    return node.hasError();
+    return node.hasError;
   },
 
   // Passes when the node's tree belongs to an injection layer, rather than the

src/wasm-tree-sitter-grammar.js

@@ -1,12 +1,70 @@
 const fs = require('fs');
 const path = require('path');
+const Grim = require('grim');
+const dedent = require('dedent');
 const Parser = require('./web-tree-sitter');
 const { CompositeDisposable, Emitter } = require('event-kit');
 const { File } = require('pathwatcher');
 const { normalizeDelimiters } = require('./comment-utils.js');
 
 const parserInitPromise = Parser.init();
 
+function isPosition(obj) {
+  return ('row' in obj && 'column' in obj);
+}
+
+const ZERO_POINT = Object.freeze({ row: 0, column: 0 });
+
+const QUERY_CAPTURES_DEPRECATION_EXPLANATION = dedent`\
+  The \`captures\` method available on Tree-sitter query objects uses a new
+  function signature; the old signature is deprecated. The new signature is
+  \`(node, options)\`. If you want to limit a query to a specific range,
+  specify \`startPosition\` and \`endPosition\` properties within \`options\`.
+`;
+
+let didWrapQueryCaptures = false;
+
+// When `web-tree-sitter` harmonized its API with that of `node-tree-sitter`,
+// some function signatures changed. The most impactful one for us is probably
+// `Query#captures`, since two crucial positional arguments were moved into a
+// trailing options argument.
+//
+// We've changed all of our usages, but it's possible some community packages
+// won't have been able to update yet. We should emit a deprecation message in
+// those cases and restructure the arguments on the fly.
+function wrapQueryCaptures(query) {
+  didWrapQueryCaptures = true;
+  let QueryPrototype = Object.getPrototypeOf(query);
+  let originalCaptures = QueryPrototype.captures;
+  // We put `node` into its own argument so that this new function’s `length`
+  // property matches that of the old function. (Both are inaccurate, but they
+  // should nonetheless agree.)
+  QueryPrototype.captures = function captures(node, ...args) {
+    // When do we think a consumer is using the old signature?
+    if (
+      // If there are too many arguments and either the second or third
+      // argument looks like a position…
+      args.length >= 2 && (isPosition(args[0]) || isPosition(args[1])) ||
+      // …or if the second argument looks like a position instead of an options
+      // object.
+      isPosition(args[0])
+    ) {
+      Grim.deprecate(QUERY_CAPTURES_DEPRECATION_EXPLANATION);
+      let startPosition = isPosition(args[0]) ? args[0] : ZERO_POINT;
+      let endPosition = isPosition(args[1]) ? args[1] : args[0];
+      let originalOptions = args[2] ?? {};
+      let newOptions = {
+        ...originalOptions,
+        startPosition,
+        endPosition
+      };
+      return originalCaptures.call(this, node, newOptions);
+    } else {
+      return originalCaptures.call(this, node, ...args);
+    }
+  };
+}
+
 // Extended: This class holds an instance of a Tree-sitter grammar.
 module.exports = class WASMTreeSitterGrammar {
   constructor(registry, grammarPath, params) {
@@ -274,6 +332,15 @@ module.exports = class WASMTreeSitterGrammar {
           // if (inDevMode) { console.time(timeTag); }
           query = language.query(this[queryType]);
 
+          // We want to augment the `Query` class to add backward compatibility
+          // for the `captures` method. But since `web-tree-sitter` doesn’t
+          // export references to these inner Tree-sitter classes, we have to
+          // wait until we’re holding an instance of a `Query` and grab its
+          // prototype. Luckily, we still only have to do this once.
+          if (!didWrapQueryCaptures) {
+            wrapQueryCaptures(query);
+          }
+
           // if (inDevMode) { console.timeEnd(timeTag); }
           this.queryCache.set(queryType, query);
           resolve(query);

src/wasm-tree-sitter-language-mode.js

@@ -12,6 +12,8 @@ const { commentStringsFromDelimiters, getDelimitersForScope } = require('./comme
 
 const createTree = require('./rb-tree');
 
+const ONE_CHAR_FORWARD_TRAVERSAL = Object.freeze(Point(0, 1));
+
 const FEATURE_ASYNC_INDENT = true;
 const FEATURE_ASYNC_PARSE = true;
 
@@ -83,7 +85,7 @@ function resolveNodePosition(node, descriptor) {
   return result[lastPart];
 }
 
-// Patch tree-sitter syntax nodes the same way `TreeSitterLanguageMode` did so
+// Patch Tree-sitter syntax nodes the same way `TreeSitterLanguageMode` did so
 // that we don't break anything that relied on `range` being present.
 function ensureNodeIsPatched(node) {
   let done = node.range && node.range instanceof Range;
@@ -108,7 +110,7 @@ function ensureNodeIsPatched(node) {
   });
 }
 
-// Compares “informal” points like the ones in a tree-sitter tree; saves us
+// Compares “informal” points like the ones in a Tree-sitter tree; saves us
 // from having to convert them to actual `Point`s.
 function comparePoints(a, b) {
   const rows = a.row - b.row;
@@ -429,7 +431,7 @@ class WASMTreeSitterLanguageMode {
     return true;
   }
 
-  // Resolves the next time that all tree-sitter trees are clean — or
+  // Resolves the next time that all Tree-sitter trees are clean — or
   // immediately, if they're clean at the time of invocation.
   //
   // Resolves with metadata about the previous transaction that may be useful
@@ -461,7 +463,7 @@ class WASMTreeSitterLanguageMode {
   }
 
   // Alias for `atTransactionEnd` for packages that used the implementation
-  // details of the legacy tree-sitter system.
+  // details of the legacy Tree-sitter system.
   parseCompletePromise() {
     return this.atTransactionEnd();
   }
@@ -535,7 +537,7 @@ class WASMTreeSitterLanguageMode {
   }
 
   // Behaves like `scopeDescriptorForPosition`, but returns a list of
-  // tree-sitter node names. Useful for understanding tree-sitter parsing or
+  // Tree-sitter node names. Useful for understanding Tree-sitter parsing or
   // for writing syntax highlighting query files.
   syntaxTreeScopeDescriptorForPosition(point) {
     point = this.normalizePointForPositionQuery(point);
@@ -573,7 +575,7 @@ class WASMTreeSitterLanguageMode {
     );
 
     let scopes = matches.map(({ node }) => (
-      node.isNamed() ? node.type : `"${node.type}"`
+      node.isNamed ? node.type : `"${node.type}"`
     ));
     scopes.unshift(this.grammar.scopeName);
 
@@ -1689,7 +1691,10 @@ class FoldResolver {
     // boundary ends at the same point that another one starts, the ending
     // boundary will be visited first.
     let boundaries = createTree(compareBoundaries);
-    let captures = this.layer.queries.foldsQuery.captures(rootNode, start, end);
+    let captures = this.layer.queries.foldsQuery.captures(
+      rootNode,
+      { startPosition: start, endPosition: end }
+    );
 
     for (let capture of captures) {
       // NOTE: Currently, the first fold to match for a given starting position
@@ -2021,7 +2026,7 @@ class HighlightIterator {
     //
     let [result, openScopes] = iterator.seek(start, endRow);
 
-    if (rootLanguageLayer?.tree?.rootNode.hasChanges()) {
+    if (rootLanguageLayer?.tree?.rootNode.hasChanges) {
       // The tree is dirty. We should keep going — if we stop now, then the
       // user will see a flash of unhighlighted text over this whole range. But
       // we should also schedule a re-highlight at the end of the transaction,
@@ -2655,7 +2660,7 @@ class LanguageLayer {
 
   isDirty() {
     if (!this.tree) { return false; }
-    return this.tree.rootNode.hasChanges();
+    return this.tree.rootNode.hasChanges;
   }
 
   inspect() {
@@ -2667,7 +2672,7 @@ class LanguageLayer {
     if (this.destroyed) { return; }
     this.destroyed = true;
 
-    // Clean up all tree-sitter trees.
+    // Clean up all Tree-sitter trees.
     let temporaryTrees = this.temporaryTrees ?? [];
     let trees = new Set([this.tree, this.lastSyntaxTree, ...temporaryTrees]);
     trees = [...trees];
@@ -2751,7 +2756,10 @@ class LanguageLayer {
     let boundaries = createTree(compareBoundaries);
     let extent = this.getExtent();
 
-    let captures = this.queries.highlightsQuery?.captures(this.tree.rootNode, from, to) ?? [];
+    let captures = this.queries.highlightsQuery?.captures(
+      this.tree.rootNode,
+      { startPosition: from, endPosition: to }
+    ) ?? [];
     this.scopeResolver.reset();
 
     for (let capture of captures) {
@@ -3003,7 +3011,7 @@ class LanguageLayer {
     if (!this.languageMode.useAsyncParsing) {
       // Practically speaking, updates that affect _only this layer_ will happen
       // synchronously, because we've made sure not to call this method until the
-      // root grammar's tree-sitter parser has been loaded. But we can't load any
+      // root grammar's Tree-sitter parser has been loaded. But we can't load any
       // potential injection layers' languages because we don't know which ones
       // we'll need _until_ we parse this layer's tree for the first time.
       //
@@ -3029,7 +3037,7 @@ class LanguageLayer {
         await this.currentParsePromise;
       } while (
         !this.destroyed &&
-        (!this.tree || this.tree.rootNode.hasChanges())
+        (!this.tree || this.tree.rootNode.hasChanges)
       );
 
       this.currentParsePromise = null;
@@ -3046,8 +3054,10 @@ class LanguageLayer {
     if (!this.queries.localsQuery) { return []; }
     let captures = this.queries.localsQuery.captures(
       this.tree.rootNode,
-      point,
-      point + 1
+      {
+        startPosition: point,
+        endPosition: point.translate(ONE_CHAR_FORWARD_TRAVERSAL)
+      }
     );
 
     captures = captures.filter(cap => {
@@ -3074,11 +3084,11 @@ class LanguageLayer {
     let globalScope = this.tree.rootNode;
 
     if (!captures) {
+      let { startPosition, endPosition } = globalScope;
       captures = this.groupLocalsCaptures(
         this.queries.localsQuery.captures(
           globalScope,
-          globalScope.startPosition,
-          globalScope.endPosition
+          { startPosition, endPosition }
         )
       );
     }
@@ -3382,7 +3392,7 @@ class LanguageLayer {
       // transaction's tree later on.
       this.lastSyntaxTree = tree;
 
-      // Like legacy tree-sitter, we're patching syntax nodes so that they have
+      // Like legacy Tree-sitter, we're patching syntax nodes so that they have
       // a `range` property that returns a `Range`. We're doing this for
       // compatibility, but we can't get a reference to the node class itself;
       // we have to wait until we have an instance and grab the prototype from
@@ -3568,8 +3578,10 @@ class LanguageLayer {
     // the character in column X.
     let captures = this.queries.highlightsQuery?.captures(
       this.tree.rootNode,
-      point,
-      { row: point.row, column: point.column + 1 }
+      {
+        startPosition: point,
+        endPosition: point.translate(ONE_CHAR_FORWARD_TRAVERSAL)
+      }
     ) ?? [];
 
     let results = [];
@@ -4063,16 +4075,19 @@ class OpenScopeMap extends Map {
   }
 
   removeLastOccurrenceOf(scopeId) {
-    let keys = [...this.keys()];
-    keys.reverse();
-    for (let key of keys) {
+    let candidateKey;
+    // Of the keys whose values include this scope, find the one that occurs
+    // latest in the document.
+    for (let key of this.keys()) {
       let value = this.get(key);
-      if (value.includes(scopeId)) {
-        removeLastOccurrenceOf(value, scopeId);
-        return true;
+      if (!value.includes(scopeId)) continue;
+      if (!candidateKey || comparePoints(key, candidateKey) === 1) {
+        candidateKey = key;
       }
     }
-    return false;
+    if (!candidateKey) return false;
+    removeLastOccurrenceOf(this.get(candidateKey), scopeId);
+    return true;
   }
 }
 
@@ -4098,7 +4113,7 @@ class Index extends Map {
 // intersections with range already in the list, those intersections are
 // combined into one larger range.
 //
-// Assumes all ranges are instances of `Range` rather than tree-sitter range
+// Assumes all ranges are instances of `Range` rather than Tree-sitter range
 // specs.
 class RangeList {
   constructor() {
@@ -4359,8 +4374,10 @@ class IndentResolver {
     // Perform the Phase 1 capture.
     let indentCaptures = indentsQuery.captures(
       indentTree.rootNode,
-      { row: comparisonRow, column: 0 },
-      { row: row, column: 0 }
+      {
+        startPosition: { row: comparisonRow, column: 0 },
+        endPosition: { row: row, column: 0 }
+      }
     );
 
     // Keep track of the first `@indent` capture on the line. When balancing
@@ -4571,8 +4588,10 @@ class IndentResolver {
       // Perform the Phase 2 capture.
       let dedentCaptures = indentsQuery.captures(
         indentTree.rootNode,
-        { row: row - 1, column: Infinity },
-        { row: row + 1, column: 0 }
+        {
+          startPosition: { row: row - 1, column: Infinity },
+          endPosition: { row: row + 1, column: 0 }
+        }
       );
 
       let currentRowText = lineText.trim();
@@ -4994,8 +5013,10 @@ class IndentResolver {
 
     const indents = indentsQuery.captures(
       indentTree.rootNode,
-      { row: row - 1, column: Infinity },
-      { row: row + 1, column: 0 }
+      {
+        startPosition: { row: row - 1, column: Infinity },
+        endPosition: { row: row + 1, column: 0 }
+      }
     );
 
     let lineText = this.buffer.lineForRow(row).trim();