Bump tree-sitter to 0.24 (not a release)

CHANGELOG.md

@@ -50,6 +50,7 @@ This name should be decided amongst the team before the release.
 - [#747](https://github.com/tweag/topiary/pull/747) Added support for specifying paths to prebuilt grammars in Topiary's configuration
 
 ### Changed
+- [#794](https://github.com/tweag/topiary/pull/794) Bump the `tree-sitter` dependency to 0.24 @ZedThree
 - [#780](https://github.com/tweag/topiary/pull/780) Measuring scopes are now independent from captures order
 - [#790](https://github.com/tweag/topiary/pull/790) No longer merge config files by default, use priority instead.
 

Cargo.toml

@@ -70,17 +70,19 @@ prettydiff = { version = "0.6.4", default-features = false }
 rayon = "1.10.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+streaming-iterator = "0.1.9"
 tempfile = "3.12"
 test-log = "0.2"
 tokio = "1.32"
 tokio-test = "0.4"
 toml = "0.8"
-tree-sitter = "0.22.6"
+tree-sitter = "0.24"
+tree-sitter-language = "0.1"
 # tree-sitter-bash = { git = "https://github.com/tree-sitter/tree-sitter-bash", rev = "1b0321ee85701d5036c334a6f04761cdc672e64c" }
 # tree-sitter-css = { git = "https://github.com/tree-sitter/tree-sitter-css.git", rev = "02b4ee757654b7d54fe35352fd8e53a8a4385d42" }
-tree-sitter-json = { git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "94f5c527b2965465956c2000ed6134dd24daf2a7" }
+tree-sitter-json = { git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "4d770d31f732d50d3ec373865822fbe659e47c75" }
 # tree-sitter-nickel = { git = "https://github.com/nickel-lang/tree-sitter-nickel", rev = "43433d8477b24cd13acaac20a66deda49b7e2547" }
-tree-sitter-ocaml = { git = "https://github.com/tree-sitter/tree-sitter-ocaml.git", rev = "036226e5edb410aec004cc7ac0f4b2014dd04a0e" }
+tree-sitter-ocaml = { git = "https://github.com/tree-sitter/tree-sitter-ocaml.git", rev = "98c2130c59ca7553b47086f91c5d22180151ad55" }
 # tree-sitter-ocamllex = { git = "https://github.com/314eter/tree-sitter-ocamllex.git", rev = "4b9898ccbf198602bb0dec9cd67cc1d2c0a4fad2" }
 # tree-sitter-query = { git = "https://github.com/nvim-treesitter/tree-sitter-query", rev = "a0ccc351e5e868ec1f8135e97aa3b53c663cf2df" }
 # tree-sitter-rust = { git = "https://github.com/tree-sitter/tree-sitter-rust.git", rev = "e0e8b6de6e4aa354749c794f5f36a906dcccda74" }
@@ -96,8 +98,3 @@ topiary-tree-sitter-facade = { version = "0.5.1", path = "./topiary-tree-sitter-
 topiary-core = { version = "0.5.1", path = "./topiary-core" }
 topiary-config = { version = "0.5.1", path = "./topiary-config" }
 topiary-queries = { version = "0.5.1", path = "./topiary-queries" }
-
-# tree-sitter-json's dependency on Tree-sitter is looser than ours, so
-# we have to pin its version to maintain API compatibility
-[patch."https://github.com/tree-sitter/tree-sitter-json"]
-tree-sitter = "0.22.6"

topiary-config/Cargo.toml

@@ -22,6 +22,7 @@ serde = { workspace = true, features = ["derive"] }
 tempfile.workspace = true
 toml.workspace = true
 tree-sitter.workspace = true
+tree-sitter-language.workspace = true
 
 topiary-tree-sitter-facade.workspace = true
 topiary-web-tree-sitter-sys.workspace = true

topiary-config/src/language.rs

@@ -156,9 +156,9 @@ impl Language {
         };
 
         let language = unsafe {
-            let language_fn: Symbol<unsafe extern "C" fn() -> tree_sitter::Language> =
+            let language_fn: Symbol<unsafe extern "C" fn() -> *const ()> =
                 library.get(language_fn_name.as_bytes())?;
-            language_fn()
+            tree_sitter_language::LanguageFn::from_raw(*language_fn)
         };
         std::mem::forget(library);
         Ok(topiary_tree_sitter_facade::Language::from(language))

topiary-core/Cargo.toml

@@ -19,8 +19,10 @@ pretty_assertions = { workspace = true }
 prettydiff = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+streaming-iterator = { workspace = true }
 topiary-tree-sitter-facade = { workspace = true }
 tokio = { workspace = true, features = ["macros", "rt"] }
+tree-sitter = { workspace = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 futures = { workspace = true }

topiary-core/src/lib.rs

@@ -188,7 +188,7 @@ pub enum Operation {
 /// let input = "[1,2]".to_string();
 /// let mut input = input.as_bytes();
 /// let mut output = Vec::new();
-/// let json = tree_sitter_json::language();
+/// let json = topiary_tree_sitter_facade::Language::from(tree_sitter_json::LANGUAGE);
 ///
 /// let mut query_file = BufReader::new(File::open("../topiary-queries/queries/json.scm").expect("query file"));
 /// let mut query_content = String::new();
@@ -389,7 +389,7 @@ mod tests {
         let mut input = r#"{"foo":{"bar"}}"#.as_bytes();
         let mut output = Vec::new();
         let query_content = "(#language! json)";
-        let grammar = topiary_tree_sitter_facade::Language::from(tree_sitter_json::language());
+        let grammar = topiary_tree_sitter_facade::Language::from(tree_sitter_json::LANGUAGE);
         let language = Language {
             name: "json".to_owned(),
             query: TopiaryQuery::new(&grammar, query_content).unwrap(),
@@ -425,7 +425,7 @@ mod tests {
 
         let mut output = Vec::new();
         let query_content = fs::read_to_string("../topiary-queries/queries/json.scm").unwrap();
-        let grammar = tree_sitter_json::language().into();
+        let grammar = tree_sitter_json::LANGUAGE.into();
         let language = Language {
             name: "json".to_owned(),
             query: TopiaryQuery::new(&grammar, &query_content).unwrap(),

topiary-core/src/tree_sitter.rs

@@ -1,8 +1,9 @@
 use std::{collections::HashSet, fmt::Display};
 
 use serde::Serialize;
+use streaming_iterator::StreamingIterator;
 use topiary_tree_sitter_facade::{
-    Node, Parser, Point, Query, QueryCapture, QueryCursor, QueryPredicate, Tree,
+    Node, Parser, Point, Query, QueryCapture, QueryCursor, QueryMatch, QueryPredicate, Tree,
 };
 
 use crate::{
@@ -158,11 +159,26 @@ impl<'a> NodeExt for Node<'a> {
     }
 }
 
+impl<'a> NodeExt for tree_sitter::Node<'a> {
+    fn display_one_based(&self) -> String {
+        format!(
+            "{{Node {:?} {} - {}}}",
+            self.kind(),
+            Position::from(<tree_sitter::Point as Into<Point>>::into(
+                self.start_position()
+            )),
+            Position::from(<tree_sitter::Point as Into<Point>>::into(
+                self.end_position()
+            )),
+        )
+    }
+}
+
 #[derive(Debug)]
 // A struct to statically store the public fields of query match results,
 // to avoid running queries twice.
 struct LocalQueryMatch<'a> {
-    pattern_index: u32,
+    pattern_index: usize,
     captures: Vec<QueryCapture<'a>>,
 }
 
@@ -219,7 +235,8 @@ pub fn apply_query(
     let mut matches: Vec<LocalQueryMatch> = Vec::new();
     let capture_names = query.query.capture_names();
 
-    for query_match in query.query.matches(&root, source, &mut cursor) {
+    let mut query_matches = query.query.matches(&root, source, &mut cursor);
+    while let Some(query_match) = query_matches.next() {
         let local_captures: Vec<QueryCapture> = query_match.captures().collect();
 
         matches.push(LocalQueryMatch {
@@ -272,9 +289,9 @@ pub fn apply_query(
             }
 
             // Fetch from pattern_positions, otherwise insert
-            let pos = pattern_positions[m.pattern_index as usize].unwrap_or_else(|| {
-                let pos = query.pattern_position(m.pattern_index as usize);
-                pattern_positions[m.pattern_index as usize] = Some(pos);
+            let pos = pattern_positions[m.pattern_index].unwrap_or_else(|| {
+                let pos = query.pattern_position(m.pattern_index);
+                pattern_positions[m.pattern_index] = Some(pos);
                 pos
             });
 

topiary-tree-sitter-facade/Cargo.toml

@@ -14,6 +14,8 @@ description = """
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 tree-sitter.workspace = true
+tree-sitter-language.workspace = true
+streaming-iterator.workspace = true
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 js-sys.workspace = true

topiary-tree-sitter-facade/src/language.rs

@@ -80,10 +80,12 @@ mod native {
         }
     }
 
-    impl From<tree_sitter::Language> for Language {
+    impl From<tree_sitter_language::LanguageFn> for Language {
         #[inline]
-        fn from(inner: tree_sitter::Language) -> Self {
-            Self { inner }
+        fn from(inner: tree_sitter_language::LanguageFn) -> Self {
+            Self {
+                inner: tree_sitter::Language::new(inner),
+            }
         }
     }
 

topiary-tree-sitter-facade/src/parser.rs

@@ -4,7 +4,7 @@
 mod native {
     use crate::{
         error::{IncludedRangesError, LanguageError, ParserError},
-        language::Language,
+        language::{Language, LanguageRef},
         logger::{Logger, LoggerReturn},
         point::Point,
         range::Range,
@@ -31,7 +31,7 @@ mod native {
         }
 
         #[inline]
-        pub fn language(&self) -> Option<Language> {
+        pub fn language(&self) -> Option<LanguageRef> {
             self.inner.language().map(Into::into)
         }
 

topiary-tree-sitter-facade/src/query.rs

@@ -2,8 +2,9 @@
 mod native {
     use crate::{
         error::QueryError, language::Language, node::Node, query_cursor::QueryCursor,
-        query_match::QueryMatch, query_predicate::QueryPredicate,
+        query_predicate::QueryPredicate,
     };
+    use streaming_iterator::StreamingIterator;
 
     pub struct Query {
         pub(crate) inner: tree_sitter::Query,
@@ -17,16 +18,19 @@ mod native {
         }
 
         #[inline]
-        pub fn matches<'a, 'tree: 'a>(
-            &'a self,
+        pub fn matches<
+            'query,
+            'cursor: 'query,
+            'tree: 'query,
+            T: tree_sitter::TextProvider<I> + 'query,
+            I: AsRef<[u8]> + 'query,
+        >(
+            &'query self,
             node: &Node<'tree>,
-            source: &'a [u8],
-            cursor: &'a mut QueryCursor,
-        ) -> impl Iterator<Item = QueryMatch<'a>> + 'a {
-            cursor
-                .inner
-                .matches(&self.inner, node.inner, source)
-                .map(Into::into)
+            source: T,
+            cursor: &'cursor mut QueryCursor,
+        ) -> impl StreamingIterator<Item = tree_sitter::QueryMatch<'query, 'tree>> {
+            cursor.inner.matches(&self.inner, node.inner, source)
         }
 
         #[inline]
@@ -35,8 +39,7 @@ mod native {
         }
 
         #[inline]
-        pub fn general_predicates(&self, index: u32) -> Vec<QueryPredicate> {
-            let index = index as usize;
+        pub fn general_predicates(&self, index: usize) -> Vec<QueryPredicate> {
             self.inner
                 .general_predicates(index)
                 .iter()