Stable Stream Token interface (#6707)

.github/workflows/ci.yml

@@ -60,6 +60,7 @@ jobs:
           - test --profile=ci-dev -p cairo-lang-test-utils
           - test --profile=ci-dev -p cairo-lang-utils --features=serde,parity-scale-codec,schemars,testing,env_logger
           - test --profile=ci-dev -p cairo-lang-utils --no-default-features --features=serde,parity-scale-codec
+          - test --profile=ci-dev -p cairo-lang-primitive-token
           - test --profile=ci-dev -p tests
     steps:
       - uses: actions/checkout@v4

Cargo.toml

@@ -49,6 +49,7 @@ members = [
     "crates/cairo-lang-sierra-generator",
     "crates/cairo-lang-sierra-to-casm",
     "crates/cairo-lang-sierra-type-size",
+    "crates/cairo-lang-primitive-token",
     "crates/cairo-lang-starknet",
     "crates/cairo-lang-starknet-classes",
     "crates/cairo-lang-syntax",

crates/cairo-lang-primitive-token/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "cairo-lang-primitive-token"
+version = "1.0.0"
+edition.workspace = true
+repository.workspace = true
+license-file.workspace = true
+description = "Primitive representation of a TokenStream"

crates/cairo-lang-primitive-token/src/lib.rs

@@ -0,0 +1,35 @@
+#![deny(missing_docs)]
+//! This crate defines unfiorm and primitive form of the TokenStream.
+//! We want this to be as stable as possible and limit the changes here to bare minimum.
+
+/// Primitive representation of a token's span.
+pub struct PrimitiveSpan {
+    /// Start position of the span.
+    pub start: usize,
+    /// End position of the span.
+    pub end: usize,
+}
+
+/// Primitive representation of a single token.
+pub struct PrimitiveToken {
+    /// Plain code content that the token represents (includes whitespaces).
+    pub content: String,
+    /// Span of the token.
+    pub span: Option<PrimitiveSpan>,
+}
+
+impl PrimitiveToken {
+    /// Creates a new primitive token based upon content and provided span.
+    pub fn new(content: String, span: Option<PrimitiveSpan>) -> Self {
+        Self { content, span }
+    }
+}
+
+/// Trait that defines an object that can be turned into a PrimitiveTokenStream iterator.
+pub trait ToPrimitiveTokenStream {
+    /// Iterator type for PrimitiveTokens.
+    type Iter: Iterator<Item = PrimitiveToken>;
+
+    /// Method that turns given item to a PrimitiveTokenStream iterator.
+    fn to_primitive_token_stream(&self) -> Self::Iter;
+}

crates/cairo-lang-syntax/Cargo.toml

@@ -10,6 +10,7 @@ description = "Cairo syntax representation."
 cairo-lang-debug = { path = "../cairo-lang-debug", version = "~2.9.0" }
 cairo-lang-filesystem = { path = "../cairo-lang-filesystem", version = "~2.9.0" }
 cairo-lang-utils = { path = "../cairo-lang-utils", version = "~2.9.0" }
+cairo-lang-primitive-token = { path = "../cairo-lang-primitive-token", version = "1.0.0"}
 num-bigint = { workspace = true, default-features = true }
 num-traits = { workspace = true, default-features = true }
 salsa.workspace = true

crates/cairo-lang-syntax/src/node/mod.rs

@@ -26,6 +26,7 @@ pub mod key_fields;
 pub mod kind;
 pub mod stable_ptr;
 pub mod utils;
+pub mod with_db;
 
 #[cfg(test)]
 mod ast_test;

crates/cairo-lang-syntax/src/node/with_db.rs

@@ -0,0 +1,49 @@
+use cairo_lang_primitive_token::{PrimitiveSpan, PrimitiveToken, ToPrimitiveTokenStream};
+
+use super::SyntaxNode;
+use super::db::SyntaxGroup;
+
+pub struct SyntaxNodeWithDb<'a, Db: SyntaxGroup> {
+    node: &'a SyntaxNode,
+    db: &'a Db,
+}
+
+impl<'a, Db: SyntaxGroup> SyntaxNodeWithDb<'a, Db> {
+    pub fn new(node: &'a SyntaxNode, db: &'a Db) -> Self {
+        Self { node, db }
+    }
+}
+
+impl<'a, Db: SyntaxGroup> ToPrimitiveTokenStream for SyntaxNodeWithDb<'a, Db> {
+    type Iter = SyntaxNodeWithDbIterator<'a, Db>;
+
+    fn to_primitive_token_stream(&self) -> Self::Iter {
+        // The lifetime of the iterator should extend 'a because it derives from both node and db
+        SyntaxNodeWithDbIterator::new(Box::new(self.node.tokens(self.db)), self.db)
+    }
+}
+
+pub struct SyntaxNodeWithDbIterator<'a, Db: SyntaxGroup> {
+    inner: Box<dyn Iterator<Item = SyntaxNode> + 'a>,
+    db: &'a Db,
+}
+
+impl<'a, Db: SyntaxGroup> SyntaxNodeWithDbIterator<'a, Db> {
+    pub fn new(inner: Box<dyn Iterator<Item = SyntaxNode> + 'a>, db: &'a Db) -> Self {
+        Self { inner, db }
+    }
+}
+
+impl<Db: SyntaxGroup> Iterator for SyntaxNodeWithDbIterator<'_, Db> {
+    type Item = PrimitiveToken;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next().map(|node| {
+            let span = node.span(self.db).to_str_range();
+            PrimitiveToken {
+                content: node.get_text(self.db),
+                span: Some(PrimitiveSpan { start: span.start, end: span.end }),
+            }
+        })
+    }
+}

scripts/release_crates.sh

@@ -53,9 +53,9 @@ CRATES_TO_PUBLISH=(
 )
 
 # Assert that the number of crates to publish is equal to the number of crates in the workspace
-# - 4 (the number of crates that are for internal use only).
+# - 5 (the number of crates that are for internal use only).
 NUM_CRATES_IN_WORKSPACE=$(find crates/ -name Cargo.toml | wc -l) 
-if [ "${#CRATES_TO_PUBLISH[@]}" -ne "$((NUM_CRATES_IN_WORKSPACE - 4))" ]; then
+if [ "${#CRATES_TO_PUBLISH[@]}" -ne "$((NUM_CRATES_IN_WORKSPACE - 5))" ]; then
     echo "The number of crates to publish is not equal to the number of crates in the workspace, 
     new crates were probably added, please update the list of crates to publish."
     exit 1