refactor: Use location instead of usize

lib/src/parser/default_impl/mod.rs

@@ -52,14 +52,14 @@ impl ParserTrait for DefaultParser {
         }
 
         // TODO: Parse no result
-        Err(ParseError::InvalidToken(0))
+        Err(ParseError::InvalidToken(Location { mark: 0, offset: 0 }))
     }
 
     #[inline]
     fn parse_expression(&self, lexer: &mut StLexer) -> Result<Expression, ParseError> {
         match DefaultParserImpl::new(lexer.into_iter()).parse_expression()? {
             Some(expr) => Ok(expr),
-            None => Err(ParseError::InvalidToken(0)),
+            None => Err(ParseError::InvalidToken(Location { mark: 0, offset: 0 })),
         }
     }
 }
@@ -210,7 +210,7 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
         let tok = self.next_token()?;
         match &tok.kind {
             TokenKind::Identifier(ident) => Ok(ident.clone()),
-            _ => Err(ParseError::InvalidToken(0)),
+            _ => Err(ParseError::InvalidToken(Location { mark: 0, offset: 0 })),
         }
     }
 
@@ -226,7 +226,7 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
             }
         }
 
-        Err(ParseError::expect_tokens(0, tokens))
+        Err(ParseError::expect_tokens(tok.location, tokens))
     }
 
     fn parse_type(&mut self) -> ParseResult<Type> {
@@ -371,7 +371,10 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
             return Ok(Some(Declaration::new_struct(name, fields)));
         }
 
-        Err(ParseError::UnexpectedToken(0, vec![format!("{:?}", tok)]))
+        Err(ParseError::UnexpectedToken(
+            Location { mark: 0, offset: 0 },
+            vec![format!("{:?}", tok)],
+        ))
     }
 
     fn parse_enum_field_decl(&mut self) -> ParseResult<Arc<Variable>> {

lib/src/parser/lalrpop_impl/mod.rs

@@ -6,12 +6,18 @@ use once_cell::sync::Lazy;
 
 lalrpop_mod!(st, "/parser/lalrpop_impl/st.rs");
 
-type LalrPopLexerItem = (usize, TokenKind, usize);
+type LalrPopLexerItem = (Location, TokenKind, Location);
 type LalrPopLexerResult = Result<LalrPopLexerItem, LexicalError>;
 
 impl From<Token> for LalrPopLexerItem {
     fn from(tok: Token) -> Self {
-        (tok.pos.mark, tok.kind, tok.pos.offset)
+        // TODO: multi-line token support
+        let end_loc = Location {
+            mark: tok.location.mark,
+            offset: tok.location.offset + tok.length,
+        };
+
+        (tok.location, tok.kind, end_loc)
     }
 }
 

lib/src/parser/lalrpop_impl/st.lalrpop

@@ -6,7 +6,7 @@ use smallvec::smallvec;
 grammar;
 
 extern {
-    type Location = usize;
+    type Location = Location;
     type Error = LexicalError;
 
     enum TokenKind {

lib/src/parser/lexer.rs

@@ -1,5 +1,5 @@
 use crate::ast::*;
-use crate::parser::token::{Token, TokenPosition};
+use crate::parser::token::{Location, Token};
 use crate::parser::{Buffer, IterBuffer, StreamBuffer, TokenKind};
 use crate::prelude::StString;
 use bitflags::bitflags;
@@ -579,7 +579,7 @@ impl<'input> StLexer<'input> {
     fn next_raw(&mut self) -> Option<<Self as Iterator>::Item> {
         let mut tok = Token {
             length: 1,
-            pos: TokenPosition {
+            location: Location {
                 mark: self.buffer.current_line(),
                 offset: self.buffer.current_offset(),
             },
@@ -750,13 +750,13 @@ mod test {
         let mut lexer = StLexerBuilder::new().build_str(s);
 
         let x = lexer.next().unwrap().unwrap();
-        assert_eq!(x.pos.offset, 0);
-        assert_eq!(x.pos.mark, 0);
+        assert_eq!(x.location.offset, 0);
+        assert_eq!(x.location.mark, 0);
         assert_eq!(x.length, 2);
         assert_eq!(x.kind, TokenKind::If);
 
         let x = lexer.next().unwrap().unwrap();
-        assert_eq!(x.pos.offset, 3);
+        assert_eq!(x.location.offset, 3);
         assert_eq!(x.length, 3);
         assert!(matches!(x.kind, TokenKind::Identifier(_)));
 
@@ -769,18 +769,18 @@ mod test {
         let mut lexer = StLexerBuilder::new().build_str(s);
 
         let x = lexer.next().unwrap().unwrap();
-        assert_eq!(x.pos.offset, 0);
+        assert_eq!(x.location.offset, 0);
         assert_eq!(x.length, 1);
         assert!(matches!(x.kind, TokenKind::Literal(_)));
 
         let x = lexer.next().unwrap().unwrap();
-        assert_eq!(x.pos.offset, 2);
+        assert_eq!(x.location.offset, 2);
         assert_eq!(x.length, 1);
         assert!(matches!(x.kind, TokenKind::Plus));
 
         let x = lexer.next().unwrap().unwrap();
-        assert_eq!(x.pos.mark, 1);
-        assert_eq!(x.pos.offset, 0);
+        assert_eq!(x.location.mark, 1);
+        assert_eq!(x.location.offset, 0);
         assert_eq!(x.length, 1);
         assert!(matches!(x.kind, TokenKind::Identifier(_)));
     }
@@ -791,7 +791,7 @@ mod test {
         let mut lexer = StLexerBuilder::new().build_str(s);
 
         let x = lexer.next().unwrap().unwrap();
-        assert_eq!(x.pos.offset, 0);
+        assert_eq!(x.location.offset, 0);
         assert_eq!(x.length, 6);
         assert!(matches!(x.kind, TokenKind::Identifier(_)));
     }
@@ -862,8 +862,8 @@ mod test {
         assert!(matches!(x.kind, TokenKind::Identifier(..)));
         let y = lexer.next().unwrap().unwrap();
         assert!(matches!(y.kind, TokenKind::Identifier(..)));
-        assert_eq!(y.pos.mark, 1);
-        assert_eq!(y.pos.offset, 0);
+        assert_eq!(y.location.mark, 1);
+        assert_eq!(y.location.offset, 0);
 
         let s = "a\n\rb";
         let mut lexer = StLexerBuilder::new().build_str(s);
@@ -872,8 +872,8 @@ mod test {
         assert!(matches!(x.kind, TokenKind::Identifier(..)));
         let y = lexer.next().unwrap().unwrap();
         assert!(matches!(y.kind, TokenKind::Identifier(..)));
-        assert_eq!(y.pos.mark, 1);
-        assert_eq!(y.pos.offset, 0);
+        assert_eq!(y.location.mark, 1);
+        assert_eq!(y.location.offset, 0);
 
         let s = "a\n\nb";
         let mut lexer = StLexerBuilder::new().build_str(s);
@@ -882,8 +882,8 @@ mod test {
         assert!(matches!(x.kind, TokenKind::Identifier(..)));
         let y = lexer.next().unwrap().unwrap();
         assert!(matches!(y.kind, TokenKind::Identifier(..)));
-        assert_eq!(y.pos.mark, 2);
-        assert_eq!(y.pos.offset, 0);
+        assert_eq!(y.location.mark, 2);
+        assert_eq!(y.location.offset, 0);
     }
 
     #[test]

lib/src/parser/mod.rs

@@ -14,7 +14,7 @@ mod operator;
 pub use operator::Operator;
 
 mod token;
-pub use token::TokenKind;
+pub use token::{Location, TokenKind};
 
 #[macro_export]
 macro_rules! parse_statement {
@@ -32,22 +32,22 @@ mod test;
 pub enum ParseError {
     LexerError(LexicalError),
     UnexpectedEnd,
-    InvalidToken(usize),
+    InvalidToken(Location),
     InvalidTokenAt(String),
-    UnexpectedToken(usize, Vec<String>),
+    UnexpectedToken(Location, Vec<String>),
 }
 
 impl ParseError {
-    pub fn expect_tokens(pos: usize, tokens: &[TokenKind]) -> Self {
+    pub fn expect_tokens(loc: Location, tokens: &[TokenKind]) -> Self {
         let tokens: Vec<_> = tokens.iter().map(|x| x.into()).collect();
 
-        Self::UnexpectedToken(pos, tokens)
+        Self::UnexpectedToken(loc, tokens)
     }
 }
 
 #[cfg(feature = "lalrpop_parser")]
-impl From<lalrpop_util::ParseError<usize, TokenKind, LexicalError>> for ParseError {
-    fn from(e: lalrpop_util::ParseError<usize, TokenKind, LexicalError>) -> Self {
+impl From<lalrpop_util::ParseError<Location, TokenKind, LexicalError>> for ParseError {
+    fn from(e: lalrpop_util::ParseError<Location, TokenKind, LexicalError>) -> Self {
         match e {
             lalrpop_util::ParseError::InvalidToken { location: loc } => {
                 ParseError::InvalidToken(loc)

lib/src/parser/token.rs

@@ -1,8 +1,8 @@
 use crate::parser::*;
 use std::fmt::{self, Display, Formatter};
 
-#[derive(Debug)]
-pub struct TokenPosition {
+#[derive(Debug, Clone, Default, Copy)]
+pub struct Location {
     pub mark: usize,
     pub offset: usize,
 }
@@ -11,15 +11,15 @@ pub struct TokenPosition {
 pub struct Token {
     pub kind: TokenKind,
     pub length: usize,
-    pub pos: TokenPosition,
+    pub location: Location,
 }
 
 impl Token {
     pub fn new(kind: TokenKind, start_pos: usize, end_pos: usize) -> Self {
         Self {
             kind,
             length: 0,
-            pos: TokenPosition { mark: 0, offset: 0 },
+            location: Location { mark: 0, offset: 0 },
         }
     }
 }
@@ -29,7 +29,7 @@ impl Default for Token {
         Self {
             kind: TokenKind::None,
             length: 0,
-            pos: TokenPosition { mark: 0, offset: 0 },
+            location: Location { mark: 0, offset: 0 },
         }
     }
 }

lsp/src/lsp.rs

@@ -182,22 +182,22 @@ impl LanguageServer for StcLsp {
         let mut last_offset = 0;
         let mut tokens: Vec<SemanticToken> = vec![];
         for tok in lexer.flatten() {
-            if tok.pos.mark != last_line {
+            if tok.location.mark != last_line {
                 last_offset = 0;
             }
 
             let (tt, tm) = semantic_token_type_id(&tok.kind);
             let token = SemanticToken {
-                delta_line: (tok.pos.mark - last_line) as u32,
-                delta_start: (tok.pos.offset - last_offset) as u32,
+                delta_line: (tok.location.mark - last_line) as u32,
+                delta_start: (tok.location.offset - last_offset) as u32,
                 length: tok.length as u32,
                 token_type: tt,
                 token_modifiers_bitset: tm,
             };
             tokens.push(token);
 
-            last_line = tok.pos.mark;
-            last_offset = tok.pos.offset;
+            last_line = tok.location.mark;
+            last_offset = tok.location.offset;
         }
 
         Ok(Some(SemanticTokensResult::Tokens(SemanticTokens {