From ad8e6ea4c82b5644a6d2d9047b6de52b1f7eace6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9F=B3=E5=8D=9A=E6=96=87?= Date: Sat, 13 Apr 2024 02:47:32 +0800 Subject: [PATCH] refactor: rename tok -> TokenKind --- lib/src/parser/lalrpop_impl/mod.rs | 4 +- lib/src/parser/lalrpop_impl/st.lalrpop | 144 +++++++++--------- lib/src/parser/lexer.rs | 186 +++++++++++------------ lib/src/parser/mod.rs | 8 +- lib/src/parser/operator.rs | 70 ++++----- lib/src/parser/token.rs | 196 ++++++++++++------------- lib/src/utils/stringify.rs | 46 +++--- lsp/src/lsp.rs | 28 ++-- 8 files changed, 341 insertions(+), 341 deletions(-) diff --git a/lib/src/parser/lalrpop_impl/mod.rs b/lib/src/parser/lalrpop_impl/mod.rs index 571517c..98cfd80 100644 --- a/lib/src/parser/lalrpop_impl/mod.rs +++ b/lib/src/parser/lalrpop_impl/mod.rs @@ -4,12 +4,12 @@ use lalrpop_util::lalrpop_mod; lalrpop_mod!(st, "/parser/lalrpop_impl/st.rs"); -type LalrPopLexerItem = (usize, Tok, usize); +type LalrPopLexerItem = (usize, TokenKind, usize); type LalrPopLexerResult = Result; impl From for LalrPopLexerItem { fn from(tok: Token) -> Self { - (tok.pos.line, tok.tok, tok.pos.offset) + (tok.pos.line, tok.kind, tok.pos.offset) } } diff --git a/lib/src/parser/lalrpop_impl/st.lalrpop b/lib/src/parser/lalrpop_impl/st.lalrpop index 07d31eb..272de02 100644 --- a/lib/src/parser/lalrpop_impl/st.lalrpop +++ b/lib/src/parser/lalrpop_impl/st.lalrpop @@ -9,60 +9,60 @@ extern { type Location = usize; type Error = lexer::LexicalError; - enum Tok { - "." => Tok::Access, - "+" => Tok::Plus, - "-" => Tok::Minus, - "*" => Tok::Multiply, - "**" => Tok::Power, - "/" => Tok::Division, - "(" => Tok::LeftParentheses, - ")" => Tok::RightParentheses, - "," => Tok::Comma, - ";" => Tok::Semicolon, - ":=" => Tok::Assign, - "=>" => Tok::AssignRight, - ":" => Tok::Colon, - "=" => Tok::Equal, - "<>" => Tok::NotEqual, - ">" => Tok::Greater, - ">=" => Tok::GreaterEqual, - "<" => Tok::Less, - "<=" => Tok::LessEqual, - "|" => Tok::BitOr, - "&" => Tok::BitAnd, - "XOR" => Tok::Xor, - "NOT" => Tok::Not, - "MOD" => Tok::Mod, - "IF" => Tok::If, - "THEN" => Tok::Then, - "ELSE" => Tok::Else, - "ELSEIF" => Tok::ElseIf, - "END_IF" => Tok::EndIf, - "FUNCTION" => Tok::Function, - "END_FUNCTION" => Tok::EndFunction, - "PROGRAM" => Tok::Program, - "END_PROGRAM" => Tok::EndProgram, - "STRUCT" => Tok::Struct, - "END_STRUCT" => Tok::EndStruct, - "VAR" => Tok::Var, - "VAR_GLOBAL" => Tok::VarGlobal, - "VAR_INPUT" => Tok::VarInput, - "VAR_INOUT" => Tok::VarInOut, - "VAR_OUTPUT" => Tok::VarOutput, - "VAR_TEMP" => Tok::VarTemp, - "VAR_STAT" => Tok::VarStat, - "END_VAR" => Tok::EndVar, - "RETAIN" => Tok::Retain, - "PERSISTENT" => Tok::Persistent, - "TYPE" => Tok::Type, - "END_TYPE" => Tok::EndType, - "INT" => Tok::Int, - "BOOL" => Tok::Bool, - "REAL" => Tok::Real, - "BYTE" => Tok::Byte, - "LITERAL" => Tok::Literal(), - "IDENTIFIER" => Tok::Identifier(), + enum TokenKind { + "." => TokenKind::Access, + "+" => TokenKind::Plus, + "-" => TokenKind::Minus, + "*" => TokenKind::Multiply, + "**" => TokenKind::Power, + "/" => TokenKind::Division, + "(" => TokenKind::LeftParentheses, + ")" => TokenKind::RightParentheses, + "," => TokenKind::Comma, + ";" => TokenKind::Semicolon, + ":=" => TokenKind::Assign, + "=>" => TokenKind::AssignRight, + ":" => TokenKind::Colon, + "=" => TokenKind::Equal, + "<>" => TokenKind::NotEqual, + ">" => TokenKind::Greater, + ">=" => TokenKind::GreaterEqual, + "<" => TokenKind::Less, + "<=" => TokenKind::LessEqual, + "|" => TokenKind::BitOr, + "&" => TokenKind::BitAnd, + "XOR" => TokenKind::Xor, + "NOT" => TokenKind::Not, + "MOD" => TokenKind::Mod, + "IF" => TokenKind::If, + "THEN" => TokenKind::Then, + "ELSE" => TokenKind::Else, + "ELSEIF" => TokenKind::ElseIf, + "END_IF" => TokenKind::EndIf, + "FUNCTION" => TokenKind::Function, + "END_FUNCTION" => TokenKind::EndFunction, + "PROGRAM" => TokenKind::Program, + "END_PROGRAM" => TokenKind::EndProgram, + "STRUCT" => TokenKind::Struct, + "END_STRUCT" => TokenKind::EndStruct, + "VAR" => TokenKind::Var, + "VAR_GLOBAL" => TokenKind::VarGlobal, + "VAR_INPUT" => TokenKind::VarInput, + "VAR_INOUT" => TokenKind::VarInOut, + "VAR_OUTPUT" => TokenKind::VarOutput, + "VAR_TEMP" => TokenKind::VarTemp, + "VAR_STAT" => TokenKind::VarStat, + "END_VAR" => TokenKind::EndVar, + "RETAIN" => TokenKind::Retain, + "PERSISTENT" => TokenKind::Persistent, + "TYPE" => TokenKind::Type, + "END_TYPE" => TokenKind::EndType, + "INT" => TokenKind::Int, + "BOOL" => TokenKind::Bool, + "REAL" => TokenKind::Real, + "BYTE" => TokenKind::Byte, + "LITERAL" => TokenKind::Literal(), + "IDENTIFIER" => TokenKind::Identifier(), } } @@ -158,9 +158,9 @@ BitAndExpr: Expression = { EquExpr, } -EquOp: Tok = { - "=" => Tok::Equal, - "<>" => Tok::NotEqual, +EquOp: TokenKind = { + "=" => TokenKind::Equal, + "<>" => TokenKind::NotEqual, } EquExpr: Expression = { @@ -168,11 +168,11 @@ EquExpr: Expression = { CmpExpr, } -CmpOp: Tok = { - ">" => Tok::Greater, - ">=" => Tok::GreaterEqual, - "<" => Tok::Less, - "<=" => Tok::LessEqual, +CmpOp: TokenKind = { + ">" => TokenKind::Greater, + ">=" => TokenKind::GreaterEqual, + "<" => TokenKind::Less, + "<=" => TokenKind::LessEqual, } CmpExpr: Expression = { @@ -181,9 +181,9 @@ CmpExpr: Expression = { } /// Low-priority op -ExprOp: Tok = { - "+" => Tok::Plus, - "-" => Tok::Minus, +ExprOp: TokenKind = { + "+" => TokenKind::Plus, + "-" => TokenKind::Minus, }; /// Low-priority op expression @@ -193,10 +193,10 @@ OpExpr: Expression = { } /// High-priority op -FactorOp: Tok = { - "*" => Tok::Multiply, - "/" => Tok::Division, - "MOD" => Tok::Mod, +FactorOp: TokenKind = { + "*" => TokenKind::Multiply, + "/" => TokenKind::Division, + "MOD" => TokenKind::Mod, }; /// High-priority factor @@ -211,9 +211,9 @@ PowerExpr: Expression = { } /// Top priority unary op -UnaryOp: Tok = { - "-" => Tok::Minus, - "NOT" => Tok::Not, +UnaryOp: TokenKind = { + "-" => TokenKind::Minus, + "NOT" => TokenKind::Not, }; /// Top priority unary factor diff --git a/lib/src/parser/lexer.rs b/lib/src/parser/lexer.rs index b3f30fb..73cbcf5 100644 --- a/lib/src/parser/lexer.rs +++ b/lib/src/parser/lexer.rs @@ -1,5 +1,5 @@ use crate::ast::*; -use crate::parser::{Buffer, StreamBuffer, StringBuffer, Tok}; +use crate::parser::{Buffer, StreamBuffer, StringBuffer, TokenKind}; use smallmap::Map; use std::cmp::Ordering; use std::fmt::{self, Display, Formatter}; @@ -42,15 +42,15 @@ pub struct TokenPosition { } pub struct Token { - pub tok: Tok, + pub kind: TokenKind, pub length: usize, pub pos: TokenPosition, } impl Token { - pub fn new(tok: Tok, start_pos: usize, end_pos: usize) -> Self { + pub fn new(kind: TokenKind, start_pos: usize, end_pos: usize) -> Self { Self { - tok, + kind, length: 0, pos: TokenPosition { line: 0, offset: 0 }, } @@ -60,7 +60,7 @@ impl Token { impl Default for Token { fn default() -> Self { Self { - tok: Tok::None, + kind: TokenKind::None, length: 0, pos: TokenPosition { line: 0, offset: 0 }, } @@ -121,8 +121,8 @@ impl StString { } } -impl From for StString { - fn from(value: Tok) -> Self { +impl From for StString { + fn from(value: TokenKind) -> Self { StString::Origin(Into::::into(&value)) } } @@ -239,20 +239,20 @@ impl LiteralValue { impl Display for LiteralValue { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self { - LiteralValue::Bit(BitValue::Zero) => write!(f, "{}#{}", Tok::Bit, 0), - LiteralValue::Bit(BitValue::One) => write!(f, "{}#{}", Tok::Bit, 1), - LiteralValue::Bool(x) => write!(f, "{}#{}", Tok::Bool, x), - LiteralValue::Int(x) => write!(f, "{}#{}", Tok::Int, x), - LiteralValue::UInt(x) => write!(f, "{}#{}", Tok::UInt, x), - LiteralValue::Byte(x) => write!(f, "{}#{}", Tok::Byte, x), - LiteralValue::SInt(x) => write!(f, "{}#{}", Tok::SInt, x), - LiteralValue::DInt(x) => write!(f, "{}#{}", Tok::DInt, x), - LiteralValue::UDInt(x) => write!(f, "{}#{}", Tok::UDInt, x), - LiteralValue::LInt(x) => write!(f, "{}#{}", Tok::LInt, x), - LiteralValue::ULInt(x) => write!(f, "{}#{}", Tok::ULInt, x), - LiteralValue::Real(x) => write!(f, "{}#{}", Tok::Real, x), - LiteralValue::LReal(x) => write!(f, "{}#{}", Tok::LReal, x), - LiteralValue::String(s) => write!(f, "{}#{}", Tok::String, s), + LiteralValue::Bit(BitValue::Zero) => write!(f, "{}#{}", TokenKind::Bit, 0), + LiteralValue::Bit(BitValue::One) => write!(f, "{}#{}", TokenKind::Bit, 1), + LiteralValue::Bool(x) => write!(f, "{}#{}", TokenKind::Bool, x), + LiteralValue::Int(x) => write!(f, "{}#{}", TokenKind::Int, x), + LiteralValue::UInt(x) => write!(f, "{}#{}", TokenKind::UInt, x), + LiteralValue::Byte(x) => write!(f, "{}#{}", TokenKind::Byte, x), + LiteralValue::SInt(x) => write!(f, "{}#{}", TokenKind::SInt, x), + LiteralValue::DInt(x) => write!(f, "{}#{}", TokenKind::DInt, x), + LiteralValue::UDInt(x) => write!(f, "{}#{}", TokenKind::UDInt, x), + LiteralValue::LInt(x) => write!(f, "{}#{}", TokenKind::LInt, x), + LiteralValue::ULInt(x) => write!(f, "{}#{}", TokenKind::ULInt, x), + LiteralValue::Real(x) => write!(f, "{}#{}", TokenKind::Real, x), + LiteralValue::LReal(x) => write!(f, "{}#{}", TokenKind::LReal, x), + LiteralValue::String(s) => write!(f, "{}#{}", TokenKind::String, s), } } } @@ -307,7 +307,7 @@ impl StLexerOptions { pub struct StLexer<'a> { buffer: Box, - keywords: Map, + keywords: Map, options: StLexerOptions, } @@ -325,7 +325,7 @@ macro_rules! keywords { pub struct StLexerBuilder { options: StLexerOptions, - keywords: Map, + keywords: Map, } impl Default for StLexerBuilder { @@ -353,38 +353,38 @@ impl StLexerBuilder { fn init(mut self) -> Self { let keywords = keywords![ - Tok::BitAnd, - Tok::BitOr, - Tok::Xor, - Tok::Mod, - Tok::Not, - Tok::If, - Tok::Else, - Tok::Then, - Tok::ElseIf, - Tok::EndIf, - Tok::Function, - Tok::EndFunction, - Tok::Program, - Tok::EndProgram, - Tok::Struct, - Tok::EndStruct, - Tok::Var, - Tok::VarInput, - Tok::VarInOut, - Tok::VarOutput, - Tok::VarGlobal, - Tok::VarTemp, - Tok::EndVar, - Tok::Retain, - Tok::Persistent, - Tok::Type, - Tok::EndType, - Tok::Int, - Tok::Bool, - Tok::Byte, - Tok::Real, - Tok::LReal + TokenKind::BitAnd, + TokenKind::BitOr, + TokenKind::Xor, + TokenKind::Mod, + TokenKind::Not, + TokenKind::If, + TokenKind::Else, + TokenKind::Then, + TokenKind::ElseIf, + TokenKind::EndIf, + TokenKind::Function, + TokenKind::EndFunction, + TokenKind::Program, + TokenKind::EndProgram, + TokenKind::Struct, + TokenKind::EndStruct, + TokenKind::Var, + TokenKind::VarInput, + TokenKind::VarInOut, + TokenKind::VarOutput, + TokenKind::VarGlobal, + TokenKind::VarTemp, + TokenKind::EndVar, + TokenKind::Retain, + TokenKind::Persistent, + TokenKind::Type, + TokenKind::EndType, + TokenKind::Int, + TokenKind::Bool, + TokenKind::Byte, + TokenKind::Real, + TokenKind::LReal ]; self.keywords = keywords; @@ -416,7 +416,7 @@ impl<'input> StLexer<'input> { let start_with_zero = ch == '0'; if start_with_zero && self.buffer.peek1() != Some('.') { - tok.tok = Tok::Literal(LiteralValue::Bit(BitValue::Zero)); + tok.kind = TokenKind::Literal(LiteralValue::Bit(BitValue::Zero)); return Some(Ok(tok)); } @@ -431,7 +431,7 @@ impl<'input> StLexer<'input> { } _ => { tok.length = s.len(); - tok.tok = Tok::Literal(LiteralValue::UInt(s.parse().unwrap())); + tok.kind = TokenKind::Literal(LiteralValue::UInt(s.parse().unwrap())); return Some(Ok(tok)); } } @@ -454,7 +454,7 @@ impl<'input> StLexer<'input> { } _ => { tok.length = s.len(); - tok.tok = Tok::Literal(LiteralValue::LReal(s)); + tok.kind = TokenKind::Literal(LiteralValue::LReal(s)); return Some(Ok(tok)); } } @@ -492,7 +492,7 @@ impl<'input> StLexer<'input> { } Some('\"') => { self.buffer.consume1(); - tok.tok = Tok::Literal(LiteralValue::String(s)); + tok.kind = TokenKind::Literal(LiteralValue::String(s)); return Some(Ok(tok)); } Some(c) => { @@ -532,7 +532,7 @@ impl<'input> StLexer<'input> { } x => { tok.length = str.len(); - tok.tok = self.keywords_or_identifier(str); + tok.kind = self.keywords_or_identifier(str); return Some(Ok(tok)); } } @@ -540,7 +540,7 @@ impl<'input> StLexer<'input> { } fn parse_whitespace(&mut self, mut tok: Token) -> LexerResult { - tok.tok = Tok::Whitespace; + tok.kind = TokenKind::Whitespace; loop { match self.buffer.peek1() { @@ -559,34 +559,34 @@ impl<'input> StLexer<'input> { tok.length = 2; match (ch, self.buffer.peek1()) { - ('<', Some('=')) => tok.tok = Tok::LessEqual, - ('<', Some('>')) => tok.tok = Tok::NotEqual, + ('<', Some('=')) => tok.kind = TokenKind::LessEqual, + ('<', Some('>')) => tok.kind = TokenKind::NotEqual, ('<', _) => { - tok.tok = Tok::Less; + tok.kind = TokenKind::Less; tok.length = 1; } - ('>', Some('=')) => tok.tok = Tok::GreaterEqual, + ('>', Some('=')) => tok.kind = TokenKind::GreaterEqual, ('>', _) => { - tok.tok = Tok::Greater; + tok.kind = TokenKind::Greater; tok.length = 1; } - (':', Some('=')) => tok.tok = Tok::Assign, + (':', Some('=')) => tok.kind = TokenKind::Assign, (':', _) => { - tok.tok = Tok::Colon; + tok.kind = TokenKind::Colon; tok.length = 1; } - ('*', Some('*')) => tok.tok = Tok::Power, + ('*', Some('*')) => tok.kind = TokenKind::Power, ('*', _) => { - tok.tok = Tok::Multiply; + tok.kind = TokenKind::Multiply; tok.length = 1; } - ('=', Some('>')) => tok.tok = Tok::AssignRight, + ('=', Some('>')) => tok.kind = TokenKind::AssignRight, ('=', _) => { - tok.tok = Tok::Equal; + tok.kind = TokenKind::Equal; tok.length = 1; } @@ -600,14 +600,14 @@ impl<'input> StLexer<'input> { Some(Ok(tok)) } - fn keywords_or_identifier(&mut self, s: String) -> Tok { + fn keywords_or_identifier(&mut self, s: String) -> TokenKind { let st_str = s.into(); if let Some(keyword) = self.keywords.get(&st_str) { return keyword.clone(); } - Tok::Identifier(st_str) + TokenKind::Identifier(st_str) } fn is_valid_identifier_character(&self, ch: char) -> bool { @@ -643,47 +643,47 @@ impl<'input> StLexer<'input> { } Some('.') => { self.buffer.consume1(); - tok.tok = Tok::Access; + tok.kind = TokenKind::Access; Some(Ok(tok)) } Some('+') => { self.buffer.consume1(); - tok.tok = Tok::Plus; + tok.kind = TokenKind::Plus; Some(Ok(tok)) } Some('-') => { self.buffer.consume1(); - tok.tok = Tok::Minus; + tok.kind = TokenKind::Minus; Some(Ok(tok)) } Some('/') => { self.buffer.consume1(); - tok.tok = Tok::Division; + tok.kind = TokenKind::Division; Some(Ok(tok)) } Some('(') => { self.buffer.consume1(); - tok.tok = Tok::LeftParentheses; + tok.kind = TokenKind::LeftParentheses; Some(Ok(tok)) } Some(')') => { self.buffer.consume1(); - tok.tok = Tok::RightParentheses; + tok.kind = TokenKind::RightParentheses; Some(Ok(tok)) } Some(',') => { self.buffer.consume1(); - tok.tok = Tok::Comma; + tok.kind = TokenKind::Comma; Some(Ok(tok)) } Some(';') => { self.buffer.consume1(); - tok.tok = Tok::Semicolon; + tok.kind = TokenKind::Semicolon; Some(Ok(tok)) } Some('&') => { self.buffer.consume1(); - tok.tok = Tok::BitAnd; + tok.kind = TokenKind::BitAnd; Some(Ok(tok)) } Some('\"') => { @@ -722,7 +722,7 @@ impl Iterator for StLexer<'_> { loop { match self.next_raw() { - Some(Ok(tok)) if matches!(tok.tok, Tok::Whitespace) => {} + Some(Ok(tok)) if matches!(tok.kind, TokenKind::Whitespace) => {} x => return x, } } @@ -779,12 +779,12 @@ mod test { assert_eq!(x.pos.offset, 0); assert_eq!(x.pos.line, 0); assert_eq!(x.length, 2); - assert_eq!(x.tok, Tok::If); + assert_eq!(x.kind, TokenKind::If); let x = lexer.next().unwrap().unwrap(); assert_eq!(x.pos.offset, 3); assert_eq!(x.length, 3); - assert!(matches!(x.tok, Tok::Identifier(_))); + assert!(matches!(x.kind, TokenKind::Identifier(_))); assert!(lexer.next().is_none()); } @@ -797,18 +797,18 @@ mod test { let x = lexer.next().unwrap().unwrap(); assert_eq!(x.pos.offset, 0); assert_eq!(x.length, 1); - assert!(matches!(x.tok, Tok::Literal(_))); + assert!(matches!(x.kind, TokenKind::Literal(_))); let x = lexer.next().unwrap().unwrap(); assert_eq!(x.pos.offset, 2); assert_eq!(x.length, 1); - assert!(matches!(x.tok, Tok::Plus)); + assert!(matches!(x.kind, TokenKind::Plus)); let x = lexer.next().unwrap().unwrap(); assert_eq!(x.pos.line, 1); assert_eq!(x.pos.offset, 0); assert_eq!(x.length, 1); - assert!(matches!(x.tok, Tok::Identifier(_))); + assert!(matches!(x.kind, TokenKind::Identifier(_))); } #[test] @@ -819,7 +819,7 @@ mod test { let x = lexer.next().unwrap().unwrap(); assert_eq!(x.pos.offset, 0); assert_eq!(x.length, 6); - assert!(matches!(x.tok, Tok::Identifier(_))); + assert!(matches!(x.kind, TokenKind::Identifier(_))); } #[test] @@ -842,8 +842,8 @@ mod test { let mut lexer = StLexerBuilder::new().build_str(s); assert!(matches!( - lexer.next().unwrap().unwrap().tok, - Tok::AssignRight + lexer.next().unwrap().unwrap().kind, + TokenKind::AssignRight )); } @@ -855,6 +855,6 @@ mod test { let x = lexer.next().unwrap().unwrap(); assert_eq!(x.pos.offset, 0); assert_eq!(x.length, 5); - assert!(matches!(x.tok, Tok::Literal(LiteralValue::LReal(_)))); + assert!(matches!(x.kind, TokenKind::Literal(LiteralValue::LReal(_)))); } } diff --git a/lib/src/parser/mod.rs b/lib/src/parser/mod.rs index e926827..3086086 100644 --- a/lib/src/parser/mod.rs +++ b/lib/src/parser/mod.rs @@ -8,7 +8,7 @@ mod operator; pub use operator::Operator; mod token; -pub use token::Tok; +pub use token::TokenKind; #[derive(Clone, Debug)] pub enum ParseError { @@ -20,7 +20,7 @@ pub enum ParseError { } impl ParseError { - pub fn expect_tokens(pos: usize, tokens: &[Tok]) -> Self { + pub fn expect_tokens(pos: usize, tokens: &[TokenKind]) -> Self { let tokens: Vec<_> = tokens.iter().map(|x| x.into()).collect(); Self::UnexpectedToken(pos, tokens) @@ -28,8 +28,8 @@ impl ParseError { } #[cfg(feature = "use_lalrpop")] -impl From> for ParseError { - fn from(e: lalrpop_util::ParseError) -> Self { +impl From> for ParseError { + fn from(e: lalrpop_util::ParseError) -> Self { match e { lalrpop_util::ParseError::InvalidToken { location: loc } => { ParseError::InvalidToken(loc) diff --git a/lib/src/parser/operator.rs b/lib/src/parser/operator.rs index 2368652..85587db 100644 --- a/lib/src/parser/operator.rs +++ b/lib/src/parser/operator.rs @@ -1,4 +1,4 @@ -use crate::parser::Tok; +use crate::parser::TokenKind; use std::fmt::{Display, Formatter}; #[derive(Debug, Clone, Copy, Hash)] @@ -48,25 +48,25 @@ impl Operator { } } -impl From for Operator { - fn from(value: Tok) -> Self { +impl From for Operator { + fn from(value: TokenKind) -> Self { match value { - Tok::Plus => Operator::Plus, - Tok::Minus => Operator::Minus, - Tok::Not => Operator::Not, - Tok::Multiply => Operator::Multiply, - Tok::Division => Operator::Division, - Tok::BitOr => Operator::BitOr, - Tok::BitAnd => Operator::BitAnd, - Tok::Mod => Operator::Mod, - Tok::Power => Operator::Power, - Tok::Xor => Operator::Xor, - Tok::Less => Operator::Less, - Tok::LessEqual => Operator::LessEqual, - Tok::Equal => Operator::Equal, - Tok::NotEqual => Operator::NotEqual, - Tok::Greater => Operator::Greater, - Tok::GreaterEqual => Operator::GreaterEqual, + TokenKind::Plus => Operator::Plus, + TokenKind::Minus => Operator::Minus, + TokenKind::Not => Operator::Not, + TokenKind::Multiply => Operator::Multiply, + TokenKind::Division => Operator::Division, + TokenKind::BitOr => Operator::BitOr, + TokenKind::BitAnd => Operator::BitAnd, + TokenKind::Mod => Operator::Mod, + TokenKind::Power => Operator::Power, + TokenKind::Xor => Operator::Xor, + TokenKind::Less => Operator::Less, + TokenKind::LessEqual => Operator::LessEqual, + TokenKind::Equal => Operator::Equal, + TokenKind::NotEqual => Operator::NotEqual, + TokenKind::Greater => Operator::Greater, + TokenKind::GreaterEqual => Operator::GreaterEqual, _ => unreachable!(), } } @@ -75,22 +75,22 @@ impl From for Operator { impl Display for Operator { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match *self { - Operator::Plus => f.write_fmt(format_args!("{}", Tok::Plus)), - Operator::Minus => f.write_fmt(format_args!("{}", Tok::Minus)), - Operator::Not => f.write_fmt(format_args!("{}", Tok::Not)), - Operator::Multiply => f.write_fmt(format_args!("{}", Tok::Multiply)), - Operator::Division => f.write_fmt(format_args!("{}", Tok::Division)), - Operator::BitOr => f.write_fmt(format_args!("{}", Tok::BitOr)), - Operator::BitAnd => f.write_fmt(format_args!("{}", Tok::BitAnd)), - Operator::Mod => f.write_fmt(format_args!("{}", Tok::Mod)), - Operator::Power => f.write_fmt(format_args!("{}", Tok::Power)), - Operator::Xor => f.write_fmt(format_args!("{}", Tok::Xor)), - Operator::Less => f.write_fmt(format_args!("{}", Tok::Less)), - Operator::LessEqual => f.write_fmt(format_args!("{}", Tok::LessEqual)), - Operator::Equal => f.write_fmt(format_args!("{}", Tok::Equal)), - Operator::NotEqual => f.write_fmt(format_args!("{}", Tok::NotEqual)), - Operator::Greater => f.write_fmt(format_args!("{}", Tok::Greater)), - Operator::GreaterEqual => f.write_fmt(format_args!("{}", Tok::GreaterEqual)), + Operator::Plus => f.write_fmt(format_args!("{}", TokenKind::Plus)), + Operator::Minus => f.write_fmt(format_args!("{}", TokenKind::Minus)), + Operator::Not => f.write_fmt(format_args!("{}", TokenKind::Not)), + Operator::Multiply => f.write_fmt(format_args!("{}", TokenKind::Multiply)), + Operator::Division => f.write_fmt(format_args!("{}", TokenKind::Division)), + Operator::BitOr => f.write_fmt(format_args!("{}", TokenKind::BitOr)), + Operator::BitAnd => f.write_fmt(format_args!("{}", TokenKind::BitAnd)), + Operator::Mod => f.write_fmt(format_args!("{}", TokenKind::Mod)), + Operator::Power => f.write_fmt(format_args!("{}", TokenKind::Power)), + Operator::Xor => f.write_fmt(format_args!("{}", TokenKind::Xor)), + Operator::Less => f.write_fmt(format_args!("{}", TokenKind::Less)), + Operator::LessEqual => f.write_fmt(format_args!("{}", TokenKind::LessEqual)), + Operator::Equal => f.write_fmt(format_args!("{}", TokenKind::Equal)), + Operator::NotEqual => f.write_fmt(format_args!("{}", TokenKind::NotEqual)), + Operator::Greater => f.write_fmt(format_args!("{}", TokenKind::Greater)), + Operator::GreaterEqual => f.write_fmt(format_args!("{}", TokenKind::GreaterEqual)), } } } diff --git a/lib/src/parser/token.rs b/lib/src/parser/token.rs index 20a9190..0846d78 100644 --- a/lib/src/parser/token.rs +++ b/lib/src/parser/token.rs @@ -2,7 +2,7 @@ use crate::parser::*; use std::fmt::{self, Display, Formatter}; #[derive(Debug, Clone, Eq, PartialEq)] -pub enum Tok { +pub enum TokenKind { None, /// ' ' or '\n', etc. Whitespace, @@ -154,128 +154,128 @@ pub enum Tok { Identifier(StString), } -impl Tok { +impl TokenKind { pub fn is_type(&self) -> bool { - matches!(self, Tok::Int | Tok::Bool) + matches!(self, TokenKind::Int | TokenKind::Bool) } pub fn is_operator(&self) -> bool { matches!( self, - Tok::Less - | Tok::LessEqual - | Tok::Greater - | Tok::GreaterEqual - | Tok::Equal - | Tok::NotEqual - | Tok::Plus - | Tok::Minus - | Tok::Division - | Tok::Multiply - | Tok::BitOr - | Tok::BitAnd - | Tok::Mod - | Tok::Power - | Tok::Not - | Tok::Xor + TokenKind::Less + | TokenKind::LessEqual + | TokenKind::Greater + | TokenKind::GreaterEqual + | TokenKind::Equal + | TokenKind::NotEqual + | TokenKind::Plus + | TokenKind::Minus + | TokenKind::Division + | TokenKind::Multiply + | TokenKind::BitOr + | TokenKind::BitAnd + | TokenKind::Mod + | TokenKind::Power + | TokenKind::Not + | TokenKind::Xor ) } pub fn is_literal(&self) -> bool { - matches!(self, Tok::Literal(_)) + matches!(self, TokenKind::Literal(_)) } } -impl From<&Tok> for String { - fn from(value: &Tok) -> Self { +impl From<&TokenKind> for String { + fn from(value: &TokenKind) -> Self { let tmp_string; let s = match value { - Tok::None => "!!!NONE!!!", - Tok::Whitespace => " ", - Tok::Access => ".", - Tok::Plus => "+", - Tok::Minus => "-", - Tok::Multiply => "*", - Tok::Power => "**", - Tok::Division => "/", - Tok::LeftParentheses => "(", - Tok::RightParentheses => ")", - Tok::Comma => ",", - Tok::Semicolon => ";", - Tok::Colon => ":", - Tok::Assign => ":=", - Tok::AssignRight => "=>", - Tok::AssignSet => "S=", - Tok::AssignReset => "R=", - Tok::Equal => "=", - Tok::NotEqual => "<>", - Tok::Greater => ">", - Tok::GreaterEqual => ">=", - Tok::Less => "<", - Tok::LessEqual => "<=", - Tok::BitOr => "OR", - Tok::BitAnd => "AND", - Tok::Deref => "^", - Tok::Mod => "MOD", - Tok::Xor => "XOR", - Tok::Not => "NOT", - Tok::Pointer => "POINTER", - Tok::Array => "ARRAY", - Tok::Of => "OF", - Tok::To => "TO", - Tok::If => "IF", - Tok::Then => "THEN", - Tok::Else => "ELSE", - Tok::ElseIf => "ELSEIF", - Tok::EndIf => "END_IF", - Tok::Function => "FUNCTION", - Tok::EndFunction => "END_FUNCTION", - Tok::Program => "PROGRAM", - Tok::EndProgram => "END_PROGRAM", - Tok::FunctionBlock => "FUNCTION_BLOCK", - Tok::EndFunctionBlock => "END_FUNCTION_BLOCK", - Tok::Struct => "STRUCT", - Tok::EndStruct => "END_STRUCT", - Tok::VarGlobal => "VAR_GLOBAL", - Tok::Var => "VAR", - Tok::VarInput => "VAR_INPUT", - Tok::VarInOut => "VAR_INOUT", - Tok::VarOutput => "VAR_OUTPUT", - Tok::VarTemp => "VAR_TEMP", - Tok::VarStat => "VAR_STAT", - Tok::EndVar => "END_VAR", - Tok::Retain => "RETAIN", - Tok::Persistent => "PERSISTENT", - Tok::Type => "TYPE", - Tok::EndType => "END_TYPE", - Tok::Int => "INT", - Tok::Real => "REAL", - Tok::LReal => "LREAL", - Tok::Bit => "BIT", - Tok::Bool => "BOOL", - Tok::SInt => "SINT", - Tok::Byte => "BYTE", - Tok::UInt => "UINT", - Tok::DInt => "DINT", - Tok::UDInt => "UDINT", - Tok::LInt => "LINT", - Tok::ULInt => "ULINT", - Tok::Time => "TIME", - Tok::LTime => "LTIME", - Tok::String => "STRING", - Tok::Literal(x) => { + TokenKind::None => "!!!NONE!!!", + TokenKind::Whitespace => " ", + TokenKind::Access => ".", + TokenKind::Plus => "+", + TokenKind::Minus => "-", + TokenKind::Multiply => "*", + TokenKind::Power => "**", + TokenKind::Division => "/", + TokenKind::LeftParentheses => "(", + TokenKind::RightParentheses => ")", + TokenKind::Comma => ",", + TokenKind::Semicolon => ";", + TokenKind::Colon => ":", + TokenKind::Assign => ":=", + TokenKind::AssignRight => "=>", + TokenKind::AssignSet => "S=", + TokenKind::AssignReset => "R=", + TokenKind::Equal => "=", + TokenKind::NotEqual => "<>", + TokenKind::Greater => ">", + TokenKind::GreaterEqual => ">=", + TokenKind::Less => "<", + TokenKind::LessEqual => "<=", + TokenKind::BitOr => "OR", + TokenKind::BitAnd => "AND", + TokenKind::Deref => "^", + TokenKind::Mod => "MOD", + TokenKind::Xor => "XOR", + TokenKind::Not => "NOT", + TokenKind::Pointer => "POINTER", + TokenKind::Array => "ARRAY", + TokenKind::Of => "OF", + TokenKind::To => "TO", + TokenKind::If => "IF", + TokenKind::Then => "THEN", + TokenKind::Else => "ELSE", + TokenKind::ElseIf => "ELSEIF", + TokenKind::EndIf => "END_IF", + TokenKind::Function => "FUNCTION", + TokenKind::EndFunction => "END_FUNCTION", + TokenKind::Program => "PROGRAM", + TokenKind::EndProgram => "END_PROGRAM", + TokenKind::FunctionBlock => "FUNCTION_BLOCK", + TokenKind::EndFunctionBlock => "END_FUNCTION_BLOCK", + TokenKind::Struct => "STRUCT", + TokenKind::EndStruct => "END_STRUCT", + TokenKind::VarGlobal => "VAR_GLOBAL", + TokenKind::Var => "VAR", + TokenKind::VarInput => "VAR_INPUT", + TokenKind::VarInOut => "VAR_INOUT", + TokenKind::VarOutput => "VAR_OUTPUT", + TokenKind::VarTemp => "VAR_TEMP", + TokenKind::VarStat => "VAR_STAT", + TokenKind::EndVar => "END_VAR", + TokenKind::Retain => "RETAIN", + TokenKind::Persistent => "PERSISTENT", + TokenKind::Type => "TYPE", + TokenKind::EndType => "END_TYPE", + TokenKind::Int => "INT", + TokenKind::Real => "REAL", + TokenKind::LReal => "LREAL", + TokenKind::Bit => "BIT", + TokenKind::Bool => "BOOL", + TokenKind::SInt => "SINT", + TokenKind::Byte => "BYTE", + TokenKind::UInt => "UINT", + TokenKind::DInt => "DINT", + TokenKind::UDInt => "UDINT", + TokenKind::LInt => "LINT", + TokenKind::ULInt => "ULINT", + TokenKind::Time => "TIME", + TokenKind::LTime => "LTIME", + TokenKind::String => "STRING", + TokenKind::Literal(x) => { tmp_string = format!("{}", x); tmp_string.as_str() } - Tok::Identifier(s) => s.origin_string(), + TokenKind::Identifier(s) => s.origin_string(), }; s.to_owned() } } -impl Display for Tok { +impl Display for TokenKind { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { write!(f, "{}", Into::::into(self)) } diff --git a/lib/src/utils/stringify.rs b/lib/src/utils/stringify.rs index cc35041..f48f026 100644 --- a/lib/src/utils/stringify.rs +++ b/lib/src/utils/stringify.rs @@ -1,5 +1,5 @@ use crate::ast::*; -use crate::parser::{BitValue, LiteralValue, Operator, Tok}; +use crate::parser::{BitValue, LiteralValue, Operator, TokenKind}; use std::fmt::Arguments; use std::io::Write; @@ -66,7 +66,7 @@ impl StringifyVisitor { impl DeclVisitor<'_> for StringifyVisitor { fn visit_function_declaration(&mut self, fun: &FunctionDeclare) { - self.write(format_args!("{} : ", Tok::Function)); + self.write(format_args!("{} : ", TokenKind::Function)); if let Some(ret_type) = fun.return_type() { self.write(format_args!("{} ", ret_type)) } @@ -80,7 +80,7 @@ impl DeclVisitor<'_> for StringifyVisitor { // new group if current_scope != Some(v.scope()) { if current_scope.is_some() { - self.writeln(format_args!("{}", Tok::EndVar)); + self.writeln(format_args!("{}", TokenKind::EndVar)); } self.writeln(format_args!("{}", variable_scope_start_tok(v.scope()))); @@ -101,21 +101,21 @@ impl DeclVisitor<'_> for StringifyVisitor { // last group end if current_scope.is_some() { - self.writeln(format_args!("{}", Tok::EndVar)); + self.writeln(format_args!("{}", TokenKind::EndVar)); } } - self.writeln(format_args!("{}", Tok::EndFunction)); + self.writeln(format_args!("{}", TokenKind::EndFunction)); } fn visit_enum_declaration(&mut self, decl: &'_ EnumDeclare) { self.writeln(format_args!( "{} {} {}", - Tok::Type, + TokenKind::Type, decl.name().origin_string(), - Tok::Colon + TokenKind::Colon )); - self.writeln(format_args!("{}", Tok::LeftParentheses)); + self.writeln(format_args!("{}", TokenKind::LeftParentheses)); // fields self.indent += 1; @@ -124,26 +124,26 @@ impl DeclVisitor<'_> for StringifyVisitor { self.write_indent(); self.write(format_args!("{}", field.name().origin_string())); if let Some(val) = field.initial() { - self.write(format_args!(" {} ", Tok::Assign)); + self.write(format_args!(" {} ", TokenKind::Assign)); self.visit_expression(val); } if field_count == index + 1 { self.writeln(format_args!("")); } else { - self.writeln(format_args!("{}", Tok::Comma)) + self.writeln(format_args!("{}", TokenKind::Comma)) } } self.indent -= 1; // closed type, like: ) DINT; - self.write(format_args!("{}", Tok::RightParentheses)); + self.write(format_args!("{}", TokenKind::RightParentheses)); if let Some(ty) = decl.ty() { self.write(format_args!(" {}", ty)); } - self.writeln(format_args!("{}", Tok::Semicolon)); + self.writeln(format_args!("{}", TokenKind::Semicolon)); - self.writeln(format_args!("{}", Tok::EndType)) + self.writeln(format_args!("{}", TokenKind::EndType)) } } impl AstVisitor<'_> for StringifyVisitor { @@ -173,11 +173,11 @@ impl AstVisitor<'_> for StringifyVisitor { fn visit_call_expression(&mut self, call: &'_ CallExpression) { self.visit_expression(call.callee()); - self.write(format_args!("{}", Tok::LeftParentheses)); + self.write(format_args!("{}", TokenKind::LeftParentheses)); let mut first = true; for arg in call.arguments() { if !first { - self.write(format_args!("{} ", Tok::Comma)); + self.write(format_args!("{} ", TokenKind::Comma)); } self.visit_expression(arg); @@ -187,7 +187,7 @@ impl AstVisitor<'_> for StringifyVisitor { } } - self.write(format_args!("{}", Tok::RightParentheses)); + self.write(format_args!("{}", TokenKind::RightParentheses)); } fn visit_expr_statement(&mut self, stmt: &ExprStatement) { @@ -243,8 +243,8 @@ impl AstVisitor<'_> for StringifyVisitor { if operands.len() == 1 { match op { - Operator::Not => self.write(format_args!("{} ", Tok::Not)), - Operator::Minus => self.write(format_args!("{}", Tok::Minus)), + Operator::Not => self.write(format_args!("{} ", TokenKind::Not)), + Operator::Minus => self.write(format_args!("{}", TokenKind::Minus)), _ => panic!("invalid unary operator!"), }; @@ -272,21 +272,21 @@ impl AstVisitor<'_> for StringifyVisitor { fn visit_assign_expression(&mut self, assign: &AssignExpression) { self.visit_expression(assign.left()); - self.write(format_args!(" {} ", Tok::Assign)); + self.write(format_args!(" {} ", TokenKind::Assign)); self.visit_expression(assign.right()); } fn visit_compo_access_expression(&mut self, compo: &CompoAccessExpression) { self.visit_expression(compo.left()); - self.write(format_args!("{}", Tok::Access)); + self.write(format_args!("{}", TokenKind::Access)); self.visit_expression(compo.right()); } } -fn variable_scope_start_tok(class: VariableFlags) -> Tok { +fn variable_scope_start_tok(class: VariableFlags) -> TokenKind { match class { - VariableFlags::NONE => Tok::Var, - VariableFlags::GLOBAL => Tok::VarGlobal, + VariableFlags::NONE => TokenKind::Var, + VariableFlags::GLOBAL => TokenKind::VarGlobal, _ => unimplemented!(), } } diff --git a/lsp/src/lsp.rs b/lsp/src/lsp.rs index bbecb38..22a4d3c 100644 --- a/lsp/src/lsp.rs +++ b/lsp/src/lsp.rs @@ -1,7 +1,7 @@ use crate::lsp_types::{TokenModifiers, TokenTypes}; use serde_json::Value; -use stc::parser::{StLexerBuilder, Tok}; +use stc::parser::{StLexerBuilder, TokenKind}; use strum::IntoEnumIterator; use tower_lsp::jsonrpc; use tower_lsp::jsonrpc::Result; @@ -9,23 +9,23 @@ use tower_lsp::lsp_types::*; use tower_lsp::{Client, LanguageServer}; use tracing::*; -fn semantic_token_type_id(tok: &Tok) -> (u32, u32) { +fn semantic_token_type_id(tok: &TokenKind) -> (u32, u32) { match tok { - Tok::Identifier(_) => (TokenTypes::Variable as u32, TokenModifiers::None as u32), - Tok::Literal(_) => (TokenTypes::Number as u32, TokenModifiers::None as u32), - Tok::String => (TokenTypes::String as u32, TokenModifiers::None as u32), + TokenKind::Identifier(_) => (TokenTypes::Variable as u32, TokenModifiers::None as u32), + TokenKind::Literal(_) => (TokenTypes::Number as u32, TokenModifiers::None as u32), + TokenKind::String => (TokenTypes::String as u32, TokenModifiers::None as u32), // operators op if op.is_operator() => (TokenTypes::Operator as u32, TokenModifiers::None as u32), // builtin-types - Tok::Int => (TokenTypes::Type as u32, TokenModifiers::None as u32), + TokenKind::Int => (TokenTypes::Type as u32, TokenModifiers::None as u32), // keywords - Tok::If - | Tok::Then - | Tok::EndIf - | Tok::Var - | Tok::EndVar - | Tok::Program - | Tok::EndProgram => (TokenTypes::Keyword as u32, TokenModifiers::None as u32), + TokenKind::If + | TokenKind::Then + | TokenKind::EndIf + | TokenKind::Var + | TokenKind::EndVar + | TokenKind::Program + | TokenKind::EndProgram => (TokenTypes::Keyword as u32, TokenModifiers::None as u32), _ => (TokenTypes::None as u32, TokenModifiers::None as u32), } } @@ -121,7 +121,7 @@ impl LanguageServer for StcLsp { last_offset = 0; } - let (tt, tm) = semantic_token_type_id(&tok.tok); + let (tt, tm) = semantic_token_type_id(&tok.kind); let token = SemanticToken { delta_line: (tok.pos.line - last_line) as u32, delta_start: (tok.pos.offset - last_offset) as u32,