From a938d3c6fe74c3e251261ac42bc08467687dc953 Mon Sep 17 00:00:00 2001 From: vighnesh153 Date: Sat, 10 Feb 2024 20:40:11 +0530 Subject: [PATCH] feat: add lexing for != and && and || and ^= --- .../javalang/common/lexer/Lexer.kt | 52 +++++++++++++++++-- .../javalang/common/tokens/TokenType.kt | 2 +- .../javalang/common/lexer/LexerTest.kt | 50 ++++++++++++++++++ 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt b/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt index 7cdf0fb7..6c8aca93 100644 --- a/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt +++ b/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt @@ -114,10 +114,54 @@ fun Lexer.nextToken(): Token { } } - '!' -> t = Token(tokenType = TokenType.BANG, tokenLiteral = TokenType.BANG.value) - '&' -> t = Token(tokenType = TokenType.AMPERSAND, tokenLiteral = TokenType.AMPERSAND.value) - '|' -> t = Token(tokenType = TokenType.VERTICAL_BAR, tokenLiteral = TokenType.VERTICAL_BAR.value) - '^' -> t = Token(tokenType = TokenType.CARET, tokenLiteral = TokenType.CARET.value) + '!' -> { + val peek = peekCharacter() + t = when (peek) { + '=' -> { + readNextCharacter() + Token(tokenType = TokenType.BANG_EQUALS, tokenLiteral = TokenType.BANG_EQUALS.value) + } + + else -> Token(tokenType = TokenType.BANG, tokenLiteral = TokenType.BANG.value) + } + } + + '&' -> { + val peek = peekCharacter() + t = when (peek) { + '&' -> { + readNextCharacter() + Token(tokenType = TokenType.DOUBLE_AMPERSAND, tokenLiteral = TokenType.DOUBLE_AMPERSAND.value) + } + + else -> Token(tokenType = TokenType.AMPERSAND, tokenLiteral = TokenType.AMPERSAND.value) + } + } + + '|' -> { + val peek = peekCharacter() + t = when (peek) { + '|' -> { + readNextCharacter() + Token(tokenType = TokenType.DOUBLE_VERTICAL_BAR, tokenLiteral = TokenType.DOUBLE_VERTICAL_BAR.value) + } + + else -> Token(tokenType = TokenType.VERTICAL_BAR, tokenLiteral = TokenType.VERTICAL_BAR.value) + } + } + + '^' -> { + val peek = peekCharacter() + t = when (peek) { + '=' -> { + readNextCharacter() + Token(tokenType = TokenType.CARET_EQUALS, tokenLiteral = TokenType.CARET_EQUALS.value) + } + + else -> Token(tokenType = TokenType.CARET, tokenLiteral = TokenType.CARET.value) + } + } + '?' -> t = Token(tokenType = TokenType.QUESTION, tokenLiteral = TokenType.QUESTION.value) ':' -> t = Token(tokenType = TokenType.COLON, tokenLiteral = TokenType.COLON.value) '.' -> t = Token(tokenType = TokenType.DOT, tokenLiteral = TokenType.DOT.value) diff --git a/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt b/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt index 3dd062f1..eb0f932d 100644 --- a/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt +++ b/jvm-tools/interpreters/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt @@ -40,7 +40,7 @@ enum class TokenType(val value: String) { MODULUS("%"), MODULUS_EQUALS("%="), BANG("!"), - NOT_EQUALS("!="), + BANG_EQUALS("!="), AMPERSAND("&"), DOUBLE_AMPERSAND("&&"), diff --git a/jvm-tools/interpreters/languages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt b/jvm-tools/interpreters/languages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt index b1ad1c6e..8dfff836 100644 --- a/jvm-tools/interpreters/languages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt +++ b/jvm-tools/interpreters/languages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt @@ -32,6 +32,14 @@ a /= b/c = d a %= b %c = d +a!=b !c =d + +a &&b &c&&d &e + +a|b||c||d|e + +a^b^c^=d^e + """.trimIndent() val expectedTokens = listOf( @@ -166,6 +174,48 @@ a %= b %c = d ExpectedToken(id = 74, tokenType = TokenType.EQUALS, tokenLiteral = "="), ExpectedToken(id = 75, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"), + // a!=b !c =d + ExpectedToken(id = 76, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"), + ExpectedToken(id = 77, tokenType = TokenType.BANG_EQUALS, tokenLiteral = "!="), + ExpectedToken(id = 78, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"), + ExpectedToken(id = 79, tokenType = TokenType.BANG, tokenLiteral = "!"), + ExpectedToken(id = 80, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"), + ExpectedToken(id = 81, tokenType = TokenType.EQUALS, tokenLiteral = "="), + ExpectedToken(id = 82, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"), + + //a &&b &c&&d &e + ExpectedToken(id = 83, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"), + ExpectedToken(id = 84, tokenType = TokenType.DOUBLE_AMPERSAND, tokenLiteral = "&&"), + ExpectedToken(id = 85, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"), + ExpectedToken(id = 86, tokenType = TokenType.AMPERSAND, tokenLiteral = "&"), + ExpectedToken(id = 87, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"), + ExpectedToken(id = 88, tokenType = TokenType.DOUBLE_AMPERSAND, tokenLiteral = "&&"), + ExpectedToken(id = 89, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"), + ExpectedToken(id = 90, tokenType = TokenType.AMPERSAND, tokenLiteral = "&"), + ExpectedToken(id = 91, tokenType = TokenType.IDENTIFIER, tokenLiteral = "e"), + + // a|b||c||d|e + ExpectedToken(id = 92, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"), + ExpectedToken(id = 93, tokenType = TokenType.VERTICAL_BAR, tokenLiteral = "|"), + ExpectedToken(id = 94, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"), + ExpectedToken(id = 95, tokenType = TokenType.DOUBLE_VERTICAL_BAR, tokenLiteral = "||"), + ExpectedToken(id = 96, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"), + ExpectedToken(id = 97, tokenType = TokenType.DOUBLE_VERTICAL_BAR, tokenLiteral = "||"), + ExpectedToken(id = 98, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"), + ExpectedToken(id = 99, tokenType = TokenType.VERTICAL_BAR, tokenLiteral = "|"), + ExpectedToken(id = 100, tokenType = TokenType.IDENTIFIER, tokenLiteral = "e"), + + // a^b^c^=d^e + ExpectedToken(id = 101, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"), + ExpectedToken(id = 102, tokenType = TokenType.CARET, tokenLiteral = "^"), + ExpectedToken(id = 103, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"), + ExpectedToken(id = 104, tokenType = TokenType.CARET, tokenLiteral = "^"), + ExpectedToken(id = 105, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"), + ExpectedToken(id = 106, tokenType = TokenType.CARET_EQUALS, tokenLiteral = "^="), + ExpectedToken(id = 107, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"), + ExpectedToken(id = 108, tokenType = TokenType.CARET, tokenLiteral = "^"), + ExpectedToken(id = 109, tokenType = TokenType.IDENTIFIER, tokenLiteral = "e"), + // eof ExpectedToken(id = -1, tokenType = Token.EOF.tokenType, tokenLiteral = Token.EOF.tokenLiteral), )