feat: add lexing for != and && and || and ^=

vighnesh153 · Feb 10, 2024 · a938d3c · a938d3c
1 parent 6f0413f
commit a938d3c
Show file tree

Hide file tree

Showing 3 changed files with 99 additions and 5 deletions.
diff --git a/...rs/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt b/...rs/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt
@@ -114,10 +114,54 @@ fun Lexer.nextToken(): Token {
             }
         }
 
-        '!' -> t = Token(tokenType = TokenType.BANG, tokenLiteral = TokenType.BANG.value)
-        '&' -> t = Token(tokenType = TokenType.AMPERSAND, tokenLiteral = TokenType.AMPERSAND.value)
-        '|' -> t = Token(tokenType = TokenType.VERTICAL_BAR, tokenLiteral = TokenType.VERTICAL_BAR.value)
-        '^' -> t = Token(tokenType = TokenType.CARET, tokenLiteral = TokenType.CARET.value)
+        '!' -> {
+            val peek = peekCharacter()
+            t = when (peek) {
+                '=' -> {
+                    readNextCharacter()
+                    Token(tokenType = TokenType.BANG_EQUALS, tokenLiteral = TokenType.BANG_EQUALS.value)
+                }
+
+                else -> Token(tokenType = TokenType.BANG, tokenLiteral = TokenType.BANG.value)
+            }
+        }
+
+        '&' -> {
+            val peek = peekCharacter()
+            t = when (peek) {
+                '&' -> {
+                    readNextCharacter()
+                    Token(tokenType = TokenType.DOUBLE_AMPERSAND, tokenLiteral = TokenType.DOUBLE_AMPERSAND.value)
+                }
+
+                else -> Token(tokenType = TokenType.AMPERSAND, tokenLiteral = TokenType.AMPERSAND.value)
+            }
+        }
+
+        '|' -> {
+            val peek = peekCharacter()
+            t = when (peek) {
+                '|' -> {
+                    readNextCharacter()
+                    Token(tokenType = TokenType.DOUBLE_VERTICAL_BAR, tokenLiteral = TokenType.DOUBLE_VERTICAL_BAR.value)
+                }
+
+                else -> Token(tokenType = TokenType.VERTICAL_BAR, tokenLiteral = TokenType.VERTICAL_BAR.value)
+            }
+        }
+
+        '^' -> {
+            val peek = peekCharacter()
+            t = when (peek) {
+                '=' -> {
+                    readNextCharacter()
+                    Token(tokenType = TokenType.CARET_EQUALS, tokenLiteral = TokenType.CARET_EQUALS.value)
+                }
+
+                else -> Token(tokenType = TokenType.CARET, tokenLiteral = TokenType.CARET.value)
+            }
+        }
+
         '?' -> t = Token(tokenType = TokenType.QUESTION, tokenLiteral = TokenType.QUESTION.value)
         ':' -> t = Token(tokenType = TokenType.COLON, tokenLiteral = TokenType.COLON.value)
         '.' -> t = Token(tokenType = TokenType.DOT, tokenLiteral = TokenType.DOT.value)

diff --git a/...nguages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt b/...nguages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt
@@ -40,7 +40,7 @@ enum class TokenType(val value: String) {
     MODULUS("%"),
     MODULUS_EQUALS("%="),
     BANG("!"),
-    NOT_EQUALS("!="),
+    BANG_EQUALS("!="),
 
     AMPERSAND("&"),
     DOUBLE_AMPERSAND("&&"),

diff --git a/...anguages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt b/...anguages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt
@@ -32,6 +32,14 @@ a /= b/c = d
 
 a %=  b  %c = d
 
+a!=b !c =d
+
+a &&b &c&&d &e
+
+a|b||c||d|e
+
+a^b^c^=d^e
+
         """.trimIndent()
 
         val expectedTokens = listOf(
@@ -166,6 +174,48 @@ a %=  b  %c = d
             ExpectedToken(id = 74, tokenType = TokenType.EQUALS, tokenLiteral = "="),
             ExpectedToken(id = 75, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
 
+            // a!=b !c =d
+            ExpectedToken(id = 76, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"),
+            ExpectedToken(id = 77, tokenType = TokenType.BANG_EQUALS, tokenLiteral = "!="),
+            ExpectedToken(id = 78, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"),
+            ExpectedToken(id = 79, tokenType = TokenType.BANG, tokenLiteral = "!"),
+            ExpectedToken(id = 80, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"),
+            ExpectedToken(id = 81, tokenType = TokenType.EQUALS, tokenLiteral = "="),
+            ExpectedToken(id = 82, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
+
+            //a &&b &c&&d &e
+            ExpectedToken(id = 83, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"),
+            ExpectedToken(id = 84, tokenType = TokenType.DOUBLE_AMPERSAND, tokenLiteral = "&&"),
+            ExpectedToken(id = 85, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"),
+            ExpectedToken(id = 86, tokenType = TokenType.AMPERSAND, tokenLiteral = "&"),
+            ExpectedToken(id = 87, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"),
+            ExpectedToken(id = 88, tokenType = TokenType.DOUBLE_AMPERSAND, tokenLiteral = "&&"),
+            ExpectedToken(id = 89, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
+            ExpectedToken(id = 90, tokenType = TokenType.AMPERSAND, tokenLiteral = "&"),
+            ExpectedToken(id = 91, tokenType = TokenType.IDENTIFIER, tokenLiteral = "e"),
+
+            // a|b||c||d|e
+            ExpectedToken(id = 92, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"),
+            ExpectedToken(id = 93, tokenType = TokenType.VERTICAL_BAR, tokenLiteral = "|"),
+            ExpectedToken(id = 94, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"),
+            ExpectedToken(id = 95, tokenType = TokenType.DOUBLE_VERTICAL_BAR, tokenLiteral = "||"),
+            ExpectedToken(id = 96, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"),
+            ExpectedToken(id = 97, tokenType = TokenType.DOUBLE_VERTICAL_BAR, tokenLiteral = "||"),
+            ExpectedToken(id = 98, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
+            ExpectedToken(id = 99, tokenType = TokenType.VERTICAL_BAR, tokenLiteral = "|"),
+            ExpectedToken(id = 100, tokenType = TokenType.IDENTIFIER, tokenLiteral = "e"),
+
+            // a^b^c^=d^e
+            ExpectedToken(id = 101, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"),
+            ExpectedToken(id = 102, tokenType = TokenType.CARET, tokenLiteral = "^"),
+            ExpectedToken(id = 103, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"),
+            ExpectedToken(id = 104, tokenType = TokenType.CARET, tokenLiteral = "^"),
+            ExpectedToken(id = 105, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"),
+            ExpectedToken(id = 106, tokenType = TokenType.CARET_EQUALS, tokenLiteral = "^="),
+            ExpectedToken(id = 107, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
+            ExpectedToken(id = 108, tokenType = TokenType.CARET, tokenLiteral = "^"),
+            ExpectedToken(id = 109, tokenType = TokenType.IDENTIFIER, tokenLiteral = "e"),
+
             // eof
             ExpectedToken(id = -1, tokenType = Token.EOF.tokenType, tokenLiteral = Token.EOF.tokenLiteral),
         )