feat: add lexing for /= and %=

vighnesh153 · Feb 9, 2024 · 6f0413f · 6f0413f
1 parent b924fc5
commit 6f0413f
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 12 deletions.
diff --git a/...rs/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt b/...rs/languages/java-lang-common/src/main/kotlin/interpreters/javalang/common/lexer/Lexer.kt
@@ -50,9 +50,7 @@ fun Lexer.nextToken(): Token {
                     Token(tokenType = TokenType.PLUS_EQUALS, tokenLiteral = TokenType.PLUS_EQUALS.value)
                 }
 
-                else -> {
-                    Token(tokenType = TokenType.PLUS, tokenLiteral = TokenType.PLUS.value)
-                }
+                else -> Token(tokenType = TokenType.PLUS, tokenLiteral = TokenType.PLUS.value)
             }
         }
 
@@ -72,9 +70,7 @@ fun Lexer.nextToken(): Token {
                     Token(tokenType = TokenType.MINUS_EQUALS, tokenLiteral = TokenType.MINUS_EQUALS.value)
                 }
 
-                else -> {
-                    Token(tokenType = TokenType.MINUS, tokenLiteral = TokenType.MINUS.value)
-                }
+                else -> Token(tokenType = TokenType.MINUS, tokenLiteral = TokenType.MINUS.value)
             }
         }
 
@@ -86,15 +82,38 @@ fun Lexer.nextToken(): Token {
                     Token(tokenType = TokenType.ASTERISK_EQUALS, tokenLiteral = TokenType.ASTERISK_EQUALS.value)
                 }
 
-                else -> {
-                    Token(tokenType = TokenType.ASTERISK, tokenLiteral = TokenType.ASTERISK.value)
+                else -> Token(tokenType = TokenType.ASTERISK, tokenLiteral = TokenType.ASTERISK.value)
+            }
+        }
+
+        '/' -> {
+            val peek = peekCharacter()
+            t = when (peek) {
+                '=' -> {
+                    readNextCharacter()
+                    Token(
+                        tokenType = TokenType.FORWARD_SLASH_EQUALS,
+                        tokenLiteral = TokenType.FORWARD_SLASH_EQUALS.value
+                    )
                 }
+
+                else -> Token(tokenType = TokenType.FORWARD_SLASH, tokenLiteral = TokenType.FORWARD_SLASH.value)
             }
         }
 
-        '/' -> t = Token(tokenType = TokenType.FORWARD_SLASH, tokenLiteral = TokenType.FORWARD_SLASH.value)
         '\\' -> t = Token(tokenType = TokenType.BACK_SLASH, tokenLiteral = TokenType.BACK_SLASH.value)
-        '%' -> t = Token(tokenType = TokenType.MODULUS, tokenLiteral = TokenType.MODULUS.value)
+        '%' -> {
+            val peek = peekCharacter()
+            t = when (peek) {
+                '=' -> {
+                    readNextCharacter()
+                    Token(tokenType = TokenType.MODULUS_EQUALS, tokenLiteral = TokenType.MODULUS_EQUALS.value)
+                }
+
+                else -> Token(tokenType = TokenType.MODULUS, tokenLiteral = TokenType.MODULUS.value)
+            }
+        }
+
         '!' -> t = Token(tokenType = TokenType.BANG, tokenLiteral = TokenType.BANG.value)
         '&' -> t = Token(tokenType = TokenType.AMPERSAND, tokenLiteral = TokenType.AMPERSAND.value)
         '|' -> t = Token(tokenType = TokenType.VERTICAL_BAR, tokenLiteral = TokenType.VERTICAL_BAR.value)

diff --git a/...nguages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt b/...nguages/java-lang-common/src/main/kotlin/interpreters/javalang/common/tokens/TokenType.kt
@@ -41,8 +41,6 @@ enum class TokenType(val value: String) {
     MODULUS_EQUALS("%="),
     BANG("!"),
     NOT_EQUALS("!="),
-    GREATER_THAN_EQUALS_TO(">="),
-    LESS_THAN_EQUALS_TO("<="),
 
     AMPERSAND("&"),
     DOUBLE_AMPERSAND("&&"),
@@ -69,8 +67,10 @@ enum class TokenType(val value: String) {
     RIGHT_SQUARE_BRACKET("]"),
     LEFT_ANGLE_BRACKET("<"),
     DOUBLE_LEFT_ANGLE_BRACKET("<<"),
+    LEFT_ANGLE_BRACKET_EQUALS("<="),
     RIGHT_ANGLE_BRACKET(">"),
     DOUBLE_RIGHT_ANGLE_BRACKET(">>"),
+    RIGHT_ANGLE_BRACKET_EQUALS(">="),
     TRIPLE_RIGHT_ANGLE_BRACKET(">>>"),
 
     // keywords

diff --git a/...anguages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt b/...anguages/java-lang-common/src/test/kotlin/interpreters/javalang/common/lexer/LexerTest.kt
@@ -28,6 +28,10 @@ y-= z++ - --a
 
 a *=b*= c = d
 
+a /= b/c = d
+
+a %=  b  %c = d
+
         """.trimIndent()
 
         val expectedTokens = listOf(
@@ -144,6 +148,24 @@ a *=b*= c = d
             ExpectedToken(id = 60, tokenType = TokenType.EQUALS, tokenLiteral = "="),
             ExpectedToken(id = 61, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
 
+            // a /= b/c = d
+            ExpectedToken(id = 62, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"),
+            ExpectedToken(id = 63, tokenType = TokenType.FORWARD_SLASH_EQUALS, tokenLiteral = "/="),
+            ExpectedToken(id = 64, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"),
+            ExpectedToken(id = 65, tokenType = TokenType.FORWARD_SLASH, tokenLiteral = "/"),
+            ExpectedToken(id = 66, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"),
+            ExpectedToken(id = 67, tokenType = TokenType.EQUALS, tokenLiteral = "="),
+            ExpectedToken(id = 68, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
+
+            //a %=  b  %c = d
+            ExpectedToken(id = 69, tokenType = TokenType.IDENTIFIER, tokenLiteral = "a"),
+            ExpectedToken(id = 70, tokenType = TokenType.MODULUS_EQUALS, tokenLiteral = "%="),
+            ExpectedToken(id = 71, tokenType = TokenType.IDENTIFIER, tokenLiteral = "b"),
+            ExpectedToken(id = 72, tokenType = TokenType.MODULUS, tokenLiteral = "%"),
+            ExpectedToken(id = 73, tokenType = TokenType.IDENTIFIER, tokenLiteral = "c"),
+            ExpectedToken(id = 74, tokenType = TokenType.EQUALS, tokenLiteral = "="),
+            ExpectedToken(id = 75, tokenType = TokenType.IDENTIFIER, tokenLiteral = "d"),
+
             // eof
             ExpectedToken(id = -1, tokenType = Token.EOF.tokenType, tokenLiteral = Token.EOF.tokenLiteral),
         )