Support string literals

kubukoz · Oct 2, 2023 · fb0ee2e · fb0ee2e
1 parent 3afcf44
commit fb0ee2e
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 3 deletions.
diff --git a/modules/parser/src/main/scala/playground/smithyql/parser/v2/scanner.scala b/modules/parser/src/main/scala/playground/smithyql/parser/v2/scanner.scala
@@ -26,8 +26,8 @@ object TokenKind {
   case object KW_USE extends TokenKind
   case object KW_SERVICE extends TokenKind
   case object KW_BOOLEAN extends TokenKind
-  case object KW_NUMBER extends TokenKind
-  case object KW_STRING extends TokenKind
+  case object LIT_NUMBER extends TokenKind
+  case object LIT_STRING extends TokenKind
   case object KW_NULL extends TokenKind
 
   case object DOT extends TokenKind
@@ -116,7 +116,21 @@ object Scanner {
       "=" -> TokenKind.EQ,
     )
 
-    val readOne: PartialFunction[Unit, Unit] = readIdent.orElse(readPunctuation)
+    val readStringLiteral: PartialFunction[Unit, Unit] = {
+      case _ if remaining.startsWith("\"") =>
+        val (str, rest) = remaining.tail.span(_ != '\"')
+        if (rest.isEmpty) { // hit EOF
+          add(TokenKind.LIT_STRING("\"" + str))
+          remaining = rest
+        } else {
+          add(TokenKind.LIT_STRING("\"" + str + "\""))
+          remaining = rest.tail
+        }
+    }
+
+    val readOne: PartialFunction[Unit, Unit] = readIdent
+      .orElse(readPunctuation)
+      .orElse(readStringLiteral)
 
     // split "whitespace" string into chains of contiguous newlines OR whitespace characters.
     def whitespaceChains(

diff --git a/modules/parser/src/test/scala/playground/smithyql/parser/v2/ScannerTests.scala b/modules/parser/src/test/scala/playground/smithyql/parser/v2/ScannerTests.scala
@@ -287,4 +287,59 @@ object ScannerTests extends SimpleIOSuite with Checkers {
     )
   )
 
+  // string literals
+  scanTest(
+    "\"hello world\""
+  )(
+    List(
+      TokenKind.LIT_STRING("\"hello world\"")
+    )
+  )
+
+  scanTest(
+    explicitName = "String literal that never ends",
+    input = "\"hello world",
+  )(
+    List(
+      TokenKind.LIT_STRING("\"hello world")
+    )
+  )
+
+  scanTest(
+    explicitName = "Multiple string literals",
+    input = "\"hello world\", \"foo bar\"",
+  )(
+    List(
+      TokenKind.LIT_STRING("\"hello world\""),
+      TokenKind.COMMA(","),
+      TokenKind.SPACE(" "),
+      TokenKind.LIT_STRING("\"foo bar\""),
+    )
+  )
+
+  scanTest(
+    explicitName = "Multiple string literals, second one not closed",
+    input = "\"hello world\", \"foo bar",
+  )(
+    List(
+      TokenKind.LIT_STRING("\"hello world\""),
+      TokenKind.COMMA(","),
+      TokenKind.SPACE(" "),
+      TokenKind.LIT_STRING("\"foo bar"),
+    )
+  )
+
+  scanTest(
+    explicitName = "Multiple string literals, first one not closed",
+    input = "\"hello world, \"foo bar\"",
+  )(
+    List(
+      TokenKind.LIT_STRING("\"hello world, \""),
+      TokenKind.IDENT("foo"),
+      TokenKind.SPACE(" "),
+      TokenKind.IDENT("bar"),
+      TokenKind.LIT_STRING("\""),
+    )
+  )
+
 }