From 4f85ba5bab77c8ff89b0f7cab3a1a4dd9cf34273 Mon Sep 17 00:00:00 2001
From: Katarzyna Marek <kasia@marek.net>
Date: Tue, 5 Dec 2023 19:17:56 +0100
Subject: [PATCH] improvement: add semantic tokens for escape sequences

---
 .../metals/SemanticTokensProvider.scala       | 77 +++++++++++++++++++
 .../scala/tests/SemanticTokensLspSuite.scala  | 12 +++
 2 files changed, 89 insertions(+)

diff --git a/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
index 725cbdb62d1..0f64b7f3848 100644
--- a/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
+++ b/metals/src/main/scala/scala/meta/internal/metals/SemanticTokensProvider.scala
@@ -152,6 +152,9 @@ object SemanticTokensProvider {
       case comm: Token.Comment if isDocString(comm.value) =>
         val (toAdd, delta0) = makeDocStringTokens(comm, delta)
         (toAdd, nodesIterator, delta0)
+      case EscapableString(text, isInterpolation) =>
+        val (toAdd, delta0) = makeStringTokens(text, isInterpolation, delta)
+        (toAdd, nodesIterator, delta0)
       case _ =>
         val (tokenType, tokenModifier, remainingNodes) =
           getTypeAndMod(tk, nodesIterator, isScala3)
@@ -402,6 +405,67 @@ object SemanticTokensProvider {
     (buffer.toList, delta)
   }
 
+  private def makeStringTokens(
+      text: String,
+      isInterpolation: Boolean,
+      initialDelta: Line,
+  ): (List[Integer], Line) = {
+    val buffer = ListBuffer.empty[Integer]
+    var delta = initialDelta
+    val currentPart = new StringBuilder()
+
+    def emitToken(token: String, tokenType: Int, tokenModifiers: Int = 0) {
+      val (toAdd, newDelta) = convertTokensToIntList(
+        token,
+        delta,
+        tokenType,
+        tokenModifiers,
+      )
+      buffer.addAll(toAdd)
+      delta = newDelta
+    }
+
+    def emitCurrent() = {
+      val current = currentPart.result()
+      if (current.nonEmpty) {
+        emitToken(current, getTypeId(SemanticTokenTypes.String))
+        currentPart.clear()
+      }
+    }
+
+    def emitEscape(special: String) =
+      emitToken(
+        special,
+        getTypeId(SemanticTokenTypes.Operator),
+        1 << getModifierId(SemanticTokenModifiers.Documentation),
+      )
+
+    @tailrec
+    def loop(text: List[Char]): Unit = {
+      text match {
+        case '\\' :: 'u' :: rest if rest.length >= 4 =>
+          emitCurrent()
+          emitEscape(s"\\u${rest.take(4).mkString}")
+          loop(rest.drop(4))
+        case '\\' :: c :: rest =>
+          emitCurrent()
+          emitEscape("\\" + c)
+          loop(rest)
+        case '$' :: '$' :: rest if isInterpolation =>
+          emitCurrent()
+          emitEscape("$$")
+          loop(rest)
+        case Nil => emitCurrent()
+        case c :: rest =>
+          currentPart.addOne(c)
+          loop(rest)
+      }
+    }
+
+    loop(text.toList)
+    (buffer.result(), delta)
+  }
+
   case class DocstringToken(
       start: Int,
       end: Int,
@@ -486,3 +550,16 @@ object SemanticTokensProvider {
     }
   }
 }
+
+object EscapableString {
+  def unapply(token: Token): Option[(String, Boolean)] =
+    token match {
+      case str: Token.Constant.String if !str.text.startsWith("\"\"\"") =>
+        Some((str.text, false))
+      case c: Token.Constant.Char =>
+        Some((c.text, false))
+      case inter: Token.Interpolation.Part =>
+        Some((inter.text, true))
+      case _ => None
+    }
+}
diff --git a/tests/unit/src/test/scala/tests/SemanticTokensLspSuite.scala b/tests/unit/src/test/scala/tests/SemanticTokensLspSuite.scala
index f348d88fdc6..64fc652608c 100644
--- a/tests/unit/src/test/scala/tests/SemanticTokensLspSuite.scala
+++ b/tests/unit/src/test/scala/tests/SemanticTokensLspSuite.scala
@@ -242,6 +242,18 @@ class SemanticTokensLspSuite extends BaseLspSuite("SemanticTokens") {
         |""".stripMargin,
   )
 
+  check(
+    "escapes",
+    s"""|<<object>>/*keyword*/ <<O>>/*class*/ {
+        |  <<val>>/*keyword*/ <<stringEscape>>/*variable,definition,readonly*/ = <<"smth >>/*string*/<<\\n>>/*operator,documentation*/<<\\">>/*operator,documentation*/<< rest>>/*string*/<<\\n>>/*operator,documentation*/<<">>/*string*/
+        |  <<val>>/*keyword*/ <<multilineString>>/*variable,definition,readonly*/ = <<\"\"\"\\n\"\"\">>/*string*/
+        |  <<val>>/*keyword*/ <<charEscape>>/*variable,definition,readonly*/ = <<'>>/*string*/<<\\n>>/*operator,documentation*/<<'>>/*string*/
+        |  <<val>>/*keyword*/ <<interpolatorEscape>>/*variable,definition,readonly*/ = <<s>>/*keyword*/<<">>/*string*/<<$$$$>>/*operator,documentation*/<<smth >>/*string*/<<\\n>>/*operator,documentation*/<<\\">>/*operator,documentation*/<< rest>>/*string*/<<">>/*string*/
+        |  <<val>>/*keyword*/ <<unicode>>/*variable,definition,readonly*/ = <<s>>/*keyword*/<<\"\"\">>/*string*/<<\\u202c>>/*operator,documentation*/<<\"\"\">>/*string*/
+        |}
+        |""".stripMargin,
+  )
+
   def check(
       name: TestOptions,
       expected: String,