IDEA-CR-58318: TextMate: use non-strict utf-8 encoding (IDEA-232576)

To avoid endless loop described here: jruby/joni#17 (cherry picked from commit 21619a0255e1facf7e1aaa5879ca36956b98e45a) GitOrigin-RevId: 7c698742fa33d97047e98b7bbee9e5307844712b
JetBrains · Feb 12, 2020 · 63d2b77 · 63d2b77
1 parent bf4755b
commit 63d2b77
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 1 deletion.
diff --git a/plugins/textmate/src/org/jetbrains/plugins/textmate/regex/RegexUtil.java b/plugins/textmate/src/org/jetbrains/plugins/textmate/regex/RegexUtil.java
@@ -2,6 +2,7 @@
 
 import com.google.common.base.Preconditions;
 import com.intellij.openapi.util.TextRange;
+import org.jcodings.specific.NonStrictUTF8Encoding;
 import org.jcodings.specific.UTF8Encoding;
 import org.jetbrains.annotations.NotNull;
 
@@ -14,7 +15,7 @@ public static int charOffsetByByteOffset(byte[] stringBytes, int byteOffset) {
       return 0;
     }
     Preconditions.checkPositionIndex(byteOffset, stringBytes.length);
-    return UTF8Encoding.INSTANCE.strLength(stringBytes, 0, byteOffset);
+    return NonStrictUTF8Encoding.INSTANCE.strLength(stringBytes, 0, byteOffset);
   }
 
   public static int byteOffsetByCharOffset(@NotNull CharSequence charSequence, int charOffset) {

diff --git a/plugins/textmate/testData/lexer/markdown/unknown_utf.md b/plugins/textmate/testData/lexer/markdown/unknown_utf.md
@@ -0,0 +1 @@
+## 🚧🚧🚧 UNDER CONSTRUCTION 🚧🚧🚧
diff --git a/plugins/textmate/testData/lexer/markdown/unknown_utf_after.md b/plugins/textmate/testData/lexer/markdown/unknown_utf_after.md
@@ -0,0 +1,4 @@
+text.html.markdown markup.heading.${1/(#)(#)?(#)?(#)?(#)?(#)?/${6:?6:${5:?5:${4:?4:${3:?3:${2:?2:1}}}}}/}.markdown punctuation.definition.heading.markdown: [0, 2], {##}
+text.html.markdown markup.heading.${1/(#)(#)?(#)?(#)?(#)?(#)?/${6:?6:${5:?5:${4:?4:${3:?3:${2:?2:1}}}}}/}.markdown: [2, 3], { }
+text.html.markdown markup.heading.${1/(#)(#)?(#)?(#)?(#)?(#)?/${6:?6:${5:?5:${4:?4:${3:?3:${2:?2:1}}}}}/}.markdown entity.name.section.markdown: [3, 29], {🚧🚧🚧 UNDER CONSTRUCTION }
+text.html.markdown meta.paragraph.markdown: [29, 35], {🚧🚧🚧}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		## 🚧🚧🚧 UNDER CONSTRUCTION 🚧🚧🚧