This is to fix: sillsdev/serval#424.

sillsdev · Jul 10, 2024 · f648aa4 · f648aa4
1 parent 6979680
commit f648aa4
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 1 deletion.
diff --git a/src/SIL.Machine/Corpora/UsfmTokenizer.cs b/src/SIL.Machine/Corpora/UsfmTokenizer.cs
@@ -385,7 +385,16 @@ public IReadOnlyList<UsfmToken> Tokenize(string usfm, bool preserveWhitespace =
                         else if (tokens[i - 1].Type == UsfmTokenType.End)
                         {
                             // Insert space token after * of end marker
-                            int colNum = usfm.Length + 1 - Math.Max(usfm.LastIndexOf('\n', index), 0);
+                            int colNum;
+                            if (index >= usfm.Length)
+                            {
+                                colNum = usfm.Length + 1;
+                            }
+                            else
+                            {
+                                colNum = usfm.Length + 1 - Math.Max(usfm.LastIndexOf('\n', index), 0);
+                            }
+
                             tokens.Insert(
                                 i,
                                 new UsfmToken(UsfmTokenType.Text, null, " ", null)

diff --git a/tests/SIL.Machine.Tests/Corpora/UsfmMemoryTextTests.cs b/tests/SIL.Machine.Tests/Corpora/UsfmMemoryTextTests.cs
@@ -49,6 +49,23 @@ public void GetRows_LastSegment()
         });
     }
 
+    [Test]
+    public void GetRows_Ending_ParagraphMarker()
+    {
+        //The ending paragraph marker should not crash the parser.
+        TextRow[] rows = GetRows(
+            @"\id MAT - Test
+\c 1
+\v 1 Descriptive title\x - \xo 18:16 \xt  hello world\x*\p
+"
+        );
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(rows, Has.Length.EqualTo(1));
+        });
+    }
+
     private static TextRow[] GetRows(string usfm, bool includeMarkers = false, bool includeAllText = false)
     {
         UsfmMemoryText text =