Move to tokenization tests

sillsdev · Jul 10, 2024 · 156cbec · 156cbec
1 parent f648aa4
commit 156cbec
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 18 deletions.
diff --git a/tests/SIL.Machine.Tests/Corpora/UsfmMemoryTextTests.cs b/tests/SIL.Machine.Tests/Corpora/UsfmMemoryTextTests.cs
@@ -49,23 +49,6 @@ public void GetRows_LastSegment()
         });
     }
 
-    [Test]
-    public void GetRows_Ending_ParagraphMarker()
-    {
-        //The ending paragraph marker should not crash the parser.
-        TextRow[] rows = GetRows(
-            @"\id MAT - Test
-\c 1
-\v 1 Descriptive title\x - \xo 18:16 \xt  hello world\x*\p
-"
-        );
-
-        Assert.Multiple(() =>
-        {
-            Assert.That(rows, Has.Length.EqualTo(1));
-        });
-    }
-
     private static TextRow[] GetRows(string usfm, bool includeMarkers = false, bool includeAllText = false)
     {
         UsfmMemoryText text =

diff --git a/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs b/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs
@@ -1,4 +1,5 @@
-using NUnit.Framework;
+using System.Text;
+using NUnit.Framework;
 
 namespace SIL.Machine.Corpora;
 
@@ -47,8 +48,39 @@ public void Detokenize()
         Assert.That(result, Is.EqualTo(usfm));
     }
 
+    [Test]
+    public void Tokenize_Ending_ParagraphMarker()
+    {
+        //The ending paragraph marker should not crash the parser.
+        TextRow[] rows = GetRows(
+            @"\id MAT - Test
+\c 1
+\v 1 Descriptive title\x - \xo 18:16 \xt  hello world\x*\p
+"
+        );
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(rows, Has.Length.EqualTo(1));
+        });
+    }
+
     private static string ReadUsfm()
     {
         return File.ReadAllText(Path.Combine(CorporaTestHelpers.UsfmTestProjectPath, "41MATTes.SFM"));
     }
+
+    private static TextRow[] GetRows(string usfm, bool includeMarkers = false, bool includeAllText = false)
+    {
+        UsfmMemoryText text =
+            new(
+                new UsfmStylesheet("usfm.sty"),
+                Encoding.UTF8,
+                "MAT",
+                usfm.Trim().ReplaceLineEndings("\r\n") + "\r\n",
+                includeMarkers: includeMarkers,
+                includeAllText: includeAllText
+            );
+        return text.GetRows().ToArray();
+    }
 }