fix: add tests for whitespace token issue in JiebaTokenizer

RasaHQ · Apr 28, 2024 · 2805e5d · 2805e5d
1 parent df21c83
commit 2805e5d
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/tests/nlu/tokenizers/test_jieba_tokenizer.py b/tests/nlu/tokenizers/test_jieba_tokenizer.py
@@ -37,6 +37,11 @@ def create_jieba(config: Optional[Dict] = None) -> JiebaTokenizer:
             ["Micheal", "你好", "吗", "？"],
             [(0, 7), (7, 9), (9, 10), (10, 11)],
         ),
+        (
+            "安装 rasa 应用",
+            ["安装", "rasa", "应用"],
+            [(0, 2), (3, 7), (8, 10)],
+        ),
     ],
 )
 def test_jieba(text, expected_tokens, expected_indices):