update tokenization test (new tokenizer with pad token)

delphi-suite · May 15, 2024 · 3de26ce · 3de26ce
1 parent a135a4f
commit 3de26ce
Showing 1 changed file with 5 additions and 5 deletions.
diff --git a/tests/dataset/test_tokeniation.py b/tests/dataset/test_tokeniation.py
@@ -86,11 +86,11 @@ def test_tokenize_dataset(tokenizer):
     feature_name = get_random_feature_name()
     dataset = Dataset.from_dict({feature_name: documents})
     expected = [
-        [0, 431, 440, 260, 1, 46, 499, 1945, 368, 3443, 15],
-        [0, 15, 340, 576, 355, 337, 1887, 1, 431, 440, 260],
-        [0, 260, 399, 13, 314, 260, 560, 1005, 13, 402, 284],
-        [0, 284, 260, 2606, 1, 431, 440, 260, 399, 13, 402],
-        [0, 402, 284, 260, 1, 1370, 268, 415, 484, 412, 15],
+        [0, 432, 441, 261, 1, 47, 500, 1946, 369, 3444, 16],
+        [0, 16, 341, 577, 356, 338, 1888, 1, 432, 441, 261],
+        [0, 261, 400, 14, 315, 261, 561, 1006, 14, 403, 285],
+        [0, 285, 261, 2607, 1, 432, 441, 261, 400, 14, 403],
+        [0, 403, 285, 261, 1, 1371, 269, 416, 485, 413, 16],
     ]
     actual = [x for x in tokenize_dataset(dataset, tokenizer, SEQ_LEN, BATCH_SIZE)]
     assert actual == expected