fix

InternLM · Oct 11, 2023 · 5b543b3 · 5b543b3
1 parent c7bd9e3
commit 5b543b3
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/xtuner/dataset/utils.py b/xtuner/dataset/utils.py
@@ -38,7 +38,7 @@ def encode_fn(example, tokenizer, max_length, input_ids_with_output=True):
     """
     if tokenizer.__class__.__name__ == 'QWenTokenizer':
         bos_token_id = []
-        eos_token_id = tokenizer.encode('<|im_end|>', add_special_tokens=False)
+        eos_token_id = tokenizer.im_end_id
     elif tokenizer.__class__.__name__ == 'ChatGLMTokenizer':
         bos_token_id = []
         eos_token_id = tokenizer.eos_token_id