Fix for some tokenizer_config formats

turboderp-org · Mar 31, 2024 · 6691c91 · 6691c91
1 parent a203a0b
commit 6691c91
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/exllamav2/tokenizer/tokenizer.py b/exllamav2/tokenizer/tokenizer.py
@@ -168,7 +168,16 @@ def get_default_token_id(config_key: str, current: int | None, default: int):
             if self.tokenizer_config_dict is not None and config_key in self.tokenizer_config_dict:
                 st = self.tokenizer_config_dict[config_key]
                 if st is None: return None
-                return self.tokenizer_model.piece_to_id(st)
+                if isinstance(st, dict):
+                    stc: str | None = st.get("content", None)
+                    if stc is None:
+                        return None
+                    else:
+                        return self.tokenizer_model.piece_to_id(stc)
+                elif isinstance(st, str):
+                    return self.tokenizer_model.piece_to_id(st)
+                else:
+                    return None
             else:
                 return default