Add missing __hash__ implementation to TransformerTokenizer

benchmarks/common.py

@@ -1,6 +1,7 @@
 from typing import List, Tuple, Union
 
 import torch
+from datasets.fingerprint import Hasher
 from transformers import AutoTokenizer, PreTrainedTokenizer
 
 
@@ -90,6 +91,9 @@ def convert_token_to_string(self, token: str) -> str:
 
         return string
 
+    def __hash__(self):
+        return hash(Hasher.hash(self.tokenizer))
+
     def __eq__(self, other):
         if isinstance(other, type(self)):
             if hasattr(self, "model_name") and hasattr(self, "kwargs"):