diff --git a/benchmarks/common.py b/benchmarks/common.py
index 17c8043f..b56677e7 100644
--- a/benchmarks/common.py
+++ b/benchmarks/common.py
@@ -1,6 +1,7 @@
 from typing import List, Tuple, Union
 
 import torch
+from datasets.fingerprint import Hasher
 from transformers import AutoTokenizer, PreTrainedTokenizer
 
 
@@ -90,6 +91,9 @@ def convert_token_to_string(self, token: str) -> str:
 
         return string
 
+    def __hash__(self):
+        return hash(Hasher.hash(self.tokenizer))
+
     def __eq__(self, other):
         if isinstance(other, type(self)):
             if hasattr(self, "model_name") and hasattr(self, "kwargs"):