From 5702c808193d19a30c0711c88a4f10a512e7524b Mon Sep 17 00:00:00 2001 From: hanhainebula <2512674094@qq.com> Date: Fri, 22 Nov 2024 11:18:39 +0800 Subject: [PATCH] rm `max_length` para when using tokenizer.pad - avoid warning information --- FlagEmbedding/inference/reranker/decoder_only/base.py | 1 - FlagEmbedding/inference/reranker/decoder_only/lightweight.py | 1 - FlagEmbedding/inference/reranker/encoder_only/base.py | 1 - 3 files changed, 3 deletions(-) diff --git a/FlagEmbedding/inference/reranker/decoder_only/base.py b/FlagEmbedding/inference/reranker/decoder_only/base.py index d16b87a9..7d0d1645 100644 --- a/FlagEmbedding/inference/reranker/decoder_only/base.py +++ b/FlagEmbedding/inference/reranker/decoder_only/base.py @@ -163,7 +163,6 @@ def __call__(self, data): return self.tokenizer.pad( data, padding=True, - max_length=self.max_len, pad_to_multiple_of=8, return_tensors='pt', ) diff --git a/FlagEmbedding/inference/reranker/decoder_only/lightweight.py b/FlagEmbedding/inference/reranker/decoder_only/lightweight.py index b67892b1..87c0027d 100644 --- a/FlagEmbedding/inference/reranker/decoder_only/lightweight.py +++ b/FlagEmbedding/inference/reranker/decoder_only/lightweight.py @@ -81,7 +81,6 @@ def __call__(self, data): collected = self.tokenizer.pad( features, padding=True, - max_length=self.max_len, pad_to_multiple_of=8, return_tensors='pt', ) diff --git a/FlagEmbedding/inference/reranker/encoder_only/base.py b/FlagEmbedding/inference/reranker/encoder_only/base.py index a6c47d2c..9a1abebe 100644 --- a/FlagEmbedding/inference/reranker/encoder_only/base.py +++ b/FlagEmbedding/inference/reranker/encoder_only/base.py @@ -178,7 +178,6 @@ def compute_score_single_gpu( inputs = self.tokenizer.pad( sentences_batch, padding=True, - max_length=max_length, return_tensors='pt', **kwargs ).to(device)