From bfa7142566000997eb22df9da4b29f3dd2a04e9c Mon Sep 17 00:00:00 2001 From: Bearice Ren Date: Tue, 6 Feb 2024 15:07:42 +0900 Subject: [PATCH] fixes tiktoken error with gpt-3.5-turbo-instruct --- utils/server/tiktoken.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/utils/server/tiktoken.ts b/utils/server/tiktoken.ts index d31cd46d43..bd4b39054e 100644 --- a/utils/server/tiktoken.ts +++ b/utils/server/tiktoken.ts @@ -9,8 +9,15 @@ export const getTiktokenEncoding = async (model: string): Promise => { if (modelId.indexOf('text-davinci-') !== -1) { return new Tiktoken(p50k.bpe_ranks, p50k.special_tokens, p50k.pat_str); } - if (modelId.indexOf('gpt-3.5') !== -1 || modelId.indexOf('gpt-4') !== -1) { - return encoding_for_model(modelId, { + if (modelId.indexOf('gpt-3.5') !== -1) { + return encoding_for_model('gpt-3.5-turbo', { + '<|im_start|>': 100264, + '<|im_end|>': 100265, + '<|im_sep|>': 100266, + }); + } + if (modelId.indexOf('gpt-4') !== -1) { + return encoding_for_model('gpt-4', { '<|im_start|>': 100264, '<|im_end|>': 100265, '<|im_sep|>': 100266,