From 1aade242ec9904764f009e83bd3f19e27ab95735 Mon Sep 17 00:00:00 2001 From: IHaBiS02 Date: Tue, 14 May 2024 06:30:36 +0900 Subject: [PATCH] Add supports of gpt-4o tokenizer for reverse_proxy --- src/ts/tokenizer.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index f2d26313..ca38b90d 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -35,6 +35,11 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr case 'llama3': return await tokenizeWebTokenizers(data, 'llama') default: + // Add exception for gpt-4o tokenizers on reverse_proxy + if(db.proxyRequestModel?.startsWith('gpt4o') || + (db.proxyRequestModel === 'custom' && db.customProxyRequestModel.startsWith('gpt-4o'))) { + return await tikJS(data, 'o200k_base') + } return await tikJS(data) } }