From 614087ae9746e38e87ca1927a1d837e0396739ba Mon Sep 17 00:00:00 2001 From: kwaroran Date: Mon, 9 Sep 2024 03:46:48 +0900 Subject: [PATCH] Add cohere tokenizer --- src/ts/tokenizer.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index 1a8cf376..d0b6633f 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -19,6 +19,7 @@ export const tokenizerList = [ ['llama3', 'Llama3'], ['novellist', 'Novellist'], ['gemma', 'Gemma'], + ['cohere', 'Cohere'], ] as const export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{ @@ -39,6 +40,8 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr return await tokenizeWebTokenizers(data, 'llama') case 'gemma': return await tokenizeWebTokenizers(data, 'gemma') + case 'cohere': + return await tokenizeWebTokenizers(data, 'cohere') default: // Add exception for gpt-4o tokenizers on reverse_proxy if(db.proxyRequestModel?.startsWith('gpt4o') ||