Add cohere tokenizer

2024-09-09 03:46:48 +09:00
parent 1f4a843292
commit 614087ae97
1 changed files with 3 additions and 0 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -19,6 +19,7 @@ export const tokenizerList = [
    ['llama3', 'Llama3'],
    ['novellist', 'Novellist'],
    ['gemma', 'Gemma'],
+    ['cohere', 'Cohere'],
 ] as const

 export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
@@ -39,6 +40,8 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
                return await tokenizeWebTokenizers(data, 'llama')
            case 'gemma':
                return await tokenizeWebTokenizers(data, 'gemma')
+            case 'cohere':
+                return await tokenizeWebTokenizers(data, 'cohere')
            default:
                // Add exception for gpt-4o tokenizers on reverse_proxy
                if(db.proxyRequestModel?.startsWith('gpt4o') ||