From b62120c02c37e1102cd907d020194cc25d00f4c1 Mon Sep 17 00:00:00 2001 From: kwaroran Date: Sat, 6 Jul 2024 01:21:04 +0900 Subject: [PATCH] Add gemma tokenizer to custom --- src/ts/tokenizer.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index 15847575..7afe4fa0 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -17,6 +17,7 @@ export const tokenizerList = [ ['llama', 'Llama'], ['llama3', 'Llama3'], ['novellist', 'Novellist'], + ['gemma', 'Gemma'], ] as const export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{ @@ -35,6 +36,8 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr return await tokenizeWebTokenizers(data, 'novellist') case 'llama3': return await tokenizeWebTokenizers(data, 'llama') + case 'gemma': + return await tokenizeWebTokenizers(data, 'gemma') default: // Add exception for gpt-4o tokenizers on reverse_proxy if(db.proxyRequestModel?.startsWith('gpt4o') ||