Add gemma tokenizer to custom

2024-07-06 01:21:04 +09:00
parent 7fdd4a93aa
commit b62120c02c
1 changed files with 3 additions and 0 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -17,6 +17,7 @@ export const tokenizerList = [
    ['llama', 'Llama'],
    ['llama3', 'Llama3'],
    ['novellist', 'Novellist'],
+    ['gemma', 'Gemma'],
 ] as const

 export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
@@ -35,6 +36,8 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
                return await tokenizeWebTokenizers(data, 'novellist')
            case 'llama3':
                return await tokenizeWebTokenizers(data, 'llama')
+            case 'gemma':
+                return await tokenizeWebTokenizers(data, 'gemma')
            default:
                // Add exception for gpt-4o tokenizers on reverse_proxy
                if(db.proxyRequestModel?.startsWith('gpt4o') ||