mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add gemma tokenizer to custom

Browse Source
This commit is contained in:
kwaroran
2024-07-06 01:21:04 +09:00
parent 7fdd4a93aa
commit b62120c02c
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/ts/tokenizer.ts
View File

@@ -17,6 +17,7 @@ export const tokenizerList = [
['llama', 'Llama'], ['llama', 'Llama'],
['llama3', 'Llama3'], ['llama3', 'Llama3'],
['novellist', 'Novellist'], ['novellist', 'Novellist'],
['gemma', 'Gemma'],
] as const ] as const
export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
@@ -35,6 +36,8 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
return await tokenizeWebTokenizers(data, 'novellist') return await tokenizeWebTokenizers(data, 'novellist')
case 'llama3': case 'llama3':
return await tokenizeWebTokenizers(data, 'llama') return await tokenizeWebTokenizers(data, 'llama')
case 'gemma':
return await tokenizeWebTokenizers(data, 'gemma')
default: default:
// Add exception for gpt-4o tokenizers on reverse_proxy // Add exception for gpt-4o tokenizers on reverse_proxy
if(db.proxyRequestModel?.startsWith('gpt4o') || if(db.proxyRequestModel?.startsWith('gpt4o') ||