mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: Correct tokenize flow in tokenizer encode function

Browse Source
This commit is contained in:
sub-hub
2025-04-21 13:34:01 +09:00
committed by GitHub
parent 33d8ed4568
commit 09228f3f86
1 changed files with 28 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
src/ts/tokenizer.ts
View File

@@ -89,9 +89,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
default: default:
result = await tikJS(data, 'o200k_base'); break; result = await tikJS(data, 'o200k_base'); break;
} }
} } else if (db.aiModel === 'custom' && pluginTokenizer) {
if(db.aiModel === 'custom' && pluginTokenizer){
switch(pluginTokenizer){ switch(pluginTokenizer){
case 'mistral': case 'mistral':
result = await tokenizeWebTokenizers(data, 'mistral'); break; result = await tokenizeWebTokenizers(data, 'mistral'); break;
@@ -120,32 +118,34 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
} }
} }
if(modelInfo.tokenizer === LLMTokenizer.NovelList){ // Fallback
result = await tokenizeWebTokenizers(data, 'novellist'); if (result === undefined) {
} else if(modelInfo.tokenizer === LLMTokenizer.Claude){ if(modelInfo.tokenizer === LLMTokenizer.NovelList){
result = await tokenizeWebTokenizers(data, 'claude'); result = await tokenizeWebTokenizers(data, 'novellist');
} else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){ } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
result = await tokenizeWebTokenizers(data, 'novelai'); result = await tokenizeWebTokenizers(data, 'claude');
} else if(modelInfo.tokenizer === LLMTokenizer.Mistral){ } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
result = await tokenizeWebTokenizers(data, 'mistral'); result = await tokenizeWebTokenizers(data, 'novelai');
} else if(modelInfo.tokenizer === LLMTokenizer.Llama){ } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
result = await tokenizeWebTokenizers(data, 'llama'); result = await tokenizeWebTokenizers(data, 'mistral');
} else if(modelInfo.tokenizer === LLMTokenizer.Local){ } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
result = await tokenizeGGUFModel(data); result = await tokenizeWebTokenizers(data, 'llama');
} else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){ } else if(modelInfo.tokenizer === LLMTokenizer.Local){
result = await tikJS(data, 'o200k_base'); result = await tokenizeGGUFModel(data);
} else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){ } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
result = await tokenizeGoogleCloud(data); result = await tikJS(data, 'o200k_base');
} else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){ } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
result = await gemmaTokenize(data); result = await tokenizeGoogleCloud(data);
} else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){ } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
result = await tokenizeWebTokenizers(data, 'DeepSeek'); result = await gemmaTokenize(data);
} else if(modelInfo.tokenizer === LLMTokenizer.Cohere){ } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
result = await tokenizeWebTokenizers(data, 'cohere'); result = await tokenizeWebTokenizers(data, 'DeepSeek');
} else { } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
result = await tikJS(data); result = await tokenizeWebTokenizers(data, 'cohere');
} else {
result = await tikJS(data);
}
} }
if(db.useTokenizerCaching){ if(db.useTokenizerCaching){
encodeCache.set(cacheKey, result); encodeCache.set(cacheKey, result);
} }