mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] ooba to use llama tokenizer

Browse Source
This commit is contained in:
kwaroran
2023-12-03 21:31:14 +09:00
parent c250cc6214
commit 6a6321c5dc
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/ts/tokenizer.ts
View File

@@ -19,7 +19,11 @@ async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
if(db.aiModel.startsWith('novelai')){ if(db.aiModel.startsWith('novelai')){
return await tokenizeWebTokenizers(data, 'novelai') return await tokenizeWebTokenizers(data, 'novelai')
} }
if(db.aiModel.startsWith('local_') || db.aiModel === 'mancer' || db.aiModel === 'textgen_webui' || (db.aiModel === 'reverse_proxy' && db.reverseProxyOobaMode)){ if(db.aiModel.startsWith('local_') ||
db.aiModel === 'mancer' ||
db.aiModel === 'textgen_webui' ||
(db.aiModel === 'reverse_proxy' && db.reverseProxyOobaMode ||
db.aiModel === 'ooba')){
return await tokenizeWebTokenizers(data, 'llama') return await tokenizeWebTokenizers(data, 'llama')
} }