mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ref] tokenizer.ts

Browse Source
This commit is contained in:
kwaroran
2023-09-09 07:22:06 +09:00
parent 560f6b56e3
commit 0f32e58939
1 changed files with 0 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/ts/tokenizer.ts
View File

@@ -19,7 +19,6 @@ async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
} }
if(db.aiModel.startsWith('local_') || db.aiModel === 'mancer' || db.aiModel === 'textgen_webui'){ if(db.aiModel.startsWith('local_') || db.aiModel === 'mancer' || db.aiModel === 'textgen_webui'){
return await tokenizeWebTokenizers(data, 'llama') return await tokenizeWebTokenizers(data, 'llama')
} }
return await tikJS(data) return await tikJS(data)
} }