[fix] consistant tokenizing

2023-11-24 14:50:06 +09:00
parent 4aa6adb637
commit 26f4ce94fa
4 changed files with 23 additions and 8 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -87,9 +87,10 @@ export async function tokenize(data:string) {
    return encoded.length
 }

-export async function tokenizeAccurate(data:string) {
+export async function tokenizeAccurate(data:string, consistantChar?:boolean) {
    data = risuChatParser(data.replace('{{slot}}',''), {
-        tokenizeAccurate: true
+        tokenizeAccurate: true,
+        consistantChar: consistantChar,
    })
    const encoded = await encode(data)
    return encoded.length