[fix] ban

2023-12-07 00:03:50 +09:00
parent 111e5711f7
commit 10710b6bc2
1 changed files with 7 additions and 0 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -190,7 +190,14 @@ export async function strongBan(data:string, bias:{[key:number]:number}) {
    }


+
    for(const char of banChars){
+        const encoded = await tokenizeNum(char)
+        if(encoded.length > 0){
+            if(!unbanChars.includes(encoded[0])){
+                bias[encoded[0]] = -100
+            }
+        }
        for(const alt of charAlt){
            let fchar = char