From 10710b6bc2ad3234c8b889dccca2645f94ff2830 Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Thu, 7 Dec 2023 00:03:50 +0900
Subject: [PATCH] [fix] ban

---
 src/ts/tokenizer.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index a8c3005a..4a77f785 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -190,7 +190,14 @@ export async function strongBan(data:string, bias:{[key:number]:number}) {
     }
 
 
+
     for(const char of banChars){
+        const encoded = await tokenizeNum(char)
+        if(encoded.length > 0){
+            if(!unbanChars.includes(encoded[0])){
+                bias[encoded[0]] = -100
+            }
+        }
         for(const alt of charAlt){
             let fchar = char