[fix] tokenizers not working

src/lib/Setting/Pages/BotSettings.svelte

@@ -26,10 +26,6 @@
     export let openPresetList =false
 
     async function loadTokenize(){
-        if(lasttokens.mainPrompt !== $DataBase.mainPrompt){
-            lasttokens.mainPrompt = $DataBase.mainPrompt
-            tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
-        }
         tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
         tokens.jailbreak = await tokenize($DataBase.jailbreak)
         tokens.globalNote = await tokenize($DataBase.globalNote)
@@ -40,16 +36,13 @@
     const unsub = DataBase.subscribe(db => {
         if(db.advancedBotSettings !== advancedBotSettings){
             advancedBotSettings = db.advancedBotSettings
-            
         }
+        loadTokenize()
     })
 
     onDestroy(() => {
         unsub()
     })
-    
-    $: loadTokenize()
-
 </script>
 
 <h2 class="mb-2 text-2xl font-bold mt-2">{language.chatBot}</h2>

src/ts/tokenizer.ts

@@ -8,7 +8,9 @@ import type { OpenAIChat } from "./process";
 async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
     let db = get(DataBase)
     if(db.aiModel === 'novellist'){
-        return await tokenizeWebTokenizers(data, 'novellist')
+        const nv= await tokenizeWebTokenizers(data, 'novellist')
+        console.log(nv)
+        return nv
     }
     if(db.aiModel.startsWith('claude')){
         return await tokenizeWebTokenizers(data, 'claude')
@@ -44,10 +46,12 @@ async function tokenizeWebTokenizers(text:string, type:tokenizerType) {
                 tokenizersTokenizer = await webTokenizer.Tokenizer.fromSentencePiece(
                     await (await fetch("/token/trin/spiece.model")
                 ).arrayBuffer())
+                break
             case "claude":
                 tokenizersTokenizer = await webTokenizer.Tokenizer.fromJSON(
                     await (await fetch("/token/claude/claude.json")
                 ).arrayBuffer())
+                break
         }
         tokenizersType = type
     }