From 15d7b188c56fbd19d41374518784c56c824b3674 Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Sun, 25 Jun 2023 21:56:02 +0900
Subject: [PATCH] [fix] tokenizers not working

---
 src/lib/Setting/Pages/BotSettings.svelte | 9 +--------
 src/ts/tokenizer.ts                      | 6 +++++-
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte
index 53451a70..f79908d3 100644
--- a/src/lib/Setting/Pages/BotSettings.svelte
+++ b/src/lib/Setting/Pages/BotSettings.svelte
@@ -26,10 +26,6 @@
     export let openPresetList =false
 
     async function loadTokenize(){
-        if(lasttokens.mainPrompt !== $DataBase.mainPrompt){
-            lasttokens.mainPrompt = $DataBase.mainPrompt
-            tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
-        }
         tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
         tokens.jailbreak = await tokenize($DataBase.jailbreak)
         tokens.globalNote = await tokenize($DataBase.globalNote)
@@ -40,16 +36,13 @@
     const unsub = DataBase.subscribe(db => {
         if(db.advancedBotSettings !== advancedBotSettings){
             advancedBotSettings = db.advancedBotSettings
-            
         }
+        loadTokenize()
     })
 
     onDestroy(() => {
         unsub()
     })
-    
-    $: loadTokenize()
-
 </script>
 
 <h2 class="mb-2 text-2xl font-bold mt-2">{language.chatBot}</h2>
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index be6d1d14..4e4bf9a0 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -8,7 +8,9 @@ import type { OpenAIChat } from "./process";
 async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
     let db = get(DataBase)
     if(db.aiModel === 'novellist'){
-        return await tokenizeWebTokenizers(data, 'novellist')
+        const nv= await tokenizeWebTokenizers(data, 'novellist')
+        console.log(nv)
+        return nv
     }
     if(db.aiModel.startsWith('claude')){
         return await tokenizeWebTokenizers(data, 'claude')
@@ -44,10 +46,12 @@ async function tokenizeWebTokenizers(text:string, type:tokenizerType) {
                 tokenizersTokenizer = await webTokenizer.Tokenizer.fromSentencePiece(
                     await (await fetch("/token/trin/spiece.model")
                 ).arrayBuffer())
+                break
             case "claude":
                 tokenizersTokenizer = await webTokenizer.Tokenizer.fromJSON(
                     await (await fetch("/token/claude/claude.json")
                 ).arrayBuffer())
+                break
         }
         tokenizersType = type
     }