diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte
index 53451a70..f79908d3 100644
--- a/src/lib/Setting/Pages/BotSettings.svelte
+++ b/src/lib/Setting/Pages/BotSettings.svelte
@@ -26,10 +26,6 @@
export let openPresetList =false
async function loadTokenize(){
- if(lasttokens.mainPrompt !== $DataBase.mainPrompt){
- lasttokens.mainPrompt = $DataBase.mainPrompt
- tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
- }
tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
tokens.jailbreak = await tokenize($DataBase.jailbreak)
tokens.globalNote = await tokenize($DataBase.globalNote)
@@ -40,16 +36,13 @@
const unsub = DataBase.subscribe(db => {
if(db.advancedBotSettings !== advancedBotSettings){
advancedBotSettings = db.advancedBotSettings
-
}
+ loadTokenize()
})
onDestroy(() => {
unsub()
})
-
- $: loadTokenize()
-
{language.chatBot}
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index be6d1d14..4e4bf9a0 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -8,7 +8,9 @@ import type { OpenAIChat } from "./process";
async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
let db = get(DataBase)
if(db.aiModel === 'novellist'){
- return await tokenizeWebTokenizers(data, 'novellist')
+ const nv= await tokenizeWebTokenizers(data, 'novellist')
+ console.log(nv)
+ return nv
}
if(db.aiModel.startsWith('claude')){
return await tokenizeWebTokenizers(data, 'claude')
@@ -44,10 +46,12 @@ async function tokenizeWebTokenizers(text:string, type:tokenizerType) {
tokenizersTokenizer = await webTokenizer.Tokenizer.fromSentencePiece(
await (await fetch("/token/trin/spiece.model")
).arrayBuffer())
+ break
case "claude":
tokenizersTokenizer = await webTokenizer.Tokenizer.fromJSON(
await (await fetch("/token/claude/claude.json")
).arrayBuffer())
+ break
}
tokenizersType = type
}