[feat] accurate tokenizing

src/lib/Setting/Pages/BotSettings.svelte

@@ -5,7 +5,7 @@
     import { DataBase } from "src/ts/storage/database";
     import { customProviderStore, getCurrentPluginMax } from "src/ts/plugins/plugins";
     import { getModelMaxContext, isTauri } from "src/ts/storage/globalApi";
-    import { tokenize } from "src/ts/tokenizer";
+    import { tokenize, tokenizeAccurate } from "src/ts/tokenizer";
     import ModelList from "src/lib/UI/ModelList.svelte";
     import DropList from "src/lib/SideBars/DropList.svelte";
     import { PlusIcon, TrashIcon } from "lucide-svelte";
@@ -40,10 +40,10 @@
     export let goPromptTemplate = () => {}
 
     async function loadTokenize(){
-        tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
-        tokens.jailbreak = await tokenize($DataBase.jailbreak)
-        tokens.globalNote = await tokenize($DataBase.globalNote)
-        tokens.autoSuggest = await tokenize($DataBase.autoSuggestPrompt)
+        tokens.mainPrompt = await tokenizeAccurate($DataBase.mainPrompt)
+        tokens.jailbreak = await tokenizeAccurate($DataBase.jailbreak)
+        tokens.globalNote = await tokenizeAccurate($DataBase.globalNote)
+        tokens.autoSuggest = await tokenizeAccurate($DataBase.autoSuggestPrompt)
     }
 
     let advancedBotSettings = false

src/lib/Setting/Pages/PromptSettings.svelte

@@ -2,7 +2,7 @@
     import { ArrowLeft, PlusIcon } from "lucide-svelte";
     import { language } from "src/lang";
     import ProomptItem from "src/lib/UI/ProomptItem.svelte";
-    import type { Proompt } from "src/ts/process/proompt";
+    import { tokenizePreset, type Proompt } from "src/ts/process/proompt";
     import { templateCheck } from "src/ts/process/templates/templateCheck";
     import { DataBase } from "src/ts/storage/database";
 
@@ -10,8 +10,15 @@
     let opened = 0
     let warns: string[] = []
     export let onGoBack: () => void = () => {}
+    let tokens = 0
+    executeTokenize($DataBase.promptTemplate)
+
+    async function executeTokenize(prest: Proompt[]){
+        tokens = await tokenizePreset(prest)
+    }
 
     $: warns = templateCheck($DataBase)
+    $: executeTokenize($DataBase.promptTemplate)
 </script>
 
 <h2 class="mb-2 text-2xl font-bold mt-2 items-center flex">
@@ -71,4 +78,6 @@
         type2: 'normal'
     })
     $DataBase.promptTemplate = value
-}}><PlusIcon /></button>
+}}><PlusIcon /></button>
+
+<span class="text-textcolor2 mb-6 text-sm mt-2">{tokens} {language.tokens}</span>