[feat] add exact tokens

src/lang/en.ts

@@ -451,4 +451,6 @@ export const languageEnglish = {
     lorePlus: "LoreBook+",
     reverseProxyOobaMode: "Reverse Proxy Ooba Mode",
     joinMultiUserRoom: "Join MultiUser Room",
+    exactTokens: "Exact Tokens",
+    fixedTokens: "Approximate Tokens",
 }

src/lib/Setting/Pages/PromptSettings.svelte

@@ -11,10 +11,12 @@
     let warns: string[] = []
     export let onGoBack: () => void = () => {}
     let tokens = 0
+    let extokens = 0
     executeTokenize($DataBase.promptTemplate)
 
     async function executeTokenize(prest: Proompt[]){
-        tokens = await tokenizePreset(prest)
+        tokens = await tokenizePreset(prest, true)
+        extokens = await tokenizePreset(prest, false)
     }
 
     $: warns = templateCheck($DataBase)
@@ -80,4 +82,5 @@
     $DataBase.promptTemplate = value
 }}><PlusIcon /></button>
 
-<span class="text-textcolor2 mb-6 text-sm mt-2">{tokens} {language.tokens}</span>
+<span class="text-textcolor2 mb-6 text-sm mt-2">{tokens} {language.fixedTokens}</span>
+<span class="text-textcolor2 mb-6 text-sm mt-2">{extokens} {language.exactTokens}</span>

src/ts/process/proompt.ts

@@ -27,13 +27,13 @@ export interface ProomptChat {
     rangeEnd: number|'end';
 }
 
-export async function tokenizePreset(proompts:Proompt[]){
+export async function tokenizePreset(proompts:Proompt[], consti:boolean = false){
     let total = 0
     for(const proompt of proompts){
         switch(proompt.type){
             case 'plain':
             case 'jailbreak':{
-                total += await tokenizeAccurate(proompt.text, true)
+                total += await tokenizeAccurate(proompt.text, consti)
                 break
             }
             case 'persona':
@@ -43,7 +43,7 @@ export async function tokenizePreset(proompts:Proompt[]){
             case 'authornote':
             case 'memory':{
                 if(proompt.innerFormat){
-                    total += await tokenizeAccurate(proompt.innerFormat, true)
+                    total += await tokenizeAccurate(proompt.innerFormat, consti)
                 }
                 break
             }