diff --git a/src/lang/en.ts b/src/lang/en.ts index 002b10fc..0a993050 100644 --- a/src/lang/en.ts +++ b/src/lang/en.ts @@ -451,4 +451,6 @@ export const languageEnglish = { lorePlus: "LoreBook+", reverseProxyOobaMode: "Reverse Proxy Ooba Mode", joinMultiUserRoom: "Join MultiUser Room", + exactTokens: "Exact Tokens", + fixedTokens: "Approximate Tokens", } \ No newline at end of file diff --git a/src/lib/Setting/Pages/PromptSettings.svelte b/src/lib/Setting/Pages/PromptSettings.svelte index e41e9c3f..e90fe5dd 100644 --- a/src/lib/Setting/Pages/PromptSettings.svelte +++ b/src/lib/Setting/Pages/PromptSettings.svelte @@ -11,10 +11,12 @@ let warns: string[] = [] export let onGoBack: () => void = () => {} let tokens = 0 + let extokens = 0 executeTokenize($DataBase.promptTemplate) async function executeTokenize(prest: Proompt[]){ - tokens = await tokenizePreset(prest) + tokens = await tokenizePreset(prest, true) + extokens = await tokenizePreset(prest, false) } $: warns = templateCheck($DataBase) @@ -80,4 +82,5 @@ $DataBase.promptTemplate = value }}> -{tokens} {language.tokens} +{tokens} {language.fixedTokens} +{extokens} {language.exactTokens} \ No newline at end of file diff --git a/src/ts/process/proompt.ts b/src/ts/process/proompt.ts index 4040b167..09dad80a 100644 --- a/src/ts/process/proompt.ts +++ b/src/ts/process/proompt.ts @@ -27,13 +27,13 @@ export interface ProomptChat { rangeEnd: number|'end'; } -export async function tokenizePreset(proompts:Proompt[]){ +export async function tokenizePreset(proompts:Proompt[], consti:boolean = false){ let total = 0 for(const proompt of proompts){ switch(proompt.type){ case 'plain': case 'jailbreak':{ - total += await tokenizeAccurate(proompt.text, true) + total += await tokenizeAccurate(proompt.text, consti) break } case 'persona': @@ -43,7 +43,7 @@ export async function tokenizePreset(proompts:Proompt[]){ case 'authornote': case 'memory':{ if(proompt.innerFormat){ - total += await tokenizeAccurate(proompt.innerFormat, true) + total += await tokenizeAccurate(proompt.innerFormat, consti) } break }