[feat] accurate tokenizing

src/ts/process/proompt.ts

@@ -1,3 +1,5 @@
+import { tokenizeAccurate } from "../tokenizer";
+
 export type Proompt = ProomptPlain|ProomptTyped|ProomptChat|ProomptAuthorNote;
 
 export interface ProomptPlain {
@@ -23,4 +25,29 @@ export interface ProomptChat {
     type: 'chat';
     rangeStart: number;
     rangeEnd: number|'end';
+}
+
+export async function tokenizePreset(proompts:Proompt[]){
+    let total = 0
+    for(const proompt of proompts){
+        switch(proompt.type){
+            case 'plain':
+            case 'jailbreak':{
+                total += await tokenizeAccurate(proompt.text)
+                break
+            }
+            case 'persona':
+            case 'description':
+            case 'lorebook':
+            case 'postEverything':
+            case 'authornote':
+            case 'memory':{
+                if(proompt.innerFormat){
+                    total += await tokenizeAccurate(proompt.innerFormat)
+                }
+                break
+            }
+        }
+    }
+    return total
 }