Make tokenizer caching to an option

src/lang/en.ts

@@ -1115,5 +1115,6 @@ export const languageEnglish = {
     fallbackWhenBlankResponse: "Fallback When Blank Response",
     doNotChangeFallbackModels: "Do Not Change Fallback Models on Preset Change",
     customModels: "Custom Models",
-    igpPrompt: "IGP Prompt"
+    igpPrompt: "IGP Prompt",
+    useTokenizerCaching: "Tokenizer Caching"
 }

src/lib/Setting/Pages/AdvancedSettings.svelte

@@ -207,6 +207,10 @@
     <Check bind:check={DBState.db.antiServerOverloads} name={language.antiServerOverload}>
     </Check>
 </div>
+<div class="flex items-center mt-4">
+    <Check bind:check={DBState.db.useTokenizerCaching} name={language.useTokenizerCaching}>
+    </Check>
+</div>
 {#if DBState.db.useExperimental}
     <div class="flex items-center mt-4">
         <Check bind:check={DBState.db.useExperimentalGoogleTranslator} name={"New Google Translate Experimental"}>

src/ts/storage/database.svelte.ts

@@ -1022,6 +1022,7 @@ export interface Database{
         flags: LLMFlags[]
     }[]
     igpPrompt:string
+    useTokenizerCaching:boolean
 }
 
 interface SeparateParameters{

src/ts/tokenizer.ts

@@ -47,18 +47,21 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
     const modelInfo = getModelInfo(db.aiModel);
     const pluginTokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer ?? "none";
 
-    const cacheKey = getHash(
+    let cacheKey = ''
-        data,
+    if(db.useTokenizerCaching){
-        db.aiModel,
+        cacheKey = getHash(
-        db.customTokenizer,
+            data,
-        db.currentPluginProvider,
+            db.aiModel,
-        db.googleClaudeTokenizing,
+            db.customTokenizer,
-        modelInfo,
+            db.currentPluginProvider,
-        pluginTokenizer
+            db.googleClaudeTokenizing,
-    );
+            modelInfo,
-    const cachedResult = encodeCache.get(cacheKey);
+            pluginTokenizer
-    if (cachedResult !== undefined) {
+        );
-        return cachedResult;
+        const cachedResult = encodeCache.get(cacheKey);
+        if (cachedResult !== undefined) {
+            return cachedResult;
+        }
     }
 
     let result: number[] | Uint32Array | Int32Array;
@@ -142,7 +145,10 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
     } else {
         result = await tikJS(data);
     }
-    encodeCache.set(cacheKey, result);
+
+    if(db.useTokenizerCaching){
+        encodeCache.set(cacheKey, result);
+    }
 
     return result;
 }