Make tokenizer caching to an option

src/lang/en.ts

@@ -1115,5 +1115,6 @@ export const languageEnglish = {
     fallbackWhenBlankResponse: "Fallback When Blank Response",
     doNotChangeFallbackModels: "Do Not Change Fallback Models on Preset Change",
     customModels: "Custom Models",
-    igpPrompt: "IGP Prompt"
+    igpPrompt: "IGP Prompt",
+    useTokenizerCaching: "Tokenizer Caching"
 }

src/lib/Setting/Pages/AdvancedSettings.svelte

@@ -207,6 +207,10 @@
     <Check bind:check={DBState.db.antiServerOverloads} name={language.antiServerOverload}>
     </Check>
 </div>
+<div class="flex items-center mt-4">
+    <Check bind:check={DBState.db.useTokenizerCaching} name={language.useTokenizerCaching}>
+    </Check>
+</div>
 {#if DBState.db.useExperimental}
     <div class="flex items-center mt-4">
         <Check bind:check={DBState.db.useExperimentalGoogleTranslator} name={"New Google Translate Experimental"}>

src/ts/storage/database.svelte.ts

@@ -1022,6 +1022,7 @@ export interface Database{
         flags: LLMFlags[]
     }[]
     igpPrompt:string
+    useTokenizerCaching:boolean
 }
 
 interface SeparateParameters{

src/ts/tokenizer.ts

@@ -47,7 +47,9 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
     const modelInfo = getModelInfo(db.aiModel);
     const pluginTokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer ?? "none";
 
-    const cacheKey = getHash(
+    let cacheKey = ''
+    if(db.useTokenizerCaching){
+        cacheKey = getHash(
             data,
             db.aiModel,
             db.customTokenizer,
@@ -60,6 +62,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
         if (cachedResult !== undefined) {
             return cachedResult;
         }
+    }
 
     let result: number[] | Uint32Array | Int32Array;
 
@@ -142,7 +145,10 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
     } else {
         result = await tikJS(data);
     }
+
+    if(db.useTokenizerCaching){
         encodeCache.set(cacheKey, result);
+    }
 
     return result;
 }