From 2a36743cb6966b3a193dbb692bde6acabf547c7f Mon Sep 17 00:00:00 2001
From: sub-hub <70351692+sub-hub@users.noreply.github.com>
Date: Mon, 21 Apr 2025 14:57:19 +0900
Subject: [PATCH] Fix: Potential problem in tikJS function

---
 src/ts/tokenizer.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index 1e433534..25fb442d 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -209,6 +209,7 @@ async function gemmaTokenize(text:string) {
 
 async function tikJS(text:string, model='cl100k_base') {
     if(!tikParser || lastTikModel !== model){
+        tikParser?.free()
         if(model === 'cl100k_base'){
             const {Tiktoken} = await import('@dqbd/tiktoken')
             const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");