mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: Potential problem in tikJS function

Browse Source
This commit is contained in:
sub-hub
2025-04-21 14:57:19 +09:00
committed by GitHub
parent 09228f3f86
commit 2a36743cb6
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/ts/tokenizer.ts
View File

@@ -209,6 +209,7 @@ async function gemmaTokenize(text:string) {
async function tikJS(text:string, model='cl100k_base') { async function tikJS(text:string, model='cl100k_base') {
if(!tikParser || lastTikModel !== model){ if(!tikParser || lastTikModel !== model){
tikParser?.free()
if(model === 'cl100k_base'){ if(model === 'cl100k_base'){
const {Tiktoken} = await import('@dqbd/tiktoken') const {Tiktoken} = await import('@dqbd/tiktoken')
const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json"); const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");