Fix: Potential problem in tikJS function

This commit is contained in:
sub-hub
2025-04-21 14:57:19 +09:00
committed by GitHub
parent 09228f3f86
commit 2a36743cb6

View File

@@ -209,6 +209,7 @@ async function gemmaTokenize(text:string) {
async function tikJS(text:string, model='cl100k_base') {
if(!tikParser || lastTikModel !== model){
tikParser?.free()
if(model === 'cl100k_base'){
const {Tiktoken} = await import('@dqbd/tiktoken')
const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");