diff --git a/src/lib/Playground/PlaygroundTokenizer.svelte b/src/lib/Playground/PlaygroundTokenizer.svelte index e69de29b..1efff614 100644 --- a/src/lib/Playground/PlaygroundTokenizer.svelte +++ b/src/lib/Playground/PlaygroundTokenizer.svelte @@ -0,0 +1,31 @@ + + +

{language.tokenizer}

+ +Input + + + +Result + + + +{outputLength} {language.tokens} diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index e6d52ff8..e02650c2 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -18,7 +18,7 @@ export const tokenizerList = [ ['novellist', 'Novellist'], ] as const -async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{ +export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{ let db = get(DataBase) if(db.aiModel === 'openrouter' || db.aiModel === 'reverse_proxy'){ switch(db.customTokenizer){