Add tokenizer playground

2024-04-24 22:06:02 +09:00
parent 90574bcdc3
commit 58513ed54a
2 changed files with 32 additions and 1 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -18,7 +18,7 @@ export const tokenizerList = [
    ['novellist', 'Novellist'],
 ] as const

-async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
+export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
    let db = get(DataBase)
    if(db.aiModel === 'openrouter' || db.aiModel === 'reverse_proxy'){
        switch(db.customTokenizer){