[feat] better nai

2023-08-04 21:00:40 +09:00
parent 6f35fce7d6
commit 4aefd681fe
15 changed files with 392 additions and 125 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -58,6 +58,7 @@ async function tokenizeWebTokenizers(text:string, type:tokenizerType) {
                tokenizersTokenizer = await webTokenizer.Tokenizer.fromSentencePiece(
                    await (await fetch("/token/nai/nerdstash_v2.model")
                ).arrayBuffer())
+                
                break
        }
        tokenizersType = type