mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] add mistral support

Browse Source
This commit is contained in:
kwaroran
2023-12-13 12:33:58 +09:00
parent 0ab32455c3
commit 6f54c90187
6 changed files with 76 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/ts/tokenizer.ts
View File

@@ -18,6 +18,9 @@ async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
if(db.aiModel.startsWith('novelai')){
return await tokenizeWebTokenizers(data, 'novelai')
}
if(db.aiModel.startsWith('mistral')){
return await tokenizeWebTokenizers(data, 'mistral')
}
if(db.aiModel.startsWith('local_') ||
db.aiModel === 'mancer' ||
db.aiModel === 'textgen_webui' ||
@@ -29,7 +32,7 @@ async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
return await tikJS(data)
}
type tokenizerType = 'novellist'|'claude'|'novelai'|'llama'
type tokenizerType = 'novellist'|'claude'|'novelai'|'llama'|'mistral'
let tikParser:Tiktoken = null
let tokenizersTokenizer:Tokenizer = null
@@ -73,6 +76,12 @@ async function tokenizeWebTokenizers(text:string, type:tokenizerType) {
tokenizersTokenizer = await webTokenizer.Tokenizer.fromSentencePiece(
await (await fetch("/token/llama/llama.model")
).arrayBuffer())
break
case 'mistral':
tokenizersTokenizer = await webTokenizer.Tokenizer.fromSentencePiece(
await (await fetch("/token/mistral/tokenizer.model")
).arrayBuffer())
break
}
tokenizersType = type