mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] accurate tokenizing

Browse Source
This commit is contained in:
kwaroran
2023-11-23 17:52:00 +09:00
parent dab121c9c7
commit 634fe418b4
6 changed files with 116 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/lib/Setting/Pages/BotSettings.svelte
View File

@@ -5,7 +5,7 @@
import { DataBase } from "src/ts/storage/database";
import { customProviderStore, getCurrentPluginMax } from "src/ts/plugins/plugins";
import { getModelMaxContext, isTauri } from "src/ts/storage/globalApi";
import { tokenize } from "src/ts/tokenizer";
import { tokenize, tokenizeAccurate } from "src/ts/tokenizer";
import ModelList from "src/lib/UI/ModelList.svelte";
import DropList from "src/lib/SideBars/DropList.svelte";
import { PlusIcon, TrashIcon } from "lucide-svelte";
@@ -40,10 +40,10 @@
export let goPromptTemplate = () => {}
async function loadTokenize(){
tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
tokens.jailbreak = await tokenize($DataBase.jailbreak)
tokens.globalNote = await tokenize($DataBase.globalNote)
tokens.autoSuggest = await tokenize($DataBase.autoSuggestPrompt)
tokens.mainPrompt = await tokenizeAccurate($DataBase.mainPrompt)
tokens.jailbreak = await tokenizeAccurate($DataBase.jailbreak)
tokens.globalNote = await tokenizeAccurate($DataBase.globalNote)
tokens.autoSuggest = await tokenizeAccurate($DataBase.autoSuggestPrompt)
}
let advancedBotSettings = false