mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] consistant tokenizing

Browse Source
This commit is contained in:
kwaroran
2023-11-24 14:50:06 +09:00
parent 4aa6adb637
commit 26f4ce94fa
4 changed files with 23 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/ts/process/proompt.ts
View File

@@ -33,7 +33,7 @@ export async function tokenizePreset(proompts:Proompt[]){
switch(proompt.type){
case 'plain':
case 'jailbreak':{
total += await tokenizeAccurate(proompt.text)
total += await tokenizeAccurate(proompt.text, true)
break
}
case 'persona':
@@ -43,7 +43,7 @@ export async function tokenizePreset(proompts:Proompt[]){
case 'authornote':
case 'memory':{
if(proompt.innerFormat){
total += await tokenizeAccurate(proompt.innerFormat)
total += await tokenizeAccurate(proompt.innerFormat, true)
}
break
}