mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] add exact tokens

Browse Source
This commit is contained in:
kwaroran
2023-11-24 18:28:21 +09:00
parent df87712cdc
commit b2cc2ddcae
3 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/ts/process/proompt.ts
View File

@@ -27,13 +27,13 @@ export interface ProomptChat {
rangeEnd: number|'end';
}
export async function tokenizePreset(proompts:Proompt[]){
export async function tokenizePreset(proompts:Proompt[], consti:boolean = false){
let total = 0
for(const proompt of proompts){
switch(proompt.type){
case 'plain':
case 'jailbreak':{
total += await tokenizeAccurate(proompt.text, true)
total += await tokenizeAccurate(proompt.text, consti)
break
}
case 'persona':
@@ -43,7 +43,7 @@ export async function tokenizePreset(proompts:Proompt[]){
case 'authornote':
case 'memory':{
if(proompt.innerFormat){
total += await tokenizeAccurate(proompt.innerFormat, true)
total += await tokenizeAccurate(proompt.innerFormat, consti)
}
break
}