mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] tokenize problem

Browse Source
This commit is contained in:
kwaroran
2023-05-29 20:41:33 +09:00
parent bdf00126cc
commit 71b02cf104
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/ts/process/index.ts
View File

@@ -93,6 +93,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
}
}
let unformated = {
'main':([] as OpenAIChat[]),
'jailbreak':([] as OpenAIChat[]),
@@ -162,10 +163,11 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
}).join('\n\n')
}).join('\n\n')) + db.maxResponse) + 150
const examples = exampleMessage(currentChar)
for(const example of examples){
currentTokens += await tokenize(example.content)
currentTokens += await tokenize(example.content) + 5
}
let chats:OpenAIChat[] = examples
@@ -215,7 +217,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
memo: msg.chatId,
name: name
})
currentTokens += (await tokenize(formedChat) + 1)
currentTokens += (await tokenize(formedChat) + 5)
}
if(nowChatroom.type === 'group'){
@@ -224,7 +226,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
role: 'system',
content: systemMsg
})
currentTokens += (await tokenize(systemMsg) + 1)
currentTokens += (await tokenize(systemMsg) + 5)
}
if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){
@@ -246,7 +248,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
return false
}
currentTokens -= (await tokenize(chats[0].content) + 1)
currentTokens -= (await tokenize(chats[0].content) + 5)
chats.splice(0, 1)
}
currentChat.lastMemory = chats[0].memo

2
src/ts/process/supaMemory.ts
View File

@@ -171,7 +171,7 @@ export async function supaMemory(chats:OpenAIChat[],currentTokens:number,maxCont
}
continue
}
const tokens = await tokenize(cont.content) + 1
const tokens = await tokenize(cont.content) + 5
if((chunkSize + tokens) > maxChunkSize){
if(stringlizedChat === ''){
stringlizedChat += `${cont.role === 'assistant' ? char.type === 'group' ? '' : char.name : db.username}: ${cont.content}\n\n`