mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] tokenize problem

Browse Source
This commit is contained in:
kwaroran
2023-05-29 20:41:33 +09:00
parent bdf00126cc
commit 71b02cf104
2 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/ts/process/index.ts
View File

@@ -93,6 +93,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
} }
} }
let unformated = { let unformated = {
'main':([] as OpenAIChat[]), 'main':([] as OpenAIChat[]),
'jailbreak':([] as OpenAIChat[]), 'jailbreak':([] as OpenAIChat[]),
@@ -162,10 +163,11 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
}).join('\n\n') }).join('\n\n')
}).join('\n\n')) + db.maxResponse) + 150 }).join('\n\n')) + db.maxResponse) + 150
const examples = exampleMessage(currentChar) const examples = exampleMessage(currentChar)
for(const example of examples){ for(const example of examples){
currentTokens += await tokenize(example.content) currentTokens += await tokenize(example.content) + 5
} }
let chats:OpenAIChat[] = examples let chats:OpenAIChat[] = examples
@@ -215,7 +217,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
memo: msg.chatId, memo: msg.chatId,
name: name name: name
}) })
currentTokens += (await tokenize(formedChat) + 1) currentTokens += (await tokenize(formedChat) + 5)
} }
if(nowChatroom.type === 'group'){ if(nowChatroom.type === 'group'){
@@ -224,7 +226,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
role: 'system', role: 'system',
content: systemMsg content: systemMsg
}) })
currentTokens += (await tokenize(systemMsg) + 1) currentTokens += (await tokenize(systemMsg) + 5)
} }
if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){ if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){
@@ -246,7 +248,7 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
return false return false
} }
currentTokens -= (await tokenize(chats[0].content) + 1) currentTokens -= (await tokenize(chats[0].content) + 5)
chats.splice(0, 1) chats.splice(0, 1)
} }
currentChat.lastMemory = chats[0].memo currentChat.lastMemory = chats[0].memo

2
src/ts/process/supaMemory.ts
View File

@@ -171,7 +171,7 @@ export async function supaMemory(chats:OpenAIChat[],currentTokens:number,maxCont
} }
continue continue
} }
const tokens = await tokenize(cont.content) + 1 const tokens = await tokenize(cont.content) + 5
if((chunkSize + tokens) > maxChunkSize){ if((chunkSize + tokens) > maxChunkSize){
if(stringlizedChat === ''){ if(stringlizedChat === ''){
stringlizedChat += `${cont.role === 'assistant' ? char.type === 'group' ? '' : char.name : db.username}: ${cont.content}\n\n` stringlizedChat += `${cont.role === 'assistant' ? char.type === 'group' ? '' : char.name : db.username}: ${cont.content}\n\n`