From 08cd214bcbd662fc41bee7e74c3798c033e219bd Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Sat, 10 Jun 2023 02:38:36 +0900
Subject: [PATCH] [fix] tokenizer problem 2

---
 src/lib/ChatScreens/DefaultChatScreen.svelte | 1 +
 src/ts/process/index.ts                      | 5 ++++-
 src/ts/tokenizer.ts                          | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/lib/ChatScreens/DefaultChatScreen.svelte b/src/lib/ChatScreens/DefaultChatScreen.svelte
index 28afd787..30db6bee 100644
--- a/src/lib/ChatScreens/DefaultChatScreen.svelte
+++ b/src/lib/ChatScreens/DefaultChatScreen.svelte
@@ -144,6 +144,7 @@
         try {
             await sendChat()            
         } catch (error) {
+            console.error(error)
             alertError(`${error}`)
         }
         rerolls.push(cloneDeep($DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message))
diff --git a/src/ts/process/index.ts b/src/ts/process/index.ts
index eac048aa..f2447726 100644
--- a/src/ts/process/index.ts
+++ b/src/ts/process/index.ts
@@ -202,7 +202,10 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
     let currentTokens = 0
     
     for(const key in unformated){
-        currentTokens += await tokenizer.tokenizeChat(unformated[key])
+        const chats = unformated[key] as OpenAIChat[]
+        for(const chat of chats){
+            currentTokens += await tokenizer.tokenizeChat(chat)
+        }
     }
 
     
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index 52717d66..2d162bd7 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -49,6 +49,7 @@ export class ChatTokenizer {
         this.useName = useName
     }
     async tokenizeChat(data:OpenAIChat) {
+        console.log(data.content)
         let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
         if(data.name && this.useName ==='name'){
             encoded += (await encode(data.name)).length