From 6fc60805e6eadb6cc722e0929909e5107a704311 Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Sat, 10 Jun 2023 02:27:54 +0900
Subject: [PATCH] [fix] tokenizer

---
 src/ts/tokenizer.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index 1cd1280f..52717d66 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -49,8 +49,10 @@ export class ChatTokenizer {
         this.useName = useName
     }
     async tokenizeChat(data:OpenAIChat) {
-        const encoded = (await encode(data.content)).length
-            + this.useName === 'name' ? (await encode(data.name)).length : 0 + this.chatAdditonalTokens
+        let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
+        if(data.name && this.useName ==='name'){
+            encoded += (await encode(data.name)).length
+        }
         return encoded
     }