From fc80552749a01a188c1a6459d69ce43f77262bbf Mon Sep 17 00:00:00 2001
From: Kwaroran <amasio1234@proton.me>
Date: Tue, 25 Feb 2025 04:39:01 +0900
Subject: [PATCH] feat: enhance tokenizeChat to accept optional countThoughts
 argument

---
 src/ts/tokenizer.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index ddeb3794..e71528a4 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -313,7 +313,9 @@ export class ChatTokenizer {
         this.chatAdditionalTokens = chatAdditionalTokens
         this.useName = useName
     }
-    async tokenizeChat(data:OpenAIChat) {
+    async tokenizeChat(data:OpenAIChat, args:{
+        countThoughts?:boolean,
+    } = {}) {
         let encoded = (await encode(data.content)).length + this.chatAdditionalTokens
         if(data.name && this.useName ==='name'){
             encoded += (await encode(data.name)).length + 1
@@ -323,7 +325,7 @@ export class ChatTokenizer {
                 encoded += await this.tokenizeMultiModal(multimodal)
             }
         }
-        if(data.thoughts && data.thoughts.length > 0){
+        if(data.thoughts && data.thoughts.length > 0 && args.countThoughts){
             for(const thought of data.thoughts){
                 encoded += (await encode(thought)).length + 1
             }