[fix] tokenizer

2023-06-10 02:27:54 +09:00
parent e88f95102b
commit 6fc60805e6
1 changed files with 4 additions and 2 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -49,8 +49,10 @@ export class ChatTokenizer {
        this.useName = useName
    }
    async tokenizeChat(data:OpenAIChat) {
-        const encoded = (await encode(data.content)).length
-            + this.useName === 'name' ? (await encode(data.name)).length : 0 + this.chatAdditonalTokens
+        let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
+        if(data.name && this.useName ==='name'){
+            encoded += (await encode(data.name)).length
+        }
        return encoded
    }