[fix] tokenizer problem 2

2023-06-10 02:38:36 +09:00
parent 6fc60805e6
commit 08cd214bcb
3 changed files with 6 additions and 1 deletions
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -49,6 +49,7 @@ export class ChatTokenizer {
        this.useName = useName
    }
    async tokenizeChat(data:OpenAIChat) {
+        console.log(data.content)
        let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
        if(data.name && this.useName ==='name'){
            encoded += (await encode(data.name)).length