mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance tokenizeChat to accept optional countThoughts argument

Browse Source
This commit is contained in:
Kwaroran
2025-02-25 04:39:01 +09:00
parent ed2957fc50
commit fc80552749
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/ts/tokenizer.ts
View File

@@ -313,7 +313,9 @@ export class ChatTokenizer {
this.chatAdditionalTokens = chatAdditionalTokens this.chatAdditionalTokens = chatAdditionalTokens
this.useName = useName this.useName = useName
} }
async tokenizeChat(data:OpenAIChat) { async tokenizeChat(data:OpenAIChat, args:{
countThoughts?:boolean,
} = {}) {
let encoded = (await encode(data.content)).length + this.chatAdditionalTokens let encoded = (await encode(data.content)).length + this.chatAdditionalTokens
if(data.name && this.useName ==='name'){ if(data.name && this.useName ==='name'){
encoded += (await encode(data.name)).length + 1 encoded += (await encode(data.name)).length + 1
@@ -323,7 +325,7 @@ export class ChatTokenizer {
encoded += await this.tokenizeMultiModal(multimodal) encoded += await this.tokenizeMultiModal(multimodal)
} }
} }
if(data.thoughts && data.thoughts.length > 0){ if(data.thoughts && data.thoughts.length > 0 && args.countThoughts){
for(const thought of data.thoughts){ for(const thought of data.thoughts){
encoded += (await encode(thought)).length + 1 encoded += (await encode(thought)).length + 1
} }