feat: add image translation feature and enhance regex list functionality

This commit is contained in:
Kwaroran
2024-12-27 15:51:29 +09:00
parent c5f5786af7
commit 191be6d5c1
12 changed files with 376 additions and 69 deletions

View File

@@ -312,6 +312,11 @@ export class ChatTokenizer {
encoded += await this.tokenizeMultiModal(multimodal)
}
}
if(data.thoughts && data.thoughts.length > 0){
for(const thought of data.thoughts){
encoded += (await encode(thought)).length + 1
}
}
return encoded
}
async tokenizeChats(data:OpenAIChat[]){