mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add validation

Browse Source 
Also revoked potentially problematic feature(add hypav2data chunk)
TODO:
1. On mid-context editing, currently that is not considered as deletion. Do have optional editedChatIndex to latter dive in more.
2. re-roll mainChunks(re-summarization) functionalities added, but not able to access it.
This commit is contained in:
LightningHyperBlaze45654
2024-12-01 13:00:00 -08:00
parent 835664a7aa
commit 60d4e33893
4 changed files with 230 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/ts/tokenizer.ts
View File

@@ -222,15 +222,15 @@ export async function tokenizeAccurate(data:string, consistantChar?:boolean) {
export class ChatTokenizer {
private chatAdditonalTokens:number
private chatAdditionalTokens:number
private useName:'name'|'noName'
constructor(chatAdditonalTokens:number, useName:'name'|'noName'){
this.chatAdditonalTokens = chatAdditonalTokens
constructor(chatAdditionalTokens:number, useName:'name'|'noName'){
this.chatAdditionalTokens = chatAdditionalTokens
this.useName = useName
}
async tokenizeChat(data:OpenAIChat) {
let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
let encoded = (await encode(data.content)).length + this.chatAdditionalTokens
if(data.name && this.useName ==='name'){
encoded += (await encode(data.name)).length + 1
}
@@ -241,17 +241,24 @@ export class ChatTokenizer {
}
return encoded
}
async tokenizeChats(data:OpenAIChat[]){
let encoded = 0
for(const chat of data){
encoded += await this.tokenizeChat(chat)
}
return encoded
}
async tokenizeMultiModal(data:MultiModal){
const db = getDatabase()
if(!supportsInlayImage()){
return this.chatAdditonalTokens
return this.chatAdditionalTokens
}
if(db.gptVisionQuality === 'low'){
return 87
}
let encoded = this.chatAdditonalTokens
let encoded = this.chatAdditionalTokens
let height = data.height ?? 0
let width = data.width ?? 0