[feat] new tokenizing

This commit is contained in:
kwaroran
2023-06-09 22:27:34 +09:00
parent cfe20c968b
commit dbf5f6b25a
5 changed files with 51 additions and 47 deletions

View File

@@ -53,15 +53,4 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
}
return text
}
export async function getNameMaxTokens(names:string[]){
let maxCharNameTokens = 0
for(const name of names){
const tokens = await tokenize(name + ': ') + 1
if(maxCharNameTokens < tokens){
maxCharNameTokens = tokens
}
}
return maxCharNameTokens
}