mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] accurate tokenizing

Browse Source
This commit is contained in:
kwaroran
2023-11-23 17:52:00 +09:00
parent dab121c9c7
commit 634fe418b4
6 changed files with 116 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/lib/Setting/Pages/BotSettings.svelte
View File

@@ -5,7 +5,7 @@
import { DataBase } from "src/ts/storage/database";
import { customProviderStore, getCurrentPluginMax } from "src/ts/plugins/plugins";
import { getModelMaxContext, isTauri } from "src/ts/storage/globalApi";
import { tokenize } from "src/ts/tokenizer";
import { tokenize, tokenizeAccurate } from "src/ts/tokenizer";
import ModelList from "src/lib/UI/ModelList.svelte";
import DropList from "src/lib/SideBars/DropList.svelte";
import { PlusIcon, TrashIcon } from "lucide-svelte";
@@ -40,10 +40,10 @@
export let goPromptTemplate = () => {}
async function loadTokenize(){
tokens.mainPrompt = await tokenize($DataBase.mainPrompt)
tokens.jailbreak = await tokenize($DataBase.jailbreak)
tokens.globalNote = await tokenize($DataBase.globalNote)
tokens.autoSuggest = await tokenize($DataBase.autoSuggestPrompt)
tokens.mainPrompt = await tokenizeAccurate($DataBase.mainPrompt)
tokens.jailbreak = await tokenizeAccurate($DataBase.jailbreak)
tokens.globalNote = await tokenizeAccurate($DataBase.globalNote)
tokens.autoSuggest = await tokenizeAccurate($DataBase.autoSuggestPrompt)
}
let advancedBotSettings = false

13
src/lib/Setting/Pages/PromptSettings.svelte
View File

@@ -2,7 +2,7 @@
import { ArrowLeft, PlusIcon } from "lucide-svelte";
import { language } from "src/lang";
import ProomptItem from "src/lib/UI/ProomptItem.svelte";
import type { Proompt } from "src/ts/process/proompt";
import { tokenizePreset, type Proompt } from "src/ts/process/proompt";
import { templateCheck } from "src/ts/process/templates/templateCheck";
import { DataBase } from "src/ts/storage/database";
@@ -10,8 +10,15 @@
let opened = 0
let warns: string[] = []
export let onGoBack: () => void = () => {}
let tokens = 0
executeTokenize($DataBase.promptTemplate)
async function executeTokenize(prest: Proompt[]){
tokens = await tokenizePreset(prest)
}
$: warns = templateCheck($DataBase)
$: executeTokenize($DataBase.promptTemplate)
</script>
<h2 class="mb-2 text-2xl font-bold mt-2 items-center flex">
@@ -71,4 +78,6 @@
type2: 'normal'
})
$DataBase.promptTemplate = value
}}><PlusIcon /></button>
}}><PlusIcon /></button>
<span class="text-textcolor2 mb-6 text-sm mt-2">{tokens} {language.tokens}</span>

8
src/lib/SideBars/CharConfig.svelte
View File

@@ -1,6 +1,6 @@
<script lang="ts">
import { language } from "../../lang";
import { tokenize } from "../../ts/tokenizer";
import { tokenize, tokenizeAccurate } from "../../ts/tokenizer";
import { DataBase, saveImage as saveAsset, type Database, type character, type groupChat } from "../../ts/storage/database";
import { selectedCharID } from "../../ts/stores";
import { PlusIcon, SmileIcon, TrashIcon, UserIcon, ActivityIcon, BookIcon, LoaderIcon, User, DnaIcon, CurlyBraces, Volume2Icon, XIcon } from 'lucide-svelte'
@@ -51,17 +51,17 @@
if(lasttokens.desc !== cha.desc){
if(cha.desc){
lasttokens.desc = cha.desc
tokens.desc = await tokenize(cha.desc)
tokens.desc = await tokenizeAccurate(cha.desc)
}
}
if(lasttokens.firstMsg !==chara.firstMessage){
lasttokens.firstMsg = chara.firstMessage
tokens.firstMsg = await tokenize(chara.firstMessage)
tokens.firstMsg = await tokenizeAccurate(chara.firstMessage)
}
}
if(lasttokens.localNote !== currentChar.data.chats[currentChar.data.chatPage].note){
lasttokens.localNote = currentChar.data.chats[currentChar.data.chatPage].note
tokens.localNote = await tokenize(currentChar.data.chats[currentChar.data.chatPage].note)
tokens.localNote = await tokenizeAccurate(currentChar.data.chats[currentChar.data.chatPage].note)
}

62
src/ts/parser.ts
View File

@@ -11,6 +11,7 @@ import { selectedCharID } from './stores';
import { calcString } from './process/infunctions';
import { findCharacterbyId } from './util';
import { getInlayImage } from './image';
import { cloneDeep } from 'lodash';
const convertora = new showdown.Converter({
simpleLineBreaks: true,
@@ -346,7 +347,14 @@ function wppParser(data:string){
const rgx = /(?:{{|<)(.+?)(?:}}|>)/gm
type matcherArg = {chatID:number,db:Database,chara:character|string,rmVar:boolean,var?:{[key:string]:string}}
type matcherArg = {
chatID:number,
db:Database,
chara:character|string,
rmVar:boolean,
var?:{[key:string]:string}
tokenizeAccurate?:boolean
}
const matcher = (p1:string,matcherArg:matcherArg) => {
if(p1.length > 10000){
return ''
@@ -390,7 +398,7 @@ const matcher = (p1:string,matcherArg:matcherArg) => {
case 'bot':{
let selectedChar = get(selectedCharID)
let currentChar = db.characters[selectedChar]
if(currentChar.type !== 'group'){
if(currentChar && currentChar.type !== 'group'){
return currentChar.name
}
if(chara){
@@ -484,6 +492,9 @@ const matcher = (p1:string,matcherArg:matcherArg) => {
return ''
}
case 'time':{
if(matcherArg.tokenizeAccurate){
return `00:00:00`
}
if(chatID === -1){
return "[Cannot get time]"
}
@@ -499,6 +510,9 @@ const matcher = (p1:string,matcherArg:matcherArg) => {
return date.toLocaleTimeString()
}
case 'date':{
if(matcherArg.tokenizeAccurate){
return `00:00:00`
}
if(chatID === -1){
return "[Cannot get time]"
}
@@ -513,6 +527,9 @@ const matcher = (p1:string,matcherArg:matcherArg) => {
return date.toLocaleDateString()
}
case 'idle_duration':{
if(matcherArg.tokenizeAccurate){
return `00:00:00`
}
if(chatID === -1){
return "[Cannot get time]"
}
@@ -622,11 +639,17 @@ const matcher = (p1:string,matcherArg:matcherArg) => {
if(p1.startsWith('random')){
if(p1.startsWith('random::')){
const randomIndex = Math.floor(Math.random() * (arra.length - 1)) + 1
if(matcherArg.tokenizeAccurate){
return arra[0]
}
return arra[randomIndex]
}
else{
const arr = p1.split(/\:|\,/g)
const randomIndex = Math.floor(Math.random() * (arr.length - 1)) + 1
if(matcherArg.tokenizeAccurate){
return arra[0]
}
return arr[randomIndex]
}
}
@@ -656,7 +679,7 @@ const smMatcher = (p1:string,matcherArg:matcherArg) => {
case 'bot':{
let selectedChar = get(selectedCharID)
let currentChar = db.characters[selectedChar]
if(currentChar.type !== 'group'){
if(currentChar && currentChar.type !== 'group'){
return currentChar.name
}
if(chara){
@@ -707,6 +730,7 @@ export function risuChatParser(da:string, arg:{
chara?:string|character|groupChat
rmVar?:boolean,
var?:{[key:string]:string}
tokenizeAccurate?:boolean
} = {}):string{
const chatID = arg.chatID ?? -1
const db = arg.db ?? get(DataBase)
@@ -724,6 +748,13 @@ export function risuChatParser(da:string, arg:{
chara = aChara
}
}
if(arg.tokenizeAccurate){
const db = arg.db ?? get(DataBase)
const selchar = chara ?? db.characters[get(selectedCharID)]
if(!selchar){
chara = 'bot'
}
}
let pointer = 0;
@@ -731,12 +762,16 @@ export function risuChatParser(da:string, arg:{
let pf = performance.now()
let v = new Uint8Array(512)
let pureMode = false
let commentMode = false
let commentLatest:string[] = [""]
let commentV = new Uint8Array(512)
const matcherObj = {
chatID: chatID,
chara: chara,
rmVar: arg.rmVar ?? false,
db: db,
var: arg.var ?? null
var: arg.var ?? null,
tokenizeAccurate: arg.tokenizeAccurate ?? false
}
while(pointer < da.length){
switch(da[pointer]){
@@ -791,6 +826,25 @@ export function risuChatParser(da:string, arg:{
pureMode = false
break
}
case 'Comment':{
if(!commentMode){
commentMode = true
commentLatest = nested.map((f) => f)
if(commentLatest[0].endsWith('\n')){
commentLatest[0] = commentLatest[0].substring(0, commentLatest[0].length - 1)
}
commentV = new Uint8Array(v)
}
break
}
case '/Comment':{
if(commentMode){
nested = commentLatest
v = commentV
commentMode = false
}
break
}
default:{
const mc = (pureMode) ? null : smMatcher(dat, matcherObj)
nested[0] += mc ?? `<${dat}>`

27
src/ts/process/proompt.ts
View File

@@ -1,3 +1,5 @@
import { tokenizeAccurate } from "../tokenizer";
export type Proompt = ProomptPlain|ProomptTyped|ProomptChat|ProomptAuthorNote;
export interface ProomptPlain {
@@ -23,4 +25,29 @@ export interface ProomptChat {
type: 'chat';
rangeStart: number;
rangeEnd: number|'end';
}
export async function tokenizePreset(proompts:Proompt[]){
let total = 0
for(const proompt of proompts){
switch(proompt.type){
case 'plain':
case 'jailbreak':{
total += await tokenizeAccurate(proompt.text)
break
}
case 'persona':
case 'description':
case 'lorebook':
case 'postEverything':
case 'authornote':
case 'memory':{
if(proompt.innerFormat){
total += await tokenizeAccurate(proompt.innerFormat)
}
break
}
}
}
return total
}

12
src/ts/tokenizer.ts
View File

@@ -4,6 +4,8 @@ import { DataBase, type character } from "./storage/database";
import { get } from "svelte/store";
import type { OpenAIChat } from "./process";
import { supportsInlayImage } from "./image";
import { risuChatParser } from "./parser";
import type { Proompt } from "./process/proompt";
async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
let db = get(DataBase)
@@ -85,6 +87,14 @@ export async function tokenize(data:string) {
return encoded.length
}
export async function tokenizeAccurate(data:string) {
data = risuChatParser(data.replace('{{slot}}',''), {
tokenizeAccurate: true
})
const encoded = await encode(data)
return encoded.length
}
export class ChatTokenizer {
@@ -149,4 +159,4 @@ export class ChatTokenizer {
export async function tokenizeNum(data:string) {
const encoded = await encode(data)
return encoded
}
}