[feat] better ain formating

2023-06-26 17:54:10 +09:00
parent 85db744019
commit 35afc5f61c
6 changed files with 214 additions and 32 deletions
--- a/src/ts/process/index.ts
+++ b/src/ts/process/index.ts
@@ -433,16 +433,19 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
        await sayTTS(currentChar, result)
    }
    else{
-        const result2 = processScriptFull(nowChatroom, reformatContent(req.result), 'editoutput')
-        result = result2.data
-        emoChanged = result2.emoChanged
-        db.characters[selectedChar].chats[selectedChat].message.push({
-            role: 'char',
-            data: result,
-            saying: currentChar.chaId
-        })
-        await sayTTS(currentChar, result)
-        setDatabase(db)
+        const msgs = req.type === 'success' ? [['char',req.result]] as const : req.type === 'multiline' ? req.result : []
+        for(const msg of msgs){
+            const result2 = processScriptFull(nowChatroom, reformatContent(msg[1]), 'editoutput')
+            result = result2.data
+            emoChanged = result2.emoChanged
+            db.characters[selectedChar].chats[selectedChat].message.push({
+                role: msg[0],
+                data: result,
+                saying: currentChar.chaId
+            })
+            await sayTTS(currentChar, result)
+            setDatabase(db)
+        }
    }

    if(req.special){
@@ -547,7 +550,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
            maxTokens: 30,
        }, 'submodel')

-        if(rq.type === 'fail' || rq.type === 'streaming'){
+        if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
            alertError(`${rq.result}`)
            return true
        }
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -3,7 +3,7 @@ import type { OpenAIChat, OpenAIChatFull } from ".";
 import { DataBase, setDatabase, type character } from "../storage/database";
 import { pluginProcess } from "../plugins/plugins";
 import { language } from "../../lang";
-import { getUnstringlizerChunks, stringlizeAINChat, stringlizeChat, unstringlizeChat } from "./stringlize";
+import { stringlizeAINChat, stringlizeChat, unstringlizeAIN, unstringlizeChat } from "./stringlize";
 import { globalFetch, isNodeServer, isTauri } from "../storage/globalApi";
 import { sleep } from "../util";
 import { createDeep } from "./deepai";
@@ -35,6 +35,13 @@ type requestDataResponse = {
    special?: {
        emotion?: string
    }
+}|{
+    type: "multiline",
+    result: ['user'|'char',string][],
+    noRetry?: boolean,
+    special?: {
+        emotion?: string
+    }
 }

 interface OaiFunctions {
@@ -57,7 +64,7 @@ export async function requestChatData(arg:requestDataArgument, model:'model'|'su
    let trys = 0
    while(true){
        const da = await requestChatDataMain(arg, model, abortSignal)
-        if(da.type === 'success' || da.type === 'streaming' || da.noRetry){
+        if(da.type !== 'fail' || da.noRetry){
            return da
        }
        
@@ -574,13 +581,14 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                tailfree: 1.0,
                rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
                model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
-                userbadwords: [":","：",": ","： "].join("<<|>>")
+                userbadwords: ["【質問】"].join("<<|>>")
            };

            const response = await globalFetch(api_server_url + '/api', {
                method: 'POST',
                headers: headers,
                body: send_body,
+                plainFetchForce: true
            });

            if(!response.ok){
@@ -598,10 +606,10 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
            }

            const result = response.data.data[0];
-
+            const unstr = unstringlizeAIN(result, formated, currentChar?.name ?? '')
            return {
-                'type': 'success',
-                'result': unstringlizeChat(result, formated, currentChar?.name ?? '')
+                'type': 'multiline',
+                'result': unstr
            }
        }
        case "deepai":{
--- a/src/ts/process/stableDiff.ts
+++ b/src/ts/process/stableDiff.ts
@@ -80,7 +80,7 @@ export async function stableDiff(currentChar:character,prompt:string){
    }, 'submodel')


-    if(rq.type === 'fail' || rq.type === 'streaming'){
+    if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
        alertError(`${rq.result}`)
        return false
    }
--- a/src/ts/process/stringlize.ts
+++ b/src/ts/process/stringlize.ts
@@ -1,5 +1,7 @@
+import { get } from "svelte/store";
 import type { OpenAIChat } from ".";
 import { tokenize } from "../tokenizer";
+import { DataBase } from "../storage/database";

 export function multiChatReplacer(){

@@ -22,7 +24,6 @@ export function stringlizeChat(formated:OpenAIChat[], char:string = ''){
 }

 export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){
-    console.log(text)
    let minIndex = -1

    const chunks = getUnstringlizerChunks(formated, char)
@@ -45,18 +46,37 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
    return text
 }

-export function getUnstringlizerChunks(formated:OpenAIChat[], char:string = ''){
+export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:'ain'|'normal' = 'normal'){
    let chunks:string[] = ["system note:", "system:","system note：", "system："]
    if(char){
-        chunks.push(`${char}:`)
+        if(mode === 'ain'){
+            chunks.push(`${char} `)
+            chunks.push(`${char}　`)
+        }
+        else{
+            chunks.push(`${char}:`)
+            chunks.push(`${char}：`)
+            chunks.push(`${char}: `)
+            chunks.push(`${char}： `) 
+        }
    }

    for(const form of formated){
        if(form.name){
-            chunks.push(`${form.name}:`)
-            chunks.push(`${form.name}：`)
-            chunks.push(`${form.name}: `)
-            chunks.push(`${form.name}： `) 
+            if(mode === 'ain'){
+                if(!chunks.includes(`${form.name} `)){
+                    chunks.push(`${form.name} `)
+                    chunks.push(`${form.name}　`)
+                }
+            }
+            else{
+                if(!chunks.includes(`${form.name}:`)){
+                    chunks.push(`${form.name}:`)
+                    chunks.push(`${form.name}：`)
+                    chunks.push(`${form.name}: `)
+                    chunks.push(`${form.name}： `) 
+                }
+            }
        }
    }
    return chunks
@@ -64,20 +84,171 @@ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string = ''){

 export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){
    let resultString:string[] = []
+    const db = get(DataBase)
+
    for(const form of formated){
-        if(form.memo && form.memo.startsWith("newChat")){
+        console.log(form)
+        if(form.memo && form.memo.startsWith("newChat") || form.content === "[Start a new chat]"){
            resultString.push("[新しいチャットの始まり]")
            continue
        }
        if(form.role === 'system'){
            resultString.push(form.content)
        }
-        else if(form.name){
-            resultString.push(form.name + "： " + form.content)
+        else if(form.role === 'user'){
+            resultString.push(...formatToAIN(db.username, form.content))
+        }
+        else if(form.name || form.role === 'assistant'){
+            resultString.push(...formatToAIN(form.name ?? char, form.content))
        }
        else{
            resultString.push(form.content)
        }
+        console.log(resultString)
    }
-    return resultString.join('\n\n') + `\n\n${char}:`
+    return resultString.join('\n\n') + `\n\n${char} 「`
+}
+
+function extractAINOutputStrings(inputString:string, characters:string[]) {
+    let results:{
+        content:string
+        character:string
+    }[] = [];
+    
+    let remainingString = inputString;
+    
+    while (remainingString.length > 0) {
+        let characterIndex = -1;
+        let character = null;
+        for (let i = 0; i < characters.length; i++) {
+        const index = remainingString.indexOf(characters[i] + '「');
+        if (index >= 0 && (characterIndex == -1 || index < characterIndex)) {
+            character = characters[i];
+            characterIndex = index;
+        }
+        }
+    
+        if (characterIndex > 0) {
+        results.push({content: remainingString.substring(0, characterIndex).trim(), character: '[narrator]'});
+        }
+    
+        if (characterIndex == -1) {
+            results.push({content: remainingString.trim(),  character: '[narrator]'});
+            break;
+        } else {
+            let endQuoteIndex = remainingString.indexOf('」', characterIndex + character.length);
+            if (endQuoteIndex == -1) {
+                results.push({
+                character, 
+                content: remainingString.substring(characterIndex + character.length + 1).trim() // plus 1 to exclude 「
+                });
+                break;
+            } else {
+                results.push({
+                character, 
+                content: remainingString.substring(characterIndex + character.length + 1, endQuoteIndex).trim() // plus 1 to exclude 「
+                });
+                remainingString = remainingString.substring(endQuoteIndex + 1);
+            }
+        }
+    }
+
+    return results;
+}
+
+export function unstringlizeAIN(data:string,formated:OpenAIChat[], char:string = ''){
+
+    const db = get(DataBase)
+    const chunks = getUnstringlizerChunks(formated, char ,'ain')
+    let result:['char'|'user',string][] = []
+    data = `${char} 「` + data
+
+    const contents = extractAINOutputStrings(data, chunks)
+    for(const cont of contents){
+        if(cont.character === '[narrator]'){
+            if(result.length === 0){
+                result[0] = ['char', cont.content]
+            }
+            else{
+                result[result.length - 1][1] += "\n" + cont.content
+            }
+        }
+        else{
+            const role = (cont.character.trim() ===  db.username ? 'user' : 'char')
+            result.push([
+                role,
+                `「${cont.content}」`
+            ])
+        }
+    }
+
+    return result
+}
+
+
+function formatToAIN(name:string, content:string){
+    function extractContent(str:string) {
+        let result:{
+            type: "outside"|"inside"
+           content:string
+        }[] = [];
+        let lastEndIndex = 0;
+        let regex = /「(.*?)」/g;
+        let match:RegExpExecArray | null = null;
+
+        
+    
+        while ((match = regex.exec(str)) !== null) {
+            let start = match.index;
+            let end = start + match[0].length;
+            let inside = match[1];
+            
+            if (start != lastEndIndex) {
+                let outside = str.slice(lastEndIndex, start);
+                result.push({
+                    type: "outside",
+                    content: outside
+                });
+            }
+    
+            result.push({
+                type: "inside",
+                content: inside
+            });
+            
+            lastEndIndex = end;
+        }
+    
+        if (lastEndIndex < str.length) {
+            let outside = str.slice(lastEndIndex);
+            result.push({
+                type: "outside",
+                content: outside
+            });
+        }
+        
+        return result;
+    }
+
+    let quoteCounter = 0;
+    content = content.replace(/"/g, () => {
+        quoteCounter++;
+        if (quoteCounter % 2 !== 0) {
+            return '「';
+        } else {
+            return '」';
+        }
+    });
+
+    const conts = extractContent(content)
+    let strs:string[] = []
+    for(const cont of conts){
+        if(cont.type === 'inside'){
+            strs.push(`${name} 「${cont.content}」`)
+        }
+        else{
+            strs.push(cont.content)
+        }
+    }
+    return strs
 }
--- a/src/ts/process/supaMemory.ts
+++ b/src/ts/process/supaMemory.ts
@@ -123,7 +123,7 @@ export async function supaMemory(
                    formated: promptbody,
                    bias: {}
                }, 'submodel')
-                if(da.type === 'fail' || da.type === 'streaming'){
+                if(da.type === 'fail' || da.type === 'streaming' || da.type === 'multiline'){
                    return {
                        currentTokens: currentTokens,
                        chats: chats,
--- a/src/ts/storage/globalApi.ts
+++ b/src/ts/storage/globalApi.ts
@@ -380,7 +380,7 @@ export async function loadData() {

 const knownHostes = ["localhost","127.0.0.1"]

-export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:string]:string}, rawResponse?:boolean, method?:"POST"|"GET", abortSignal?:AbortSignal} = {}): Promise<{
+export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body?:any,headers?:{[key:string]:string}, rawResponse?:boolean, method?:"POST"|"GET", abortSignal?:AbortSignal} = {}): Promise<{
    ok: boolean;
    data: any;
    headers:{[key:string]:string}
@@ -414,7 +414,7 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri
        }
    
        const urlHost = (new URL(url)).hostname
-        let forcePlainFetch = (knownHostes.includes(urlHost) && (!isTauri)) || db.usePlainFetch 
+        let forcePlainFetch = (knownHostes.includes(urlHost) && (!isTauri)) || db.usePlainFetch || arg.plainFetchForce
        if(forcePlainFetch){
            try {
                let headers = arg.headers ?? {}