[feat] better ain formating

src/ts/process/index.ts

@@ -433,16 +433,19 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
         await sayTTS(currentChar, result)
     }
     else{
-        const result2 = processScriptFull(nowChatroom, reformatContent(req.result), 'editoutput')
+        const msgs = req.type === 'success' ? [['char',req.result]] as const : req.type === 'multiline' ? req.result : []
-        result = result2.data
+        for(const msg of msgs){
-        emoChanged = result2.emoChanged
+            const result2 = processScriptFull(nowChatroom, reformatContent(msg[1]), 'editoutput')
-        db.characters[selectedChar].chats[selectedChat].message.push({
+            result = result2.data
-            role: 'char',
+            emoChanged = result2.emoChanged
-            data: result,
+            db.characters[selectedChar].chats[selectedChat].message.push({
-            saying: currentChar.chaId
+                role: msg[0],
-        })
+                data: result,
-        await sayTTS(currentChar, result)
+                saying: currentChar.chaId
-        setDatabase(db)
+            })
+            await sayTTS(currentChar, result)
+            setDatabase(db)
+        }
     }
 
     if(req.special){
@@ -547,7 +550,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
             maxTokens: 30,
         }, 'submodel')
 
-        if(rq.type === 'fail' || rq.type === 'streaming'){
+        if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
             alertError(`${rq.result}`)
             return true
         }

src/ts/process/request.ts

@@ -3,7 +3,7 @@ import type { OpenAIChat, OpenAIChatFull } from ".";
 import { DataBase, setDatabase, type character } from "../storage/database";
 import { pluginProcess } from "../plugins/plugins";
 import { language } from "../../lang";
-import { getUnstringlizerChunks, stringlizeAINChat, stringlizeChat, unstringlizeChat } from "./stringlize";
+import { stringlizeAINChat, stringlizeChat, unstringlizeAIN, unstringlizeChat } from "./stringlize";
 import { globalFetch, isNodeServer, isTauri } from "../storage/globalApi";
 import { sleep } from "../util";
 import { createDeep } from "./deepai";
@@ -35,6 +35,13 @@ type requestDataResponse = {
     special?: {
         emotion?: string
     }
+}|{
+    type: "multiline",
+    result: ['user'|'char',string][],
+    noRetry?: boolean,
+    special?: {
+        emotion?: string
+    }
 }
 
 interface OaiFunctions {
@@ -57,7 +64,7 @@ export async function requestChatData(arg:requestDataArgument, model:'model'|'su
     let trys = 0
     while(true){
         const da = await requestChatDataMain(arg, model, abortSignal)
-        if(da.type === 'success' || da.type === 'streaming' || da.noRetry){
+        if(da.type !== 'fail' || da.noRetry){
             return da
         }
         
@@ -574,13 +581,14 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 tailfree: 1.0,
                 rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
                 model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
-                userbadwords: [":","：",": ","： "].join("<<|>>")
+                userbadwords: ["【質問】"].join("<<|>>")
             };
 
             const response = await globalFetch(api_server_url + '/api', {
                 method: 'POST',
                 headers: headers,
                 body: send_body,
+                plainFetchForce: true
             });
 
             if(!response.ok){
@@ -598,10 +606,10 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
             }
 
             const result = response.data.data[0];
-
+            const unstr = unstringlizeAIN(result, formated, currentChar?.name ?? '')
             return {
-                'type': 'success',
+                'type': 'multiline',
-                'result': unstringlizeChat(result, formated, currentChar?.name ?? '')
+                'result': unstr
             }
         }
         case "deepai":{

src/ts/process/stableDiff.ts

@@ -80,7 +80,7 @@ export async function stableDiff(currentChar:character,prompt:string){
     }, 'submodel')
 
 
-    if(rq.type === 'fail' || rq.type === 'streaming'){
+    if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
         alertError(`${rq.result}`)
         return false
     }

src/ts/process/stringlize.ts

@@ -1,5 +1,7 @@
+import { get } from "svelte/store";
 import type { OpenAIChat } from ".";
 import { tokenize } from "../tokenizer";
+import { DataBase } from "../storage/database";
 
 export function multiChatReplacer(){
 
@@ -22,7 +24,6 @@ export function stringlizeChat(formated:OpenAIChat[], char:string = ''){
 }
 
 export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){
-    console.log(text)
     let minIndex = -1
 
     const chunks = getUnstringlizerChunks(formated, char)
@@ -45,18 +46,37 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
     return text
 }
 
-export function getUnstringlizerChunks(formated:OpenAIChat[], char:string = ''){
+export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:'ain'|'normal' = 'normal'){
     let chunks:string[] = ["system note:", "system:","system note：", "system："]
     if(char){
-        chunks.push(`${char}:`)
+        if(mode === 'ain'){
+            chunks.push(`${char} `)
+            chunks.push(`${char}　`)
+        }
+        else{
+            chunks.push(`${char}:`)
+            chunks.push(`${char}：`)
+            chunks.push(`${char}: `)
+            chunks.push(`${char}： `) 
+        }
     }
 
     for(const form of formated){
         if(form.name){
-            chunks.push(`${form.name}:`)
+            if(mode === 'ain'){
-            chunks.push(`${form.name}：`)
+                if(!chunks.includes(`${form.name} `)){
-            chunks.push(`${form.name}: `)
+                    chunks.push(`${form.name} `)
-            chunks.push(`${form.name}： `) 
+                    chunks.push(`${form.name}　`)
+                }
+            }
+            else{
+                if(!chunks.includes(`${form.name}:`)){
+                    chunks.push(`${form.name}:`)
+                    chunks.push(`${form.name}：`)
+                    chunks.push(`${form.name}: `)
+                    chunks.push(`${form.name}： `) 
+                }
+            }
         }
     }
     return chunks
@@ -64,20 +84,171 @@ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string = ''){
 
 export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){
     let resultString:string[] = []
+    const db = get(DataBase)
+
     for(const form of formated){
-        if(form.memo && form.memo.startsWith("newChat")){
+        console.log(form)
+        if(form.memo && form.memo.startsWith("newChat") || form.content === "[Start a new chat]"){
             resultString.push("[新しいチャットの始まり]")
             continue
         }
         if(form.role === 'system'){
             resultString.push(form.content)
         }
-        else if(form.name){
+        else if(form.role === 'user'){
-            resultString.push(form.name + "： " + form.content)
+            resultString.push(...formatToAIN(db.username, form.content))
+        }
+        else if(form.name || form.role === 'assistant'){
+            resultString.push(...formatToAIN(form.name ?? char, form.content))
         }
         else{
             resultString.push(form.content)
         }
+        console.log(resultString)
     }
-    return resultString.join('\n\n') + `\n\n${char}:`
+    return resultString.join('\n\n') + `\n\n${char} 「`
+}
+
+function extractAINOutputStrings(inputString:string, characters:string[]) {
+    let results:{
+        content:string
+        character:string
+    }[] = [];
+    
+    let remainingString = inputString;
+    
+    while (remainingString.length > 0) {
+        let characterIndex = -1;
+        let character = null;
+        for (let i = 0; i < characters.length; i++) {
+        const index = remainingString.indexOf(characters[i] + '「');
+        if (index >= 0 && (characterIndex == -1 || index < characterIndex)) {
+            character = characters[i];
+            characterIndex = index;
+        }
+        }
+    
+        if (characterIndex > 0) {
+        results.push({content: remainingString.substring(0, characterIndex).trim(), character: '[narrator]'});
+        }
+    
+        if (characterIndex == -1) {
+            results.push({content: remainingString.trim(),  character: '[narrator]'});
+            break;
+        } else {
+            let endQuoteIndex = remainingString.indexOf('」', characterIndex + character.length);
+            if (endQuoteIndex == -1) {
+                results.push({
+                character, 
+                content: remainingString.substring(characterIndex + character.length + 1).trim() // plus 1 to exclude 「
+                });
+                break;
+            } else {
+                results.push({
+                character, 
+                content: remainingString.substring(characterIndex + character.length + 1, endQuoteIndex).trim() // plus 1 to exclude 「
+                });
+                remainingString = remainingString.substring(endQuoteIndex + 1);
+            }
+        }
+    }
+
+    return results;
+}
+
+export function unstringlizeAIN(data:string,formated:OpenAIChat[], char:string = ''){
+
+    const db = get(DataBase)
+    const chunks = getUnstringlizerChunks(formated, char ,'ain')
+    let result:['char'|'user',string][] = []
+    data = `${char} 「` + data
+
+    const contents = extractAINOutputStrings(data, chunks)
+    for(const cont of contents){
+        if(cont.character === '[narrator]'){
+            if(result.length === 0){
+                result[0] = ['char', cont.content]
+            }
+            else{
+                result[result.length - 1][1] += "\n" + cont.content
+            }
+        }
+        else{
+            const role = (cont.character.trim() ===  db.username ? 'user' : 'char')
+            result.push([
+                role,
+                `「${cont.content}」`
+            ])
+        }
+    }
+
+    return result
+}
+
+
+function formatToAIN(name:string, content:string){
+    function extractContent(str:string) {
+        let result:{
+            type: "outside"|"inside"
+           content:string
+        }[] = [];
+        let lastEndIndex = 0;
+        let regex = /「(.*?)」/g;
+        let match:RegExpExecArray | null = null;
+
+        
+    
+        while ((match = regex.exec(str)) !== null) {
+            let start = match.index;
+            let end = start + match[0].length;
+            let inside = match[1];
+            
+            if (start != lastEndIndex) {
+                let outside = str.slice(lastEndIndex, start);
+                result.push({
+                    type: "outside",
+                    content: outside
+                });
+            }
+    
+            result.push({
+                type: "inside",
+                content: inside
+            });
+            
+            lastEndIndex = end;
+        }
+    
+        if (lastEndIndex < str.length) {
+            let outside = str.slice(lastEndIndex);
+            result.push({
+                type: "outside",
+                content: outside
+            });
+        }
+        
+        return result;
+    }
+
+    let quoteCounter = 0;
+    content = content.replace(/"/g, () => {
+        quoteCounter++;
+        if (quoteCounter % 2 !== 0) {
+            return '「';
+        } else {
+            return '」';
+        }
+    });
+
+    const conts = extractContent(content)
+    let strs:string[] = []
+    for(const cont of conts){
+        if(cont.type === 'inside'){
+            strs.push(`${name} 「${cont.content}」`)
+        }
+        else{
+            strs.push(cont.content)
+        }
+    }
+    return strs
 }

src/ts/process/supaMemory.ts

@@ -123,7 +123,7 @@ export async function supaMemory(
                     formated: promptbody,
                     bias: {}
                 }, 'submodel')
-                if(da.type === 'fail' || da.type === 'streaming'){
+                if(da.type === 'fail' || da.type === 'streaming' || da.type === 'multiline'){
                     return {
                         currentTokens: currentTokens,
                         chats: chats,

src/ts/storage/globalApi.ts

@@ -380,7 +380,7 @@ export async function loadData() {
 
 const knownHostes = ["localhost","127.0.0.1"]
 
-export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:string]:string}, rawResponse?:boolean, method?:"POST"|"GET", abortSignal?:AbortSignal} = {}): Promise<{
+export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body?:any,headers?:{[key:string]:string}, rawResponse?:boolean, method?:"POST"|"GET", abortSignal?:AbortSignal} = {}): Promise<{
     ok: boolean;
     data: any;
     headers:{[key:string]:string}
@@ -414,7 +414,7 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri
         }
     
         const urlHost = (new URL(url)).hostname
-        let forcePlainFetch = (knownHostes.includes(urlHost) && (!isTauri)) || db.usePlainFetch 
+        let forcePlainFetch = (knownHostes.includes(urlHost) && (!isTauri)) || db.usePlainFetch || arg.plainFetchForce
         if(forcePlainFetch){
             try {
                 let headers = arg.headers ?? {}