[feat] better ain stringlizer

src/ts/process/request.ts

@@ -3,7 +3,7 @@ import type { OpenAIChat, OpenAIChatFull } from ".";
 import { DataBase, setDatabase, type character } from "../storage/database";
 import { pluginProcess } from "../plugins/plugins";
 import { language } from "../../lang";
-import { stringlizeAINChat, stringlizeChat, unstringlizeChat } from "./stringlize";
+import { getUnstringlizerChunks, stringlizeAINChat, stringlizeChat, unstringlizeChat } from "./stringlize";
 import { globalFetch, isNodeServer, isTauri } from "../storage/globalApi";
 import { sleep } from "../util";
 import { createDeep } from "./deepai";
@@ -573,7 +573,8 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 top_p: 0.7,
                 tailfree: 1.0,
                 rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
-                model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin'
+                model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
+                userbadwords: [":","：",": ","： "].join("<<|>>")
             };
 
             const response = await globalFetch(api_server_url + '/api', {

src/ts/process/stringlize.ts

@@ -24,18 +24,9 @@ export function stringlizeChat(formated:OpenAIChat[], char:string = ''){
 export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){
     console.log(text)
     let minIndex = -1
-    let chunks:string[] = ["system note:", "system:","system note：", "system："]
-    if(char){
-        chunks.push(`${char}:`)
-    }
 
-    for(const form of formated){
-        if(form.name){
-            chunks.push(`${form.name}:`)
-            chunks.push(`${form.name}：`)
-            
-        }
-    }
+    const chunks = getUnstringlizerChunks(formated, char)
+
 
     for(const chunk of chunks){
         const ind = text.indexOf(chunk)
@@ -54,6 +45,23 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
     return text
 }
 
+export function getUnstringlizerChunks(formated:OpenAIChat[], char:string = ''){
+    let chunks:string[] = ["system note:", "system:","system note：", "system："]
+    if(char){
+        chunks.push(`${char}:`)
+    }
+
+    for(const form of formated){
+        if(form.name){
+            chunks.push(`${form.name}:`)
+            chunks.push(`${form.name}：`)
+            chunks.push(`${form.name}: `)
+            chunks.push(`${form.name}： `) 
+        }
+    }
+    return chunks
+}
+
 export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){
     let resultString:string[] = []
     for(const form of formated){