From ab61c63c4ca431430cf31ba7f929a9d5a744cf6d Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Mon, 26 Jun 2023 00:45:35 +0900
Subject: [PATCH] [feat] better ain stringlizer

---
 src/ts/process/request.ts       |  5 +++--
 src/ts/process/stringlize.ts    | 30 +++++++++++++++++++-----------
 src/ts/translator/translator.ts | 10 +++++-----
 3 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 9abf0e9b..b8516440 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -3,7 +3,7 @@ import type { OpenAIChat, OpenAIChatFull } from ".";
 import { DataBase, setDatabase, type character } from "../storage/database";
 import { pluginProcess } from "../plugins/plugins";
 import { language } from "../../lang";
-import { stringlizeAINChat, stringlizeChat, unstringlizeChat } from "./stringlize";
+import { getUnstringlizerChunks, stringlizeAINChat, stringlizeChat, unstringlizeChat } from "./stringlize";
 import { globalFetch, isNodeServer, isTauri } from "../storage/globalApi";
 import { sleep } from "../util";
 import { createDeep } from "./deepai";
@@ -573,7 +573,8 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 top_p: 0.7,
                 tailfree: 1.0,
                 rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
-                model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin'
+                model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
+                userbadwords: [":","：",": ","： "].join("<<|>>")
             };
 
             const response = await globalFetch(api_server_url + '/api', {
diff --git a/src/ts/process/stringlize.ts b/src/ts/process/stringlize.ts
index 641e3b11..36b9316b 100644
--- a/src/ts/process/stringlize.ts
+++ b/src/ts/process/stringlize.ts
@@ -24,18 +24,9 @@ export function stringlizeChat(formated:OpenAIChat[], char:string = ''){
 export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){
     console.log(text)
     let minIndex = -1
-    let chunks:string[] = ["system note:", "system:","system note：", "system："]
-    if(char){
-        chunks.push(`${char}:`)
-    }
 
-    for(const form of formated){
-        if(form.name){
-            chunks.push(`${form.name}:`)
-            chunks.push(`${form.name}：`)
-            
-        }
-    }
+    const chunks = getUnstringlizerChunks(formated, char)
+
 
     for(const chunk of chunks){
         const ind = text.indexOf(chunk)
@@ -54,6 +45,23 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
     return text
 }
 
+export function getUnstringlizerChunks(formated:OpenAIChat[], char:string = ''){
+    let chunks:string[] = ["system note:", "system:","system note：", "system："]
+    if(char){
+        chunks.push(`${char}:`)
+    }
+
+    for(const form of formated){
+        if(form.name){
+            chunks.push(`${form.name}:`)
+            chunks.push(`${form.name}：`)
+            chunks.push(`${form.name}: `)
+            chunks.push(`${form.name}： `) 
+        }
+    }
+    return chunks
+}
+
 export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){
     let resultString:string[] = []
     for(const form of formated){
diff --git a/src/ts/translator/translator.ts b/src/ts/translator/translator.ts
index 15aedbe3..643c1122 100644
--- a/src/ts/translator/translator.ts
+++ b/src/ts/translator/translator.ts
@@ -28,16 +28,16 @@ export async function translate(text:string, reverse:boolean) {
         }
     }
 
-    return googleTrans(text, reverse, db.aiModel.startsWith('novellist') ? 'jp' : 'en')
+    return googleTrans(text, reverse, db.translator,db.aiModel.startsWith('novellist') ? 'jp' : 'en')
 }
 
-async function googleTrans(text:string, reverse:boolean, target:'en'|'jp') {
+async function googleTrans(text:string, reverse:boolean, from:string,target:'en'|'jp') {
     let db = get(DataBase)
     const arg = {
 
-        from: reverse ? db.translator : target,
+        from: reverse ? from : target,
 
-        to: reverse ? target : db.translator,
+        to: reverse ? target : from,
 
         host: 'translate.googleapis.com',
 
@@ -89,5 +89,5 @@ async function jpTrans(text:string) {
     if(/[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uff00-\uff9f\u4e00-\u9faf\u3400-\u4dbf]/.test(text)){
         return text
     }
-    return await googleTrans(text,false, 'jp')
+    return await googleTrans(text,false, 'en','jp')
 }
\ No newline at end of file