From cd092a5a01ddde97b2580cef6c2a1ff7174a416b Mon Sep 17 00:00:00 2001
From: Kwaroran <amasio1234@proton.me>
Date: Sun, 29 Dec 2024 04:03:21 +0900
Subject: [PATCH] feat: add translation feature and ban character set
 functionality

---
 src/lang/en.ts                                |  1 +
 src/lib/Playground/PlaygroundMenu.svelte      |  9 +++
 src/lib/Playground/PlaygroundSubtitle.svelte  | 36 +----------
 .../Playground/PlaygroundTranslation.svelte   | 62 +++++++++++++++++++
 src/lib/Setting/Pages/AdvancedSettings.svelte | 56 +++++++++++++++++
 src/ts/characterCards.ts                      | 17 ++++-
 src/ts/globalApi.svelte.ts                    | 32 ++++++++++
 src/ts/pngChunk.ts                            | 16 ++++-
 src/ts/process/processzip.ts                  |  7 ++-
 src/ts/process/request.ts                     | 13 ++++
 src/ts/storage/database.svelte.ts             |  2 +
 src/ts/translator/translator.ts               | 26 +++-----
 12 files changed, 221 insertions(+), 56 deletions(-)
 create mode 100644 src/lib/Playground/PlaygroundTranslation.svelte

diff --git a/src/lang/en.ts b/src/lang/en.ts
index e8dcd718..2145bb5d 100644
--- a/src/lang/en.ts
+++ b/src/lang/en.ts
@@ -837,4 +837,5 @@ export const languageEnglish = {
     showSavingIcon: "Show Saving Icon",
     pluginVersionWarn: "This is {{plugin_version}} version of the plugin. which is not compatible with this version of RisuAI. please update the plugin to {{required_version}} version.",
     imageTranslation: "Image Translation",
+    banCharacterset: 'Auto Regenerate On Characterset'
 }
\ No newline at end of file
diff --git a/src/lib/Playground/PlaygroundMenu.svelte b/src/lib/Playground/PlaygroundMenu.svelte
index 829557bf..2c7cc682 100644
--- a/src/lib/Playground/PlaygroundMenu.svelte
+++ b/src/lib/Playground/PlaygroundMenu.svelte
@@ -16,6 +16,7 @@
     import { joinMultiuserRoom } from "src/ts/sync/multiuser";
   import PlaygroundSubtitle from "./PlaygroundSubtitle.svelte";
   import PlaygroundImageTrans from "./PlaygroundImageTrans.svelte";
+  import PlaygroundTranslation from "./PlaygroundTranslation.svelte";
 
     let easterEggTouch = $state(0)
 
@@ -95,6 +96,11 @@
             }}>
                 <h1 class="text-2xl font-bold text-start">{language.imageTranslation}</h1>
             </button>
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" onclick={() => {
+                PlaygroundStore.set(11)
+            }}>
+                <h1 class="text-2xl font-bold text-start">{language.translator}</h1>
+            </button>
             <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" onclick={() => {
                 PlaygroundStore.set(101)
             }}>
@@ -157,6 +163,9 @@
             {#if $PlaygroundStore === 10}
                <PlaygroundImageTrans/>
             {/if}
+            {#if $PlaygroundStore === 11}
+                <PlaygroundTranslation/>
+            {/if}
             {#if $PlaygroundStore === 101}
                 <ToolConvertion/>
             {/if}  
diff --git a/src/lib/Playground/PlaygroundSubtitle.svelte b/src/lib/Playground/PlaygroundSubtitle.svelte
index 432d3a40..e289aca2 100644
--- a/src/lib/Playground/PlaygroundSubtitle.svelte
+++ b/src/lib/Playground/PlaygroundSubtitle.svelte
@@ -10,7 +10,7 @@
     import { selectFileByDom, selectSingleFile, sleep } from "src/ts/util";
     import { alertError, alertSelect } from "src/ts/alert";
     import { risuChatParser } from "src/ts/parser.svelte";
-    import { AppendableBuffer, downloadFile, globalFetch } from "src/ts/globalApi.svelte";
+    import { AppendableBuffer, downloadFile, getLanguageCodes, globalFetch } from "src/ts/globalApi.svelte";
     import SliderInput from "../UI/GUI/SliderInput.svelte";
     import SelectInput from "../UI/GUI/SelectInput.svelte";
     import OptionInput from "../UI/GUI/OptionInput.svelte";
@@ -31,40 +31,6 @@
     let mode = $state('llm')
     let sourceLang:string|null = $state(null)    
 
-    function getLanguageCodes(){
-        let languageCodes:{
-            code: string
-            name: string
-        }[] = []
-
-        for(let i=0x41;i<=0x5A;i++){
-            for(let j=0x41;j<=0x5A;j++){
-                languageCodes.push({
-                    code: String.fromCharCode(i) + String.fromCharCode(j),
-                    name: ''
-                })
-            }
-        }
-        
-        languageCodes = languageCodes.map(v => {
-            return {
-                code: v.code,
-                name: new Intl.DisplayNames([
-                    DBState.db.language === 'cn' ? 'zh' : DBState.db.language
-                ], {
-                    type: 'language',
-                    fallback: 'none'
-                }).of(v.code)
-            }
-        }).filter((a) => {
-            return a.name
-        }).sort((a, b) => a.name.localeCompare(b.name))
-
-        return languageCodes
-    }
-
-
-
     async function runLLMMode() {
         outputText = 'Loading...\n\n'
 
diff --git a/src/lib/Playground/PlaygroundTranslation.svelte b/src/lib/Playground/PlaygroundTranslation.svelte
new file mode 100644
index 00000000..61d44731
--- /dev/null
+++ b/src/lib/Playground/PlaygroundTranslation.svelte
@@ -0,0 +1,62 @@
+
+<script lang="ts">
+    import { language } from "src/lang";
+    import TextAreaInput from "../UI/GUI/TextAreaInput.svelte";
+    import { LLMCacheStorage, runTranslator } from "src/ts/translator/translator";
+    import Button from "../UI/GUI/Button.svelte";
+    import SelectInput from "../UI/GUI/SelectInput.svelte";
+    import { getLanguageCodes } from "src/ts/globalApi.svelte";
+    import OptionInput from "../UI/GUI/OptionInput.svelte";
+
+
+    const userPreferedLang = navigator.language.split('-')[0]
+
+    let r = $state('')
+    let sourceLang = $state('en')
+    let output = $state('')
+    let outputLang = $state(userPreferedLang)
+    let loading = $state(false)
+</script>
+
+
+
+<span>{language.sourceLanguage}</span>
+<SelectInput bind:value={sourceLang}>
+    {#each getLanguageCodes() as lang}
+        <OptionInput value={lang.code}>{lang.name}</OptionInput>
+    {/each}
+</SelectInput>
+<TextAreaInput bind:value={r} />
+
+<span>{language.translatorLanguage}</span>
+<SelectInput bind:value={outputLang}>
+    {#each getLanguageCodes() as lang}
+        <OptionInput value={lang.code}>{lang.name}</OptionInput>
+    {/each}
+</SelectInput>
+<TextAreaInput value={output} />
+
+<Button className="mt-4" onclick={async () => {
+    try {
+        if(loading){
+            return
+        }
+        loading = true
+        output = await runTranslator(r, false, sourceLang, outputLang)   
+        loading = false
+    } catch (error) {
+        console.error(error)
+        loading = false
+    }
+}}>
+    {#if loading}
+        Loading...
+    {:else}
+        Translate
+    {/if}
+</Button>
+<Button className="mt-4" onclick={() => {
+    LLMCacheStorage.clear()
+}}>
+    Clear Cache
+</Button>
\ No newline at end of file
diff --git a/src/lib/Setting/Pages/AdvancedSettings.svelte b/src/lib/Setting/Pages/AdvancedSettings.svelte
index 1cd0b6dd..5a4f2803 100644
--- a/src/lib/Setting/Pages/AdvancedSettings.svelte
+++ b/src/lib/Setting/Pages/AdvancedSettings.svelte
@@ -13,12 +13,53 @@
     import { installPython } from "src/ts/process/models/local";
     import { Capacitor } from "@capacitor/core";
     import { capStorageInvestigation } from "src/ts/storage/mobileStorage";
+    import Arcodion from "src/lib/UI/Arcodion.svelte";
 
     let estaStorage:{
         key:string,
         size:string,
     }[] = $state([])
 
+    const characterSets = [
+        'Latn',
+        'Hani',
+        'Arab',
+        'Deva',
+        'Cyrl',
+        'Beng',
+        'Hrkt',
+        'Telu',
+        'Hang',
+        'Taml',
+        'Thai',
+        'Gujr',
+        'Knda',
+        'Ethi',
+        'Khmr',
+        'Grek',
+        'Hebr',
+    ]
+
+    const characterSetsPreview = {
+        'Latn': "ABC",
+        'Hani': "汉漢",
+        'Arab': "اعب",
+        'Deva': "अआइ",
+        'Cyrl': "АБВ",
+        'Beng': "অআই",
+        'Hrkt': "あア",
+        'Telu': "అఆఇ",
+        'Hang': "가나다",
+        'Taml': "அஆஇ",
+        'Thai': "กขค",
+        'Gujr': "અઆઇ",
+        'Knda': "ಅಆಇ",
+        'Ethi': "ሀሁሂ",
+        'Khmr': "កខគ",
+        'Grek': "ΑΒΓ",
+        'Hebr': "אבג",
+
+    }
 </script>
 <h2 class="text-2xl font-bold mt-2">{language.advancedSettings}</h2>
 <span class="text-draculared text-xs mb-2">{language.advancedSettingsWarn}</span>
@@ -168,6 +209,21 @@
         <Check bind:check={DBState.db.usePlainFetch} name={language.forcePlainFetch}> <Help key="forcePlainFetch" unrecommended/></Check>
     </div>
 {/if}
+
+<Arcodion styled name={language.banCharacterset}>
+    {#each characterSets as set}
+        <Button styled={DBState.db.banCharacterset.includes(set) ? 'primary' : "outlined"} onclick={(e) => {
+            if (DBState.db.banCharacterset.includes(set)) {
+                DBState.db.banCharacterset = DBState.db.banCharacterset.filter((item) => item !== set)
+            } else {
+                DBState.db.banCharacterset.push(set)
+            }
+        }}>
+            {new Intl.DisplayNames([navigator.language,'en'], { type: 'script' }).of(set)} ({characterSetsPreview[set]})
+        </Button>
+    {/each}
+</Arcodion>
+
 <Button
     className="mt-4"
     onclick={async () => {
diff --git a/src/ts/characterCards.ts b/src/ts/characterCards.ts
index ff1e2892..e392a281 100644
--- a/src/ts/characterCards.ts
+++ b/src/ts/characterCards.ts
@@ -7,7 +7,7 @@ import { v4 as uuidv4, v4 } from 'uuid';
 import { characterFormatUpdate } from "./characters"
 import { AppendableBuffer, BlankWriter, checkCharOrder, downloadFile, isNodeServer, isTauri, loadAsset, LocalWriter, openURL, readImage, saveAsset, VirtualWriter } from "./globalApi.svelte"
 import { SettingsMenuIndex, ShowRealmFrameStore, selectedCharID, settingsOpen } from "./stores.svelte"
-import { convertImage, hasher } from "./parser.svelte"
+import { checkImageType, convertImage, hasher } from "./parser.svelte"
 import { CCardLib, type CharacterCardV3, type LorebookEntry } from '@risuai/ccardlib'
 import { reencodeImage } from "./process/files/inlays"
 import { PngChunk } from "./pngChunk"
@@ -1316,6 +1316,21 @@ export async function exportCharacterCard(char:character, type:'png'|'json'|'cha
                             path = `assets/${type}/${itype}/${name}.${card.data.assets[i].ext}`
                         }
                         card.data.assets[i].uri = 'embeded://' + path
+                        const imageType = checkImageType(rData)
+                        if(imageType === 'PNG' && writer instanceof CharXWriter){
+                            const metadatas:Record<string,string> = {}
+                            const gen = PngChunk.readGenerator(rData)
+                            for await (const chunk of gen){
+                                if(!chunk || chunk instanceof AppendableBuffer){
+                                    continue
+                                }
+                                metadatas[chunk.key] = chunk.value
+                            }
+                            if(Object.keys(metadatas).length > 0){
+                                const metaPath = `x_meta/${name}.json`
+                                await writer.write(metaPath, Buffer.from(JSON.stringify(metadatas, null, 4)), 6)
+                            }
+                        }
                         await writer.write(path, Buffer.from(await convertImage(rData)))
                     }
                 }
diff --git a/src/ts/globalApi.svelte.ts b/src/ts/globalApi.svelte.ts
index 1e8718e9..a82ba3db 100644
--- a/src/ts/globalApi.svelte.ts
+++ b/src/ts/globalApi.svelte.ts
@@ -2261,4 +2261,36 @@ export class PerformanceDebugger{
             this.kv[key].push(...other.kv[key])
         }
     }
+}
+
+export function getLanguageCodes(){
+    let languageCodes:{
+        code: string
+        name: string
+    }[] = []
+
+    for(let i=0x41;i<=0x5A;i++){
+        for(let j=0x41;j<=0x5A;j++){
+            languageCodes.push({
+                code: String.fromCharCode(i) + String.fromCharCode(j),
+                name: ''
+            })
+        }
+    }
+    
+    languageCodes = languageCodes.map(v => {
+        return {
+            code: v.code.toLocaleLowerCase(),
+            name: new Intl.DisplayNames([
+                DBState.db.language === 'cn' ? 'zh' : DBState.db.language
+            ], {
+                type: 'language',
+                fallback: 'none'
+            }).of(v.code)
+        }
+    }).filter((a) => {
+        return a.name
+    }).sort((a, b) => a.name.localeCompare(b.name))
+
+    return languageCodes
 }
\ No newline at end of file
diff --git a/src/ts/pngChunk.ts b/src/ts/pngChunk.ts
index 57dca89b..e74b4bb3 100644
--- a/src/ts/pngChunk.ts
+++ b/src/ts/pngChunk.ts
@@ -26,7 +26,21 @@ class StreamChunkWriter{
                 break
             }
             if(typeString === 'tEXt'){
-                pos += 12 + len
+                const endPos = 12 + len + pos
+                //get key
+                let key=''
+                while(data[pos+8] !== 0){
+                    key += String.fromCharCode(data[pos+8])
+                    pos++
+                    if(pos === endPos || key.length > 6){
+                        break
+                    }
+                }
+
+                if(key !== 'ccv3' && key !== 'chara'){
+                    await this.pushData(data.slice(pos,endPos))
+                }
+                pos = endPos
             }
             else{
                 await this.pushData(data.slice(pos,pos+12+len))
diff --git a/src/ts/process/processzip.ts b/src/ts/process/processzip.ts
index 14160f44..9641b8d0 100644
--- a/src/ts/process/processzip.ts
+++ b/src/ts/process/processzip.ts
@@ -41,7 +41,7 @@ export class CharXWriter{
         //do nothing, just to make compatible with other writer
     }
 
-    async write(key:string,data:Uint8Array|string){
+    async write(key:string,data:Uint8Array|string, level?:0|1|2|3|4|5|6|7|8|9){
         console.log('write',key)
         let dat:Uint8Array
         if(typeof data === 'string'){
@@ -52,7 +52,7 @@ export class CharXWriter{
         }
         this.writeEnd = false
         const file = new fflate.ZipDeflate(key, {
-            level: 0
+            level: level ?? 0
         });
         await this.zip.add(file)
         await file.push(dat, true)
@@ -102,6 +102,9 @@ export class CharXReader{
                     else if(file.name === 'module.risum'){
                         this.moduleData = assetData
                     }
+                    else if(file.name.endsWith('.json')){
+                        //do nothing
+                    }
                     else{
                         this.assetPromises.push((async () => {
                             const assetId = await saveAsset(assetData)
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 23f9421f..e68647c7 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -226,6 +226,19 @@ export async function requestChatData(arg:requestDataArgument, model:ModelModeEx
             }
         }
 
+        if(da.type === 'success' && db.banCharacterset?.length > 0){
+            for(const set of db.banCharacterset){
+                const checkRegex = new RegExp(`\\p{Script=${set}}`, 'gu')
+
+                if(checkRegex.test(da.result)){
+                    return {
+                        type: 'fail',
+                        result: 'Banned character found'
+                    }
+                }
+            }
+        }
+
 
         if(da.type !== 'fail' || da.noRetry){
             return da
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index 18a0115b..67c1f8a8 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -467,6 +467,7 @@ export function setDatabase(data:Database){
     data.enableCustomFlags ??= false
     data.assetMaxDifference ??= 4
     data.showSavingIcon ??= false
+    data.banCharacterset ??= []
     changeLanguage(data.language)
     setDatabaseLite(data)
 }
@@ -866,6 +867,7 @@ export interface Database{
     pluginV2: RisuPlugin[]
     showSavingIcon:boolean
     presetRegex: customscript[]
+    banCharacterset:string[]
 }
 
 interface SeparateParameters{
diff --git a/src/ts/translator/translator.ts b/src/ts/translator/translator.ts
index 587c4582..542aae0b 100644
--- a/src/ts/translator/translator.ts
+++ b/src/ts/translator/translator.ts
@@ -17,7 +17,7 @@ let cache={
     trans: ['']
 }
 
-const LLMCacheStorage = localforage.createInstance({
+export const LLMCacheStorage = localforage.createInstance({
     name: "LLMTranslateCache"
 })
 
@@ -25,10 +25,6 @@ let waitTrans = 0
 
 export async function translate(text:string, reverse:boolean) {
     let db = getDatabase()
-    const plug = await translatorPlugin(text, reverse ? db.translator: 'en', reverse ? 'en' : db.translator)
-    if(plug){
-        return plug.content
-    }
     if(!reverse){
         const ind = cache.origin.indexOf(text)
         if(ind !== -1){
@@ -112,7 +108,7 @@ async function translateMain(text:string, arg:{from:string, to:string, host:stri
     let db = getDatabase()
     if(db.translatorType === 'llm'){
         const tr = arg.to || 'en'
-        return translateLLM(text, {to: tr})
+        return translateLLM(text, {to: tr, from: arg.from})
     }
     if(db.translatorType === 'deepl'){
         const body = {
@@ -197,12 +193,7 @@ async function translateMain(text:string, arg:{from:string, to:string, host:stri
     return result
 }
 
-export async function translateVox(text:string) {
-    const plug = await translatorPlugin(text, 'en', 'ja')
-    if(plug){
-        return plug.content
-    }
-    
+export async function translateVox(text:string) {    
     return jaTrans(text)
 }
 
@@ -245,7 +236,8 @@ export async function translateHTML(html: string, reverse:boolean, charArg:simpl
     }
     if(db.translatorType === 'llm'){
         const tr = db.translator || 'en'
-        return translateLLM(html, {to: tr, regenerate})
+        const from = db.translatorInputLanguage
+        return translateLLM(html, {to: tr, from: from, regenerate})
     }
     const dom = new DOMParser().parseFromString(html, 'text/html');
     console.log(html)
@@ -454,7 +446,7 @@ function needSuperChunkedTranslate(){
     return getDatabase().translatorType === 'deeplX'
 }
 
-async function translateLLM(text:string, arg:{to:string, regenerate?:boolean}):Promise<string>{
+async function translateLLM(text:string, arg:{to:string, from:string, regenerate?:boolean}):Promise<string>{
     if(!arg.regenerate){
         const cacheMatch = await LLMCacheStorage.getItem(text)
         if(cacheMatch){
@@ -472,7 +464,7 @@ async function translateLLM(text:string, arg:{to:string, regenerate?:boolean}):P
     const charIndex = get(selectedCharID)
     const currentChar = db.characters[charIndex]
     let translatorNote
-    if (currentChar.type === "character") {
+    if (currentChar?.type === "character") {
         translatorNote = currentChar.translatorNote ?? ""
     } else {
         translatorNote = ""
@@ -480,12 +472,12 @@ async function translateLLM(text:string, arg:{to:string, regenerate?:boolean}):P
 
     let formated:OpenAIChat[] = []
     let prompt = db.translatorPrompt || `You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation.`
-    let parsedPrompt = parseChatML(prompt.replaceAll('{{slot}}', arg.to).replaceAll('{{solt::content}}', text).replaceAll('{{slot::tnote}}', translatorNote))
+    let parsedPrompt = parseChatML(prompt.replaceAll('{{slot::from}}', arg.from).replaceAll('{{slot}}', arg.to).replaceAll('{{solt::content}}', text).replaceAll('{{slot::tnote}}', translatorNote))
     if(parsedPrompt){
         formated = parsedPrompt
     }
     else{
-        prompt = prompt.replaceAll('{{slot}}', arg.to).replaceAll('{{slot::tnote}}', translatorNote)
+        prompt = prompt.replaceAll('{{slot}}', arg.to).replaceAll('{{slot::tnote}}', translatorNote).replaceAll('{{slot::from}}', arg.from)
         formated = [
             {
                 'role': 'system',