Upgrade translator playground

src/lib/Playground/PlaygroundTranslation.svelte

@@ -7,6 +7,8 @@
     import SelectInput from "../UI/GUI/SelectInput.svelte";
     import { getLanguageCodes } from "src/ts/globalApi.svelte";
     import OptionInput from "../UI/GUI/OptionInput.svelte";
+    import CheckInput from "../UI/GUI/CheckInput.svelte";
+    import { tokenize } from "src/ts/tokenizer";
 
 
     const userPreferedLang = navigator.language.split('-')[0]
@@ -16,6 +18,9 @@
     let output = $state('')
     let outputLang = $state(userPreferedLang)
     let loading = $state(false)
+    let bulk = $state(false)
+    let keepContext = $state(false)
+    let bulkProgressText = $state('')
 </script>
 
 
@@ -36,7 +41,99 @@
 </SelectInput>
 <TextAreaInput value={output} />
 
+<CheckInput bind:check={bulk}>
+    Bulk
+</CheckInput>
+{#if bulk}
+    <CheckInput bind:check={keepContext}>
+        Keep Context
+    </CheckInput>
+{/if}
+
 <Button className="mt-4" onclick={async () => {
+    bulkProgressText = ''
+    if(bulk){
+
+        const format = () => {
+            if(jsonMode){
+                const d = JSON.parse(r)
+                for(let i = 0; i < d.length; i++){
+                    if(translatedChunks[i]){
+                        d[i].text = translatedChunks[i]
+                    }
+                }
+                output = JSON.stringify(d, null, 2)
+                return
+            }
+
+            output = translatedChunks.join('\n\n')
+            console.log(output)
+        }
+        let preChunks = []
+        let prContexts:string[] = []
+        let translatedChunks: string[] = []
+        let jsonMode = false
+        if(loading){
+            return
+        }
+        loading = true
+        try {
+            const d = JSON.parse(r.trim())
+            if(Array.isArray(d)){
+                preChunks = d.map((x: { text: string }) => (x?.text ?? ""))
+            }
+            jsonMode = true
+        } catch (error) {
+            preChunks = r.split('\n\n')
+            jsonMode = false
+        }
+
+        console.log(preChunks, jsonMode)
+        let pvc = 'Previous Content is the content that was translated before the current content. This is used to keep the context of the translation. do not retranslate or return it.'
+
+        for(let i = 0; i < preChunks.length; i++){
+            try {
+                if(preChunks[i].trim().length === 0){
+                    translatedChunks.push(preChunks[i])
+                    continue
+                }
+
+                bulkProgressText = `(${i + 1} of ${preChunks.length})`
+
+                if(prContexts.length > 10){
+                    prContexts.shift()
+                }
+
+
+                const prContext = prContexts.length > 0 ? prContexts.join('\n\n') : ''
+
+                if(prContext){
+                    bulkProgressText += ` (previous ${await tokenize(prContext)} tokens)`
+                }
+
+                const translatedChunk = await runTranslator(preChunks[i], false, sourceLang, outputLang, {
+                    translatorNote: prContext ? `<Previous Content>${prContext.trim()}</Previous Content>\n${pvc}` : ""
+                })
+                if(keepContext){
+                    prContexts.push(`<Original>${preChunks[i]}</Original><Translated>${translatedChunk}</Translated>`)
+                }
+                translatedChunks.push(translatedChunk)
+            } catch (error) {
+                console.error(error)
+                translatedChunks.push(preChunks[i])
+            }
+
+            try {
+                format()
+            } catch (error) {
+                
+            }
+        }
+
+        format()
+        loading = false
+        return
+    }
     try {
         if(loading){
             return
@@ -50,7 +147,7 @@
     }
 }}>
     {#if loading}
-        Loading...
+        Loading... {bulkProgressText}
     {:else}
         Translate
     {/if}

src/ts/characterCards.ts

@@ -1371,6 +1371,7 @@ export async function exportCharacterCard(char:character, type:'png'|'json'|'cha
                         }
                         card.data.assets[i].uri = 'embeded://' + path
                         const imageType = checkImageType(rData)
+                        const metaPath = `x_meta/${name}.json`
                         if(imageType === 'PNG' && writer instanceof CharXWriter){
                             const metadatas:Record<string,string> = {}
                             const gen = PngChunk.readGenerator(rData)
@@ -1380,10 +1381,20 @@ export async function exportCharacterCard(char:character, type:'png'|'json'|'cha
                                 }
                                 metadatas[chunk.key] = chunk.value
                             }
+                            console.log(metadatas)
                             if(Object.keys(metadatas).length > 0){
-                                const metaPath = `x_meta/${name}.json`
                                 await writer.write(metaPath, Buffer.from(JSON.stringify(metadatas, null, 4)), 6)
                             }
+                            else{
+                                await writer.write(metaPath, Buffer.from(JSON.stringify({
+                                    'type': imageType
+                                }), 'utf-8'), 6)
+                            }
+                        }
+                        else{
+                            await writer.write(metaPath, Buffer.from(JSON.stringify({
+                                'type': imageType
+                            }), 'utf-8'), 6)
                         }
                         await writer.write(path, Buffer.from(await convertImage(rData)))
                     }

src/ts/process/request.ts

@@ -247,7 +247,7 @@ export async function requestChatData(arg:requestDataArgument, model:ModelModeEx
         
         try{
             const currentChar = getCurrentCharacter()
-            if(currentChar.type !== 'group'){
+            if(currentChar?.type !== 'group'){
                 const perf = performance.now()
                 const d = await runTrigger(currentChar, 'request', {
                     chat: getCurrentChat(),

src/ts/translator/translator.ts

@@ -41,7 +41,7 @@ export async function translate(text:string, reverse:boolean) {
     return runTranslator(text, reverse, db.translator,db.aiModel.startsWith('novellist') ? 'ja' : 'en')
 }
 
-export async function runTranslator(text:string, reverse:boolean, from:string,target:string) {
+export async function runTranslator(text:string, reverse:boolean, from:string,target:string, exarg?:{translatorNote?:string}) {
     const arg = {
 
         from: reverse ? from : target,
@@ -50,6 +50,7 @@ export async function runTranslator(text:string, reverse:boolean, from:string,ta
 
         host: 'translate.googleapis.com',
 
+        translatorNote: exarg?.translatorNote
     }
     const texts = text.split('\n')
     let chunks:[string,boolean][] = [['', true]]
@@ -104,11 +105,11 @@ export async function runTranslator(text:string, reverse:boolean, from:string,ta
 
 }
 
-async function translateMain(text:string, arg:{from:string, to:string, host:string}){
+async function translateMain(text:string, arg:{from:string, to:string, host:string, translatorNote?:string}){
     let db = getDatabase()
     if(db.translatorType === 'llm'){
         const tr = arg.to || 'en'
-        return translateLLM(text, {to: tr, from: arg.from})
+        return translateLLM(text, {to: tr, from: arg.from, translatorNote: arg.translatorNote})
     }
     if(db.translatorType === 'deepl'){
         const body = {
@@ -473,7 +474,7 @@ function needSuperChunkedTranslate(){
     return getDatabase().translatorType === 'deeplX'
 }
 
-async function translateLLM(text:string, arg:{to:string, from:string, regenerate?:boolean}):Promise<string>{
+async function translateLLM(text:string, arg:{to:string, from:string, regenerate?:boolean,translatorNote?:string}):Promise<string>{
     if(!arg.regenerate){
         const cacheMatch = await LLMCacheStorage.getItem(text)
         if(cacheMatch){
@@ -490,12 +491,17 @@ async function translateLLM(text:string, arg:{to:string, from:string, regenerate
     const db = getDatabase()
     const charIndex = get(selectedCharID)
     const currentChar = db.characters[charIndex]
-    let translatorNote
-    if (currentChar?.type === "character") {
+    let translatorNote = ""
+    console.log(arg.translatorNote)
+    if(arg.translatorNote){
+        translatorNote = arg.translatorNote
+    }
+    else if (currentChar?.type === "character") {
         translatorNote = currentChar.translatorNote ?? ""
     } else {
         translatorNote = ""
     }
+    console.log(translatorNote)
 
     let formated:OpenAIChat[] = []
     let prompt = db.translatorPrompt || `You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation.`