diff --git a/src/lang/en.ts b/src/lang/en.ts
index e8dcd718..2145bb5d 100644
--- a/src/lang/en.ts
+++ b/src/lang/en.ts
@@ -837,4 +837,5 @@ export const languageEnglish = {
showSavingIcon: "Show Saving Icon",
pluginVersionWarn: "This is {{plugin_version}} version of the plugin. which is not compatible with this version of RisuAI. please update the plugin to {{required_version}} version.",
imageTranslation: "Image Translation",
+ banCharacterset: 'Auto Regenerate On Characterset'
}
\ No newline at end of file
diff --git a/src/lib/Playground/PlaygroundMenu.svelte b/src/lib/Playground/PlaygroundMenu.svelte
index 829557bf..2c7cc682 100644
--- a/src/lib/Playground/PlaygroundMenu.svelte
+++ b/src/lib/Playground/PlaygroundMenu.svelte
@@ -16,6 +16,7 @@
import { joinMultiuserRoom } from "src/ts/sync/multiuser";
import PlaygroundSubtitle from "./PlaygroundSubtitle.svelte";
import PlaygroundImageTrans from "./PlaygroundImageTrans.svelte";
+ import PlaygroundTranslation from "./PlaygroundTranslation.svelte";
let easterEggTouch = $state(0)
@@ -95,6 +96,11 @@
}}>
{language.imageTranslation}
+ {
+ PlaygroundStore.set(11)
+ }}>
+ {language.translator}
+
{
PlaygroundStore.set(101)
}}>
@@ -157,6 +163,9 @@
{#if $PlaygroundStore === 10}
{/if}
+ {#if $PlaygroundStore === 11}
+
+ {/if}
{#if $PlaygroundStore === 101}
{/if}
diff --git a/src/lib/Playground/PlaygroundSubtitle.svelte b/src/lib/Playground/PlaygroundSubtitle.svelte
index 432d3a40..e289aca2 100644
--- a/src/lib/Playground/PlaygroundSubtitle.svelte
+++ b/src/lib/Playground/PlaygroundSubtitle.svelte
@@ -10,7 +10,7 @@
import { selectFileByDom, selectSingleFile, sleep } from "src/ts/util";
import { alertError, alertSelect } from "src/ts/alert";
import { risuChatParser } from "src/ts/parser.svelte";
- import { AppendableBuffer, downloadFile, globalFetch } from "src/ts/globalApi.svelte";
+ import { AppendableBuffer, downloadFile, getLanguageCodes, globalFetch } from "src/ts/globalApi.svelte";
import SliderInput from "../UI/GUI/SliderInput.svelte";
import SelectInput from "../UI/GUI/SelectInput.svelte";
import OptionInput from "../UI/GUI/OptionInput.svelte";
@@ -31,40 +31,6 @@
let mode = $state('llm')
let sourceLang:string|null = $state(null)
- function getLanguageCodes(){
- let languageCodes:{
- code: string
- name: string
- }[] = []
-
- for(let i=0x41;i<=0x5A;i++){
- for(let j=0x41;j<=0x5A;j++){
- languageCodes.push({
- code: String.fromCharCode(i) + String.fromCharCode(j),
- name: ''
- })
- }
- }
-
- languageCodes = languageCodes.map(v => {
- return {
- code: v.code,
- name: new Intl.DisplayNames([
- DBState.db.language === 'cn' ? 'zh' : DBState.db.language
- ], {
- type: 'language',
- fallback: 'none'
- }).of(v.code)
- }
- }).filter((a) => {
- return a.name
- }).sort((a, b) => a.name.localeCompare(b.name))
-
- return languageCodes
- }
-
-
-
async function runLLMMode() {
outputText = 'Loading...\n\n'
diff --git a/src/lib/Playground/PlaygroundTranslation.svelte b/src/lib/Playground/PlaygroundTranslation.svelte
new file mode 100644
index 00000000..61d44731
--- /dev/null
+++ b/src/lib/Playground/PlaygroundTranslation.svelte
@@ -0,0 +1,62 @@
+
+
+
+
+
+{language.sourceLanguage}
+
+ {#each getLanguageCodes() as lang}
+ {lang.name}
+ {/each}
+
+
+
+{language.translatorLanguage}
+
+ {#each getLanguageCodes() as lang}
+ {lang.name}
+ {/each}
+
+
+
+ {
+ try {
+ if(loading){
+ return
+ }
+ loading = true
+ output = await runTranslator(r, false, sourceLang, outputLang)
+ loading = false
+ } catch (error) {
+ console.error(error)
+ loading = false
+ }
+}}>
+ {#if loading}
+ Loading...
+ {:else}
+ Translate
+ {/if}
+
+ {
+ LLMCacheStorage.clear()
+}}>
+ Clear Cache
+
\ No newline at end of file
diff --git a/src/lib/Setting/Pages/AdvancedSettings.svelte b/src/lib/Setting/Pages/AdvancedSettings.svelte
index 1cd0b6dd..5a4f2803 100644
--- a/src/lib/Setting/Pages/AdvancedSettings.svelte
+++ b/src/lib/Setting/Pages/AdvancedSettings.svelte
@@ -13,12 +13,53 @@
import { installPython } from "src/ts/process/models/local";
import { Capacitor } from "@capacitor/core";
import { capStorageInvestigation } from "src/ts/storage/mobileStorage";
+ import Arcodion from "src/lib/UI/Arcodion.svelte";
let estaStorage:{
key:string,
size:string,
}[] = $state([])
+ const characterSets = [
+ 'Latn',
+ 'Hani',
+ 'Arab',
+ 'Deva',
+ 'Cyrl',
+ 'Beng',
+ 'Hrkt',
+ 'Telu',
+ 'Hang',
+ 'Taml',
+ 'Thai',
+ 'Gujr',
+ 'Knda',
+ 'Ethi',
+ 'Khmr',
+ 'Grek',
+ 'Hebr',
+ ]
+
+ const characterSetsPreview = {
+ 'Latn': "ABC",
+ 'Hani': "汉漢",
+ 'Arab': "اعب",
+ 'Deva': "अआइ",
+ 'Cyrl': "АБВ",
+ 'Beng': "অআই",
+ 'Hrkt': "あア",
+ 'Telu': "అఆఇ",
+ 'Hang': "가나다",
+ 'Taml': "அஆஇ",
+ 'Thai': "กขค",
+ 'Gujr': "અઆઇ",
+ 'Knda': "ಅಆಇ",
+ 'Ethi': "ሀሁሂ",
+ 'Khmr': "កខគ",
+ 'Grek': "ΑΒΓ",
+ 'Hebr': "אבג",
+
+ }
{language.advancedSettings}
{language.advancedSettingsWarn}
@@ -168,6 +209,21 @@
{/if}
+
+
+ {#each characterSets as set}
+ {
+ if (DBState.db.banCharacterset.includes(set)) {
+ DBState.db.banCharacterset = DBState.db.banCharacterset.filter((item) => item !== set)
+ } else {
+ DBState.db.banCharacterset.push(set)
+ }
+ }}>
+ {new Intl.DisplayNames([navigator.language,'en'], { type: 'script' }).of(set)} ({characterSetsPreview[set]})
+
+ {/each}
+
+
{
diff --git a/src/ts/characterCards.ts b/src/ts/characterCards.ts
index ff1e2892..e392a281 100644
--- a/src/ts/characterCards.ts
+++ b/src/ts/characterCards.ts
@@ -7,7 +7,7 @@ import { v4 as uuidv4, v4 } from 'uuid';
import { characterFormatUpdate } from "./characters"
import { AppendableBuffer, BlankWriter, checkCharOrder, downloadFile, isNodeServer, isTauri, loadAsset, LocalWriter, openURL, readImage, saveAsset, VirtualWriter } from "./globalApi.svelte"
import { SettingsMenuIndex, ShowRealmFrameStore, selectedCharID, settingsOpen } from "./stores.svelte"
-import { convertImage, hasher } from "./parser.svelte"
+import { checkImageType, convertImage, hasher } from "./parser.svelte"
import { CCardLib, type CharacterCardV3, type LorebookEntry } from '@risuai/ccardlib'
import { reencodeImage } from "./process/files/inlays"
import { PngChunk } from "./pngChunk"
@@ -1316,6 +1316,21 @@ export async function exportCharacterCard(char:character, type:'png'|'json'|'cha
path = `assets/${type}/${itype}/${name}.${card.data.assets[i].ext}`
}
card.data.assets[i].uri = 'embeded://' + path
+ const imageType = checkImageType(rData)
+ if(imageType === 'PNG' && writer instanceof CharXWriter){
+ const metadatas:Record = {}
+ const gen = PngChunk.readGenerator(rData)
+ for await (const chunk of gen){
+ if(!chunk || chunk instanceof AppendableBuffer){
+ continue
+ }
+ metadatas[chunk.key] = chunk.value
+ }
+ if(Object.keys(metadatas).length > 0){
+ const metaPath = `x_meta/${name}.json`
+ await writer.write(metaPath, Buffer.from(JSON.stringify(metadatas, null, 4)), 6)
+ }
+ }
await writer.write(path, Buffer.from(await convertImage(rData)))
}
}
diff --git a/src/ts/globalApi.svelte.ts b/src/ts/globalApi.svelte.ts
index 1e8718e9..a82ba3db 100644
--- a/src/ts/globalApi.svelte.ts
+++ b/src/ts/globalApi.svelte.ts
@@ -2261,4 +2261,36 @@ export class PerformanceDebugger{
this.kv[key].push(...other.kv[key])
}
}
+}
+
+export function getLanguageCodes(){
+ let languageCodes:{
+ code: string
+ name: string
+ }[] = []
+
+ for(let i=0x41;i<=0x5A;i++){
+ for(let j=0x41;j<=0x5A;j++){
+ languageCodes.push({
+ code: String.fromCharCode(i) + String.fromCharCode(j),
+ name: ''
+ })
+ }
+ }
+
+ languageCodes = languageCodes.map(v => {
+ return {
+ code: v.code.toLocaleLowerCase(),
+ name: new Intl.DisplayNames([
+ DBState.db.language === 'cn' ? 'zh' : DBState.db.language
+ ], {
+ type: 'language',
+ fallback: 'none'
+ }).of(v.code)
+ }
+ }).filter((a) => {
+ return a.name
+ }).sort((a, b) => a.name.localeCompare(b.name))
+
+ return languageCodes
}
\ No newline at end of file
diff --git a/src/ts/pngChunk.ts b/src/ts/pngChunk.ts
index 57dca89b..e74b4bb3 100644
--- a/src/ts/pngChunk.ts
+++ b/src/ts/pngChunk.ts
@@ -26,7 +26,21 @@ class StreamChunkWriter{
break
}
if(typeString === 'tEXt'){
- pos += 12 + len
+ const endPos = 12 + len + pos
+ //get key
+ let key=''
+ while(data[pos+8] !== 0){
+ key += String.fromCharCode(data[pos+8])
+ pos++
+ if(pos === endPos || key.length > 6){
+ break
+ }
+ }
+
+ if(key !== 'ccv3' && key !== 'chara'){
+ await this.pushData(data.slice(pos,endPos))
+ }
+ pos = endPos
}
else{
await this.pushData(data.slice(pos,pos+12+len))
diff --git a/src/ts/process/processzip.ts b/src/ts/process/processzip.ts
index 14160f44..9641b8d0 100644
--- a/src/ts/process/processzip.ts
+++ b/src/ts/process/processzip.ts
@@ -41,7 +41,7 @@ export class CharXWriter{
//do nothing, just to make compatible with other writer
}
- async write(key:string,data:Uint8Array|string){
+ async write(key:string,data:Uint8Array|string, level?:0|1|2|3|4|5|6|7|8|9){
console.log('write',key)
let dat:Uint8Array
if(typeof data === 'string'){
@@ -52,7 +52,7 @@ export class CharXWriter{
}
this.writeEnd = false
const file = new fflate.ZipDeflate(key, {
- level: 0
+ level: level ?? 0
});
await this.zip.add(file)
await file.push(dat, true)
@@ -102,6 +102,9 @@ export class CharXReader{
else if(file.name === 'module.risum'){
this.moduleData = assetData
}
+ else if(file.name.endsWith('.json')){
+ //do nothing
+ }
else{
this.assetPromises.push((async () => {
const assetId = await saveAsset(assetData)
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 23f9421f..e68647c7 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -226,6 +226,19 @@ export async function requestChatData(arg:requestDataArgument, model:ModelModeEx
}
}
+ if(da.type === 'success' && db.banCharacterset?.length > 0){
+ for(const set of db.banCharacterset){
+ const checkRegex = new RegExp(`\\p{Script=${set}}`, 'gu')
+
+ if(checkRegex.test(da.result)){
+ return {
+ type: 'fail',
+ result: 'Banned character found'
+ }
+ }
+ }
+ }
+
if(da.type !== 'fail' || da.noRetry){
return da
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index 18a0115b..67c1f8a8 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -467,6 +467,7 @@ export function setDatabase(data:Database){
data.enableCustomFlags ??= false
data.assetMaxDifference ??= 4
data.showSavingIcon ??= false
+ data.banCharacterset ??= []
changeLanguage(data.language)
setDatabaseLite(data)
}
@@ -866,6 +867,7 @@ export interface Database{
pluginV2: RisuPlugin[]
showSavingIcon:boolean
presetRegex: customscript[]
+ banCharacterset:string[]
}
interface SeparateParameters{
diff --git a/src/ts/translator/translator.ts b/src/ts/translator/translator.ts
index 587c4582..542aae0b 100644
--- a/src/ts/translator/translator.ts
+++ b/src/ts/translator/translator.ts
@@ -17,7 +17,7 @@ let cache={
trans: ['']
}
-const LLMCacheStorage = localforage.createInstance({
+export const LLMCacheStorage = localforage.createInstance({
name: "LLMTranslateCache"
})
@@ -25,10 +25,6 @@ let waitTrans = 0
export async function translate(text:string, reverse:boolean) {
let db = getDatabase()
- const plug = await translatorPlugin(text, reverse ? db.translator: 'en', reverse ? 'en' : db.translator)
- if(plug){
- return plug.content
- }
if(!reverse){
const ind = cache.origin.indexOf(text)
if(ind !== -1){
@@ -112,7 +108,7 @@ async function translateMain(text:string, arg:{from:string, to:string, host:stri
let db = getDatabase()
if(db.translatorType === 'llm'){
const tr = arg.to || 'en'
- return translateLLM(text, {to: tr})
+ return translateLLM(text, {to: tr, from: arg.from})
}
if(db.translatorType === 'deepl'){
const body = {
@@ -197,12 +193,7 @@ async function translateMain(text:string, arg:{from:string, to:string, host:stri
return result
}
-export async function translateVox(text:string) {
- const plug = await translatorPlugin(text, 'en', 'ja')
- if(plug){
- return plug.content
- }
-
+export async function translateVox(text:string) {
return jaTrans(text)
}
@@ -245,7 +236,8 @@ export async function translateHTML(html: string, reverse:boolean, charArg:simpl
}
if(db.translatorType === 'llm'){
const tr = db.translator || 'en'
- return translateLLM(html, {to: tr, regenerate})
+ const from = db.translatorInputLanguage
+ return translateLLM(html, {to: tr, from: from, regenerate})
}
const dom = new DOMParser().parseFromString(html, 'text/html');
console.log(html)
@@ -454,7 +446,7 @@ function needSuperChunkedTranslate(){
return getDatabase().translatorType === 'deeplX'
}
-async function translateLLM(text:string, arg:{to:string, regenerate?:boolean}):Promise{
+async function translateLLM(text:string, arg:{to:string, from:string, regenerate?:boolean}):Promise{
if(!arg.regenerate){
const cacheMatch = await LLMCacheStorage.getItem(text)
if(cacheMatch){
@@ -472,7 +464,7 @@ async function translateLLM(text:string, arg:{to:string, regenerate?:boolean}):P
const charIndex = get(selectedCharID)
const currentChar = db.characters[charIndex]
let translatorNote
- if (currentChar.type === "character") {
+ if (currentChar?.type === "character") {
translatorNote = currentChar.translatorNote ?? ""
} else {
translatorNote = ""
@@ -480,12 +472,12 @@ async function translateLLM(text:string, arg:{to:string, regenerate?:boolean}):P
let formated:OpenAIChat[] = []
let prompt = db.translatorPrompt || `You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation.`
- let parsedPrompt = parseChatML(prompt.replaceAll('{{slot}}', arg.to).replaceAll('{{solt::content}}', text).replaceAll('{{slot::tnote}}', translatorNote))
+ let parsedPrompt = parseChatML(prompt.replaceAll('{{slot::from}}', arg.from).replaceAll('{{slot}}', arg.to).replaceAll('{{solt::content}}', text).replaceAll('{{slot::tnote}}', translatorNote))
if(parsedPrompt){
formated = parsedPrompt
}
else{
- prompt = prompt.replaceAll('{{slot}}', arg.to).replaceAll('{{slot::tnote}}', translatorNote)
+ prompt = prompt.replaceAll('{{slot}}', arg.to).replaceAll('{{slot::tnote}}', translatorNote).replaceAll('{{slot::from}}', arg.from)
formated = [
{
'role': 'system',