diff --git a/src/lang/en.ts b/src/lang/en.ts index c6793efa..e8dcd718 100644 --- a/src/lang/en.ts +++ b/src/lang/en.ts @@ -836,4 +836,5 @@ export const languageEnglish = { home: "Home", showSavingIcon: "Show Saving Icon", pluginVersionWarn: "This is {{plugin_version}} version of the plugin. which is not compatible with this version of RisuAI. please update the plugin to {{required_version}} version.", + imageTranslation: "Image Translation", } \ No newline at end of file diff --git a/src/lib/Others/AlertComp.svelte b/src/lib/Others/AlertComp.svelte index 7ef80344..8d418612 100644 --- a/src/lib/Others/AlertComp.svelte +++ b/src/lib/Others/AlertComp.svelte @@ -472,6 +472,9 @@ {language.risuMDesc} {:else if $alertStore.submsg === 'preset'} {language.risupresetDesc} + {#if cardExportType2 === 'preset' && (DBState.db.botPresets[DBState.db.botPresetsId].image || DBState.db.botPresets[DBState.db.botPresetsId].regex?.length > 0)} + Use RisuRealm to share the preset. Preset with image or regexes cannot be exported for now. + {/if} {:else} {language.ccv3Desc} {#if cardExportType2 !== 'charx' && isCharacterHasAssets(DBState.db.characters[$selectedCharID])} diff --git a/src/lib/Playground/PlaygroundImageTrans.svelte b/src/lib/Playground/PlaygroundImageTrans.svelte index 99a8152e..a3bab9b9 100644 --- a/src/lib/Playground/PlaygroundImageTrans.svelte +++ b/src/lib/Playground/PlaygroundImageTrans.svelte @@ -2,9 +2,216 @@ import { language } from "src/lang"; import TextInput from "../UI/GUI/TextInput.svelte"; import TextAreaInput from "../UI/GUI/TextAreaInput.svelte"; + import Button from "../UI/GUI/Button.svelte"; + import { selectSingleFile } from "src/ts/util"; + import { requestChatData } from "src/ts/process/request"; + import { alertError } from "src/ts/alert"; let selLang = $state("en"); - let prompt = $state(""); + let prompt = $state('extract text chunk from the image, with all the positions and background color, and translate them to {{slot}} in a JSON format.Format of: \n\n [\n {\n "bg_hex_color": string\n "content": string\n "text_hex_color": string,\n "x_max": number,\n "x_min": number,\n "y_max": number,\n "y_min": number\n "translation": string,\n }\n]\n\n each properties is:\n - x_min, y_min, x_max, y_max: range of 0 (most left/top point of the image) to 1 (most bottom/right point of the image), it is the bounding boxes of the original text chunk.\n - bg_hex_color is the color of the background.\n - text_hex_color is the color of the text.\n - translation is the translated text.\n - content is the original text chunk.'); + let canvas: HTMLCanvasElement; + let ctx: CanvasRenderingContext2D; + let inputImage: HTMLImageElement; + let output = $state('') + let loading = $state(false); + let aspectRatio = 1; + + async function imageTranslate() { + if(loading){ + return; + } + loading = true; + try { + const file = await selectSingleFile(['png', 'jpg', 'jpeg','gif','webp','avif']); + if (!file){ + loading = false; + return; + }; + + if(!ctx){ + ctx = canvas.getContext('2d'); + } + const img = new Image(); + inputImage = img; + img.src = URL.createObjectURL(new Blob([file.data])); + await img.decode(); + aspectRatio = img.width / img.height; + canvas.width = img.width; + canvas.height = img.height; + + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.drawImage(img, 0, 0, canvas.width, canvas.height); + + + + const data = canvas.toDataURL('image/png'); + + const schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "type": "ARRAY", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "y_min": { + "type": "number" + }, + "x_min": { + "type": "number" + }, + "y_max": { + "type": "number" + }, + "x_max": { + "type": "number" + }, + "bg_hex_color": { + "type": "string" + }, + "text_hex_color": { + "type": "string" + }, + "content": { + "type": "string" + }, + "translation": { + "type": "string" + } + }, + "required": [ + "y_min", + "x_min", + "y_max", + "x_max", + "content", + "translation", + "bg_hex_color", + "text_hex_color" + ] + }, + } + + + const d = await requestChatData({ + formated: [{ + role: 'user', + content: prompt.replace('{{slot}}', selLang), + multimodals: [{ + type: 'image', + base64: data, + }], + }], + bias: {}, + schema: JSON.stringify(schema) + }, 'translate') + + if(d.type === 'streaming' || d.type === 'multiline'){ + loading = false; + return alertError('This model is not supported in the playground') + } + + if(d.type !== 'success'){ + alertError(d.result) + } + + output = d.result + output = JSON.stringify(JSON.parse(d.result), null, 2); + loading = false; + render() + } catch (error) { + alertError(JSON.stringify(error)) + } finally { + loading = false; + } + } + + + async function render() { + if(!inputImage){ + return + } + if(!ctx){ + ctx = canvas.getContext('2d'); + } + + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.drawImage(inputImage, 0, 0, canvas.width, canvas.height); + + const data = JSON.parse(output); + + for (const item of data) { + let [x_min, y_min, x_max, y_max] = [item.x_min, item.y_min, item.x_max, item.y_max]; + + if(x_min <= 1){ + x_min *= canvas.width; + y_min *= canvas.height; + x_max *= canvas.width; + y_max *= canvas.height; + } + + ctx.fillStyle = item.bg_hex_color; + ctx.fillRect(x_min, y_min, x_max - x_min, y_max - y_min); + // ctx.fillStyle = item.text_hex_color; + // ctx.fillText(item.translation, x_min, y_min); + + //make text wrap, and fit the text in the box + const text = item.translation; + const maxWidth = x_max - x_min; + const maxHeight = y_max - y_min; + const textSizes = [288, 216, 192, 144, 120, 108, 96, 84, 76, 72, 68, 64, 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 18, 16, 14, 12, 10]; + let lineHeight = 0; + + for(let i = 0; i < textSizes.length; i++){ + ctx.font = `${textSizes[i]}px Arial`; + lineHeight = textSizes[i] * 1.2; + const lines = text.split('\n'); + let totalHeight = 0; + let x = 0 + for (let n = 0; n < lines.length; n++) { + let testLine = lines[n]; + let metrics = ctx.measureText(testLine); + let testWidth = metrics.width; + x += testWidth; + if(testWidth > maxWidth){ + totalHeight = maxHeight + 1; + break + } + + if(x > maxWidth){ + totalHeight += lineHeight; + x = testWidth + } + } + console.log(x, maxWidth, totalHeight, maxHeight, textSizes[i]) + if(totalHeight < maxHeight){ + break; + } + } + let words = text.split(' '); + let line = ''; + let y = y_min + lineHeight; + for (let n = 0; n < words.length; n++) { + let testLine = line + words[n] + ' '; + let metrics = ctx.measureText(testLine); + let testWidth = metrics.width; + if (testWidth > maxWidth && n > 0) { + ctx.fillStyle = item.text_hex_color; + ctx.fillText(line, x_min, y); + line = words[n] + ' '; + y += lineHeight; + } else { + line = testLine; + } + } + ctx.fillStyle = item.text_hex_color; + ctx.fillText(line, x_min, y); + + } + + console.log('rendered') + + } @@ -13,3 +220,32 @@ {language.prompt} + + + +{#if output} + JSON + +{/if} + + + + \ No newline at end of file diff --git a/src/lib/Playground/PlaygroundMenu.svelte b/src/lib/Playground/PlaygroundMenu.svelte index 3d007539..829557bf 100644 --- a/src/lib/Playground/PlaygroundMenu.svelte +++ b/src/lib/Playground/PlaygroundMenu.svelte @@ -15,6 +15,7 @@ import ToolConvertion from "./ToolConvertion.svelte"; import { joinMultiuserRoom } from "src/ts/sync/multiuser"; import PlaygroundSubtitle from "./PlaygroundSubtitle.svelte"; + import PlaygroundImageTrans from "./PlaygroundImageTrans.svelte"; let easterEggTouch = $state(0) @@ -89,6 +90,11 @@ }}>

{language.subtitles}

+ +
+ + + +
{/if} \ No newline at end of file diff --git a/src/ts/globalApi.svelte.ts b/src/ts/globalApi.svelte.ts index 6ce26b3a..1e8718e9 100644 --- a/src/ts/globalApi.svelte.ts +++ b/src/ts/globalApi.svelte.ts @@ -547,10 +547,40 @@ export async function loadData() { else{ await forageStorage.Init() + LoadingStatusState.text = "Loading Local Save File..." + let gotStorage:Uint8Array = await forageStorage.getItem('database/database.bin') as unknown as Uint8Array + LoadingStatusState.text = "Decoding Local Save File..." + if(checkNullish(gotStorage)){ + gotStorage = encodeRisuSaveLegacy({}) + await forageStorage.setItem('database/database.bin', gotStorage) + } + try { + const decoded = await decodeRisuSave(gotStorage) + console.log(decoded) + setDatabase(decoded) + } catch (error) { + console.error(error) + const backups = await getDbBackups() + let backupLoaded = false + for(const backup of backups){ + try { + LoadingStatusState.text = `Reading Backup File ${backup}...` + const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`) as unknown as Uint8Array + setDatabase( + await decodeRisuSave(backupData) + ) + backupLoaded = true + } catch (error) {} + } + if(!backupLoaded){ + throw "Your save file is corrupted" + } + } + if(await forageStorage.checkAccountSync()){ LoadingStatusState.text = "Checking Account Sync..." let gotStorage:Uint8Array = await (forageStorage.realStorage as AccountStorage).getItem('database/database.bin', (v) => { - LoadingStatusState.text = `Loading Save File ${(v*100).toFixed(2)}%` + LoadingStatusState.text = `Loading Remote Save File ${(v*100).toFixed(2)}%` }) if(checkNullish(gotStorage)){ gotStorage = encodeRisuSaveLegacy({}) @@ -578,37 +608,8 @@ export async function loadData() { } } } - else{ - LoadingStatusState.text = "Loading Save File..." - let gotStorage:Uint8Array = await forageStorage.getItem('database/database.bin') as unknown as Uint8Array - LoadingStatusState.text = "Decoding Save File..." - if(checkNullish(gotStorage)){ - gotStorage = encodeRisuSaveLegacy({}) - await forageStorage.setItem('database/database.bin', gotStorage) - } - try { - const decoded = await decodeRisuSave(gotStorage) - console.log(decoded) - setDatabase(decoded) - } catch (error) { - console.error(error) - const backups = await getDbBackups() - let backupLoaded = false - for(const backup of backups){ - try { - LoadingStatusState.text = `Reading Backup File ${backup}...` - const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`) as unknown as Uint8Array - setDatabase( - await decodeRisuSave(backupData) - ) - backupLoaded = true - } catch (error) {} - } - if(!backupLoaded){ - throw "Your save file is corrupted" - } - } - } + LoadingStatusState.text = "Rechecking Account Sync..." + await forageStorage.checkAccountSync() LoadingStatusState.text = "Checking Drive Sync..." const isDriverMode = await checkDriverInit() if(isDriverMode){ diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index e039f7b1..900ed5e0 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -19,7 +19,7 @@ import {createParser} from 'eventsource-parser' import {Ollama} from 'ollama/dist/browser.mjs' import { applyChatTemplate } from "./templates/chatTemplate"; import { OobaParams } from "./prompt"; -import { extractJSON, getOpenAIJSONSchema } from "./templates/jsonSchema"; +import { extractJSON, getGeneralJSONSchema, getOpenAIJSONSchema } from "./templates/jsonSchema"; import { getModelInfo, LLMFlags, LLMFormat, type LLMModel } from "../model/modellist"; @@ -39,6 +39,8 @@ interface requestDataArgument{ continue?:boolean chatId?:string noMultiGen?:boolean + schema?:string + extractJson?:string } interface RequestDataArgumentExtended extends requestDataArgument{ @@ -357,6 +359,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:ModelMo targ.abortSignal = abortSignal targ.modelInfo = getModelInfo(targ.aiModel) targ.mode = model + targ.extractJson = arg.extractJson ?? db.extractJson if(targ.aiModel === 'reverse_proxy'){ targ.modelInfo.internalID = db.customProxyRequestModel targ.modelInfo.format = db.customAPIFormat @@ -694,10 +697,10 @@ async function requestOpenAI(arg:RequestDataArgumentExtended):Promise { - const extracted = extractJSON(v.message.content, db.extractJson) + const extracted = extractJSON(v.message.content, arg.extractJson) return ["char",extracted] }) @@ -999,10 +1002,10 @@ async function requestOpenAI(arg:RequestDataArgumentExtended):Promise 1){ const thought = rDatas.splice(rDatas.length-2, 1)[0] rDatas[rDatas.length-1] = `${thought}\n\n${rDatas.join('\n\n')}` @@ -1827,6 +1844,12 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise processDataItem(res.data) } + if(arg.extractJson && (db.jsonSchemaEnabled || arg.schema)){ + for(let i=0;i 1){ const thought = rDatas.splice(rDatas.length-2, 1)[0] @@ -2539,7 +2562,7 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise 0){ + for(const thought of data.thoughts){ + encoded += (await encode(thought)).length + 1 + } + } return encoded } async tokenizeChats(data:OpenAIChat[]){