[feat] added additional assets

package.json

@@ -17,6 +17,7 @@
     "@msgpack/msgpack": "3.0.0-beta2",
     "@tauri-apps/api": "1.2.0",
     "buffer": "^6.0.3",
+    "core-js": "^3.30.2",
     "dompurify": "^3.0.1",
     "exifr": "^7.1.3",
     "gpt-3-encoder": "^1.1.4",

pnpm-lock.yaml

@@ -20,6 +20,7 @@ specifiers:
   '@types/wicg-file-system-access': ^2020.9.6
   autoprefixer: ^10.4.14
   buffer: ^6.0.3
+  core-js: ^3.30.2
   dompurify: ^3.0.1
   exifr: ^7.1.3
   gpt-3-encoder: ^1.1.4
@@ -55,6 +56,7 @@ dependencies:
   '@msgpack/msgpack': 3.0.0-beta2
   '@tauri-apps/api': 1.2.0
   buffer: 6.0.3
+  core-js: 3.30.2
   dompurify: 3.0.1
   exifr: 7.1.3
   gpt-3-encoder: 1.1.4
@@ -852,6 +854,11 @@ packages:
     resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==}
     dev: true
 
+  /core-js/3.30.2:
+    resolution: {integrity: sha512-uBJiDmwqsbJCWHAwjrx3cvjbMXP7xD72Dmsn5LOJpiRmE3WbBbN5rCqQ2Qh6Ek6/eOrjlWngEynBWo4VxerQhg==}
+    requiresBuild: true
+    dev: false
+
   /crc-32/0.3.0:
     resolution: {integrity: sha512-kucVIjOmMc1f0tv53BJ/5WIX+MGLcKuoBhnGqQrgKJNqLByb/sVMWfW/Aw6hw0jgcqjJ2pi9E5y32zOIpaUlsA==}
     engines: {node: '>=0.8'}

src/lang/en.ts

@@ -71,7 +71,8 @@ export const languageEnglish = {
         personality: "A brief description about character's personality. \n\n**It is not recommended to use this option. Describe it in character description instead.**",
         scenario: "A brief description about character's scenario. \n\n**It is not recommended to use this option. Describe it in character description instead.**",
         utilityBot: "When activated, it ignores main prompt. \n\n**It is not recommended to use this option. Modifiy system prompt instead.**",
-        loreSelective: "If Selective mode is toggled, both Activation Key and Secondary key should have a match to activate the lore."
+        loreSelective: "If Selective mode is toggled, both Activation Key and Secondary key should have a match to activate the lore.",
+        additionalAssets: "Additional assets to display in your chat. \n\n - use `{{raw::<asset name>}}` to use as path.\n - use `{{img::<asset name>}}` to use as image"
     },
     setup: {
         chooseProvider: "Choose AI Provider",
@@ -251,5 +252,7 @@ export const languageEnglish = {
     streaming: "Streaming",
     chatBot:'Chat Bot',
     otherBots:'Other Bots',
-    user:"User"
+    user:"User",
+    additionalAssets:"Additional Assets",
+    editDisplay: "Modify Display"
 }

src/lang/ko.ts

@@ -237,5 +237,7 @@ export const languageKorean = {
     streaming: "스트리밍",
     chatBot:'채팅 봇',
     otherBots:'기타 봇',
-    user:"유저"
+    user:"유저",
+    editDisplay: "디스플레이 수정"
+
 }

src/lib/ChatScreens/Chat.svelte

@@ -4,7 +4,7 @@
     import AutoresizeArea from "./AutoresizeArea.svelte";
     import { alertConfirm } from "../../ts/alert";
     import { language } from "../../lang";
-    import { DataBase } from "../../ts/database";
+    import { DataBase, type character, type groupChat } from "../../ts/database";
     import { selectedCharID } from "../../ts/stores";
     import { translate } from "../../ts/translator/translator";
     import { replacePlaceholders } from "../../ts/util";
@@ -16,6 +16,7 @@
     export let rerollIcon = false
     export let onReroll = () => {}
     export let unReroll = () => {}
+    export let character:character|groupChat|null = null
     let translating = false
     let editMode = false
     export let altGreeting = false
@@ -127,10 +128,12 @@
             {#if editMode}
                 <AutoresizeArea bind:value={message} />
             {:else}
+                {#await ParseMarkdown(msgDisplay, character) then md} 
                     <span class="text chat chattext prose prose-invert"
                         style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
                         style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
-                >{@html ParseMarkdown(msgDisplay)}</span>
+                    >{@html md}</span>
+                {/await}
             {/if}
         </span>
 

src/lib/ChatScreens/CreatorQuote.svelte

@@ -6,7 +6,9 @@
             </button>
         </h1>
         <div class="ml-2 max-w-full break-words text chat chattext prose prose-invert">
-            {@html ParseMarkdown(quote)}
+            {#await ParseMarkdown(quote) then md} 
+                {@html md}
+            {/await}
         </div>
     </div>
 </div>

src/lib/ChatScreens/DefaultChatScreen.svelte

@@ -247,6 +247,7 @@
                             onReroll={reroll}
                             unReroll={unReroll}
                             isLastMemory={$DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].lastMemory === (chat.chatId ?? 'none') && $DataBase.showMemoryLimit}
+                            character={$DataBase.characters[$selectedCharID]}
                         />
                     {:else}
                         <Chat
@@ -258,10 +259,12 @@
                             unReroll={unReroll}
                             img={getCharImage(findCharacterbyId(chat.saying).image, 'css')}
                             isLastMemory={$DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].lastMemory === (chat.chatId ?? 'none') && $DataBase.showMemoryLimit}
+                            character={findCharacterbyId(chat.saying)}
                         />
                     {/if}
                 {:else}
                     <Chat
+                        character={$DataBase.characters[$selectedCharID]}
                         idx={chat.index}
                         name={$DataBase.username} 
                         message={chat.data}
@@ -273,6 +276,7 @@
             {#if $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message.length <= loadPages}
                 {#if $DataBase.characters[$selectedCharID].type !== 'group'}
                     <Chat
+                        character={$DataBase.characters[$selectedCharID]}
                         name={$DataBase.characters[$selectedCharID].name}
                         message={$DataBase.characters[$selectedCharID].firstMsgIndex === -1 ? $DataBase.characters[$selectedCharID].firstMessage :
                             $DataBase.characters[$selectedCharID].alternateGreetings[$DataBase.characters[$selectedCharID].firstMsgIndex]}

src/lib/Others/AlertComp.svelte

@@ -33,7 +33,11 @@
                 <h2 class="text-green-700 mt-0 mb-2 w-40 max-w-full">Input</h2>
             {/if}
             {#if $alertStore.type === 'markdown'}
-                <span class="text-gray-300 chattext prose prose-invert chattext2">{@html ParseMarkdown($alertStore.msg)}</span>
+                <span class="text-gray-300 chattext prose prose-invert chattext2">
+                    {#await ParseMarkdown($alertStore.msg) then msg}
+                        {@html msg}                        
+                    {/await}
+                </span>
             {:else if $alertStore.type !== 'select'}
                 <span class="text-gray-300">{$alertStore.msg}</span>
             {/if}

src/lib/SideBars/CharConfig.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
     import { language } from "../../lang";
     import { tokenize } from "../../ts/tokenizer";
-    import { DataBase, type Database, type character, type groupChat } from "../../ts/database";
+    import { DataBase, saveImage as saveAsset, type Database, type character, type groupChat } from "../../ts/database";
     import { selectedCharID } from "../../ts/stores";
     import { PlusIcon, SmileIcon, TrashIcon, UserIcon, ActivityIcon, BookIcon, LoaderIcon, User, DnaIcon, CurlyBracesIcon, Volume2Icon } from 'lucide-svelte'
     import Check from "../Others/Check.svelte";
@@ -9,7 +9,7 @@
     import LoreBook from "./LoreBookSetting.svelte";
     import { alertConfirm, alertError, alertSelectChar } from "../../ts/alert";
     import BarIcon from "./BarIcon.svelte";
-    import { findCharacterbyId } from "../../ts/util";
+    import { findCharacterbyId, selectMultipleFile } from "../../ts/util";
     import { onDestroy } from "svelte";
     import {isEqual, cloneDeep} from 'lodash'
     import Help from "../Others/Help.svelte";
@@ -578,6 +578,59 @@
             {/each}
         </table>
       
+        <span class="text-neutral-200 mt-2">{language.additionalAssets} <Help key="additionalAssets" /></span>
+        <table class="contain w-full max-w-full tabler mt-2">
+            <tr>
+                <th class="font-medium">{language.value}</th>
+                <th class="font-medium cursor-pointer w-10">
+                    <button class="hover:text-green-500" on:click={async () => {
+                        if(currentChar.type === 'character'){
+                            const da = await selectMultipleFile(['png', 'webp', 'mp4', 'mp3'])
+                            currentChar.data.additionalAssets = currentChar.data.additionalAssets ?? []
+                            if(!da){
+                                return
+                            }
+                            for(const f of da){
+                                console.log(f)
+                                const img = f.data
+                                const imgp = await saveAsset(img)
+                                const name = f.name
+                                currentChar.data.additionalAssets.push([name, imgp])
+                                currentChar.data.additionalAssets = currentChar.data.additionalAssets
+                            }
+                        }
+                    }}>
+                        <PlusIcon />
+                    </button>
+                </th>
+            </tr>
+            {#if (!currentChar.data.additionalAssets) || currentChar.data.additionalAssets.length === 0}
+                <tr>
+                    <div class="text-gray-500"> No Assets</div>
+                </tr>
+            {:else}
+                {#each currentChar.data.additionalAssets as assets, i}
+                    <tr>
+                        <td class="font-medium truncate">
+                            <input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected w-full resize-none" bind:value={currentChar.data.additionalAssets[i][0]} placeholder="..." />
+                        </td>
+                        <th class="font-medium cursor-pointer w-10">
+                            <button class="hover:text-green-500" on:click={() => {
+                                if(currentChar.type === 'character'){
+                                    currentChar.data.firstMsgIndex = -1
+                                    let additionalAssets = currentChar.data.additionalAssets
+                                    additionalAssets.splice(i, 1)
+                                    currentChar.data.additionalAssets = additionalAssets
+                                }
+                            }}>
+                                <TrashIcon />
+                            </button>
+                        </th>
+                    </tr>
+                {/each}
+            {/if}
+        </table>
+
         {#if $DataBase.showUnrecommended || currentChar.data.utilityBot}
             <div class="flex items-center mt-4">
                 <Check bind:check={currentChar.data.utilityBot}/>

src/lib/SideBars/RegexData.svelte

@@ -34,6 +34,7 @@
                 <option value="editinput">{language.editInput}</option>
                 <option value="editoutput">{language.editOutput}</option>
                 <option value="editprocess">{language.editProcess}</option>
+                <option value="editdisplay">{language.editDisplay}</option>
             </select>
             <span class="text-neutral-200 mt-6">IN:</span>
             <input class="text-neutral-200 p-2 bg-transparent input-text focus:bg-selected text-sm" bind:value={value.in}>

src/main.ts

@@ -1,4 +1,5 @@
 import "./styles.css";
+import "core-js/actual"
 import App from "./App.svelte";
 import { loadData } from "./ts/globalApi";
 import { ReadableStream, WritableStream, TransformStream } from "web-streams-polyfill/ponyfill/es2018";

src/ts/characterCards.ts

@@ -1,6 +1,6 @@
 import { get } from "svelte/store"
 import { alertConfirm, alertError, alertNormal, alertSelect, alertStore } from "./alert"
-import { DataBase, defaultSdDataFunc, type character, saveImage, setDatabase, type customscript, type loreSettings, type loreBook } from "./database"
+import { DataBase, defaultSdDataFunc, type character, setDatabase, type customscript, type loreSettings, type loreBook } from "./database"
 import { checkNullish, selectSingleFile, sleep } from "./util"
 import { language } from "src/lang"
 import { encode as encodeMsgpack, decode as decodeMsgpack } from "@msgpack/msgpack";
@@ -8,7 +8,7 @@ import { v4 as uuidv4 } from 'uuid';
 import exifr from 'exifr'
 import { PngMetadata } from "./exif"
 import { characterFormatUpdate } from "./characters"
-import { downloadFile, readImage } from "./globalApi"
+import { downloadFile, readImage, saveAsset } from "./globalApi"
 import { cloneDeep } from "lodash"
 
 
@@ -69,7 +69,7 @@ export async function importCharacter() {
                         msg: `Loading... (Getting Emotions ${i} / ${char.emotionImages.length})`
                     })
                     await sleep(10)
-                    const imgp = await saveImage(char.emotionImages[i][1] as any)
+                    const imgp = await saveAsset(char.emotionImages[i][1] as any)
                     char.emotionImages[i][1] = imgp
                 }
             }
@@ -85,7 +85,7 @@ export async function importCharacter() {
             }
 
             char.chatPage = 0
-            char.image = await saveImage(PngMetadata.filter(img))
+            char.image = await saveAsset(PngMetadata.filter(img))
             db.characters.push(characterFormatUpdate(char))
             char.chaId = uuidv4()
             setDatabase(db)
@@ -94,7 +94,7 @@ export async function importCharacter() {
         }
         else if(readed.chara){
             const charaData:OldTavernChar = JSON.parse(Buffer.from(readed.chara, 'base64').toString('utf-8'))
-            const imgp = await saveImage(PngMetadata.filter(img))
+            const imgp = await saveAsset(PngMetadata.filter(img))
             let db = get(DataBase)
             db.characters.push(convertOldTavernAndJSON(charaData, imgp))
             DataBase.set(db)
@@ -140,7 +140,7 @@ export async function characterHubImport() {
                 }
             }
             {
-                const imgp = await saveImage(PngMetadata.filter(img))
+                const imgp = await saveAsset(PngMetadata.filter(img))
                 let db = get(DataBase)
                 const charaData:OldTavernChar = JSON.parse(Buffer.from(readed.chara, 'base64').toString('utf-8'))    
                 db.characters.push(convertOldTavernAndJSON(charaData, imgp))
@@ -300,7 +300,7 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array):Promise<boole
     }
 
     const data = card.data
-    const im = img ? await saveImage(PngMetadata.filter(img)) : undefined
+    const im = img ? await saveAsset(PngMetadata.filter(img)) : undefined
     let db = get(DataBase)
 
     const risuext = cloneDeep(data.extensions.risuai)
@@ -310,6 +310,7 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array):Promise<boole
     let customScripts:customscript[] = []
     let utilityBot = false
     let sdData = defaultSdDataFunc()
+    let extAssets:[string,string][] = []
 
     if(risuext){
         if(risuext.emotions){
@@ -319,10 +320,21 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array):Promise<boole
                     msg: `Loading... (Getting Emotions ${i} / ${risuext.emotions.length})`
                 })
                 await sleep(10)
-                const imgp = await saveImage(Buffer.from(risuext.emotions[i][1], 'base64'))
+                const imgp = await saveAsset(Buffer.from(risuext.emotions[i][1], 'base64'))
                 emotions.push([risuext.emotions[i][0],imgp])
             }
         }
+        if(risuext.additionalAssets){
+            for(let i=0;i<risuext.additionalAssets.length;i++){
+                alertStore.set({
+                    type: 'wait',
+                    msg: `Loading... (Getting Assets ${i} / ${risuext.additionalAssets.length})`
+                })
+                await sleep(10)
+                const imgp = await saveAsset(Buffer.from(risuext.additionalAssets[i][1], 'base64'))
+                extAssets.push([risuext.additionalAssets[i][0],imgp])
+            }
+        }
         bias = risuext.bias ?? bias
         viewScreen = risuext.viewScreen ?? viewScreen
         customScripts = risuext.customScripts ?? customScripts
@@ -403,7 +415,8 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array):Promise<boole
             tag: data.tags,
             creator: data.creator,
             character_version: data.character_version
-        }
+        },
+        additionalAssets: extAssets
     }
 
     db.characters.push(char)
@@ -467,7 +480,8 @@ export async function exportSpecV2(char:character) {
                         viewScreen: char.viewScreen,
                         customScripts: char.customscript,
                         utilityBot: char.utilityBot,
-                        sdData: char.sdData
+                        sdData: char.sdData,
+                        additionalAssets: char.additionalAssets
                     }
                 }
             }
@@ -478,13 +492,25 @@ export async function exportSpecV2(char:character) {
             for(let i=0;i<card.data.extensions.risuai.emotions.length;i++){
                 alertStore.set({
                     type: 'wait',
-                    msg: `Loading... (Getting Emotions ${i} / ${card.data.extensions.risuai.emotions.length})`
+                    msg: `Loading... (Adding Emotions ${i} / ${card.data.extensions.risuai.emotions.length})`
                 })
                 const rData = await readImage(card.data.extensions.risuai.emotions[i][1])
                 char.emotionImages[i][1] = Buffer.from(rData).toString('base64')
             }
         }
 
+        
+        if(card.data.extensions.risuai.additionalAssets && card.data.extensions.risuai.additionalAssets.length > 0){
+            for(let i=0;i<card.data.extensions.risuai.additionalAssets.length;i++){
+                alertStore.set({
+                    type: 'wait',
+                    msg: `Loading... (Adding Additional Assets ${i} / ${card.data.extensions.risuai.additionalAssets.length})`
+                })
+                const rData = await readImage(card.data.extensions.risuai.additionalAssets[i][1])
+                char.additionalAssets[i][1] = Buffer.from(rData).toString('base64')
+            }
+        }
+    
         alertStore.set({
             type: 'wait',
             msg: 'Loading... (Writing Exif)'
@@ -538,7 +564,8 @@ type CharacterCardV2 = {
                 viewScreen?: "none" | "emotion" | "imggen",
                 customScripts?:customscript[]
                 utilityBot?: boolean,
-                sdData?:[string,string][]
+                sdData?:[string,string][],
+                additionalAssets?:[string,string][],
             }
         }
     }

src/ts/database.ts

@@ -2,7 +2,7 @@ import { get, writable } from 'svelte/store';
 import { checkNullish } from './util';
 import { changeLanguage } from '../lang';
 import type { RisuPlugin } from './process/plugins';
-import { saveImage as saveImageGlobal } from './globalApi';
+import { saveAsset as saveImageGlobal } from './globalApi';
 import { cloneDeep } from 'lodash';
 
 export const DataBase = writable({} as any as Database)
@@ -277,6 +277,7 @@ export interface character{
     ttsMode?:string
     ttsSpeech?:string
     supaMemory?:boolean
+    additionalAssets?:[string, string][]
 }
 
 

src/ts/globalApi.ts

@@ -163,7 +163,7 @@ export async function readImage(data:string) {
     }
 }
 
-export async function saveImage(data:Uint8Array, customId:string = ''){
+export async function saveAsset(data:Uint8Array, customId:string = ''){
     let id = ''
     if(customId !== ''){
         id = customId
@@ -534,7 +534,7 @@ function getBasename(data:string){
 export function getUnpargeables(db:Database) {
     let unpargeable:string[] = []
 
-    function addParge(data:string){
+    function addUnparge(data:string){
         if(!data){
             return
         }
@@ -547,16 +547,23 @@ export function getUnpargeables(db:Database) {
         }
     }
 
-    addParge(db.customBackground)
+    addUnparge(db.customBackground)
-    addParge(db.userIcon)
+    addUnparge(db.userIcon)
 
     for(const cha of db.characters){
         if(cha.image){
-            addParge(cha.image)
+            addUnparge(cha.image)
         }
         if(cha.emotionImages){
             for(const em of cha.emotionImages){
-                addParge(em[1])
+                addUnparge(em[1])
+            }
+        }
+        if(cha.type !== 'group'){
+            if(cha.additionalAssets){
+                for(const em of cha.additionalAssets){
+                    addUnparge(em[1])
+                }
             }
         }
     }

src/ts/parser.ts

@@ -1,5 +1,7 @@
 import DOMPurify from 'isomorphic-dompurify';
 import showdown from 'showdown';
+import type { character, groupChat } from './database';
+import { getFileSrc } from './globalApi';
 
 const convertor = new showdown.Converter({
     simpleLineBreaks: true,
@@ -17,7 +19,15 @@ DOMPurify.addHook("uponSanitizeElement", (node: HTMLElement, data) => {
     }
 });
 
-export function ParseMarkdown(data:string) {
+export async function ParseMarkdown(data:string, char:(character | groupChat) = null) {
+    if(char && char.type !== 'group'){
+        if(char.additionalAssets){
+            for(const asset of char.additionalAssets){
+                const assetPath = await getFileSrc(asset[1])
+                data = data.replaceAll(`{{raw::${asset[0]}}}`, assetPath).replaceAll(`{{img::${asset[0]}}}`,`<img src="${asset[0]}" />`)
+            }
+        }
+    }
     return DOMPurify.sanitize(convertor.makeHtml(data), {
         ADD_TAGS: ["iframe"],
         ADD_ATTR: ["allow", "allowfullscreen", "frameborder", "scrolling"],

src/ts/process/scripts.ts

@@ -4,7 +4,7 @@ import type { character } from "../database";
 
 const dreg = /{{data}}/g
 
-type ScriptMode = 'editinput'|'editoutput'|'editprocess'
+type ScriptMode = 'editinput'|'editoutput'|'editprocess'|'editdisplay'
 
 export function processScript(char:character, data:string, mode:ScriptMode){
     return processScriptFull(char, data, mode).data