[feat] Add support for chat stickers (#178)

# PR Checklist
- [x] Did you check if it works normally in all models? *ignore this
when it dosen't uses models*
- [x] Did you check if it works normally in all of web, local and node
hosted versions? if it dosen't, did you blocked it in those versions?
- [x] Did you added a type def?

# Description
- This commit adds support for chat stickers by allowing users to use
stickers in chat message window.
- The users can toggle show additional asset list using a button.
- Added Additional assets file extension data.
- Added option for additional assets preview.
- Optimized render when use streaming api. (prevent markdown again when
message not changed)
Added controls to Video/Audio Assets


[Video11.webm](https://github.com/kwaroran/RisuAI/assets/11344967/8980282f-d001-4afc-a6d9-6f6369b4cd4c)


[Video12.webm](https://github.com/kwaroran/RisuAI/assets/11344967/e0e153b8-eba6-48a5-b47d-d2dd287f32fd)

src/lang/cn.ts

@@ -304,6 +304,8 @@ export const languageChinese = {
     tags: "标签",
     copied: "已复制",
     useChatCopy: "使用聊天复制",
+    useChatSticker: "使用聊天贴纸",
+    useAdditionalAssetsPreview: "使用附加资源预览",
     autoTranslateInput: "使用自动翻译输入",
     enterMessageForTranslateToEnglish: "输入要翻译为英语的消息",
     recent: '最新',

src/lang/en.ts

@@ -308,6 +308,8 @@ export const languageEnglish = {
     backgroundHTML: "Background Embedding",
     copied: "Copied",
     useChatCopy: "Use Chat Message Copy",
+    useChatSticker: "Use Chat Sticker",
+    useAdditionalAssetsPreview: "Use Additional Assets Preview",
     autoTranslateInput: "Auto Translate Input",
     enterMessageForTranslateToEnglish: "Enter Message for Translate to English",
     recent: 'Recent',

src/lang/ko.ts

@@ -280,6 +280,8 @@ export const languageKorean = {
     backgroundHTML: "백그라운드 임베딩",
     copied: "복사됨",
     useChatCopy: "채팅 메시지 복사 사용",
+    useChatSticker: "채팅 스티커 사용",
+    useAdditionalAssetsPreview: "추가 에셋 미리보기 사용",
     autoTranslateInput: "입력 자동 번역",
     enterMessageForTranslateToEnglish: "영어로 번역할 메시지를 입력해주세요",
     imageCompression: "이미지 압축",

src/lib/ChatScreens/AssetInput.svelte

@@ -0,0 +1,75 @@
+<script lang="ts">
+	import { get } from 'svelte/store';
+    import { FileAudioIcon, PlusIcon } from "lucide-svelte";
+    import { DataBase, setDatabase, type character, type groupChat } from "src/ts/storage/database";
+    import { getFileSrc, saveAsset } from "src/ts/storage/globalApi";
+    import { selectMultipleFile } from "src/ts/util";
+    export let currentCharacter:character|groupChat;
+    export let onSelect:(additionalAsset:[string,string,string])=>void;
+    let assetFileExtensions:string[] = []
+    let assetFilePath:string[] = []
+
+    $:{
+        if(currentCharacter.type ==='character'){
+            if(currentCharacter.additionalAssets){
+                for(let i = 0; i < currentCharacter.additionalAssets.length; i++){
+                    // console.log('check content type ...', currentCharacter.additionalAssets[i][0], currentCharacter.additionalAssets[i][1]);
+                    if(currentCharacter.additionalAssets[i].length > 2 && currentCharacter.additionalAssets[i][2]) {
+                        assetFileExtensions[i] = currentCharacter.additionalAssets[i][2]
+                    } else {
+                        assetFileExtensions[i] = currentCharacter.additionalAssets[i][1].split('.').pop()
+                    }
+                    getFileSrc(currentCharacter.additionalAssets[i][1]).then((filePath) => {
+                        assetFilePath[i] = filePath
+                    })
+                }
+            }
+        }
+    }
+</script>
+{#if currentCharacter.type ==='character'}
+    <button class="hover:text-green-500 bg-gray-500 flex justify-center items-center w-16 h-16 m-1 rounded-md" on:click={async () => {
+        if(currentCharacter.type === 'character'){
+            const da = await selectMultipleFile(['png', 'webp', 'mp4', 'mp3', 'gif'])
+            currentCharacter.additionalAssets = currentCharacter.additionalAssets ?? []
+            if(!da){
+                return
+            }
+            for(const f of da){
+                console.log(f)
+                const img = f.data
+                const name = f.name
+                const extension = name.split('.').pop().toLowerCase()
+                const imgp = await saveAsset(img,'',extension)
+                currentCharacter.additionalAssets.push([name, imgp, extension])
+                currentCharacter = currentCharacter
+            }
+            const db = get(DataBase);
+            setDatabase(db)
+        }
+    }}>
+        <PlusIcon />
+    </button>
+    {#if currentCharacter.additionalAssets}
+        {#each currentCharacter.additionalAssets as additionalAsset, i}
+                <button on:click={()=>{
+                    onSelect(additionalAsset)
+                }}>
+                    {#if assetFilePath[i]}
+                        {#if assetFileExtensions[i] === 'mp4'}
+                            <!-- svelte-ignore a11y-media-has-caption -->
+                            <video class="w-16 h-16 m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
+                        {:else if assetFileExtensions[i] === 'mp3'}
+                            <div class='w-16 h-16 m-1 rounded-md bg-slate-500 flex flex-col justify-center items-center'>
+                                <FileAudioIcon/>
+                                <div class='w-16 px-1 text-ellipsis whitespace-nowrap overflow-hidden'>{additionalAsset[0]}</div>
+                            </div>
+                            <!-- <audio controls class="w-16 h-16 m-1 rounded-md"><source src={assetPath} type="audio/mpeg"></audio> -->
+                        {:else}
+                        <img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={additionalAsset[0]}/>
+                        {/if}
+                    {/if}
+                </button>
+        {/each}
+    {/if}
+{/if}

src/lib/ChatScreens/Chat.svelte

@@ -8,6 +8,7 @@
     import { selectedCharID } from "../../ts/stores";
     import { translate } from "../../ts/translator/translator";
     import { replacePlaceholders } from "../../ts/util";
+    
     export let message = ''
     export let name = ''
     export let isLastMemory:boolean
@@ -17,6 +18,7 @@
     export let onReroll = () => {}
     export let unReroll = () => {}
     export let character:character|groupChat|null = null
+    let md:string
     let translating = false
     let editMode = false
     let statusMessage:string = ''
@@ -24,7 +26,8 @@
 
     let msgDisplay = ''
     let msgTranslated = ''
-    let translated = false;
+    let translated = false
+
     async function rm(){
         const rm = $DataBase.askRemoval ? await alertConfirm(language.removeChat) : true
         if(rm){
@@ -53,7 +56,7 @@
         $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message = msg
     }
 
-    async function displaya(message:string){
+    async function displaya(message:string, isStreaming:boolean = false){
         if($DataBase.autoTranslate && $DataBase.translator !== ''){
             if(msgTranslated==='')
                 msgDisplay = replacePlaceholders(message, name)
@@ -64,6 +67,12 @@
         else{
             msgDisplay = replacePlaceholders(message, name)
         }
+
+        if(!md || !isStreaming || isStreaming && idx === $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message.length - 1) {
+            ParseMarkdown(msgDisplay, character, 'normal').then(mdNew=>{
+                md = mdNew
+            })
+        }
     }
 
     const setStatusMessage = (message:string, timeout:number = 0)=>{
@@ -74,7 +83,7 @@
         }, timeout)
     }
 
-    $: displaya(message)
+    $: displaya(message, $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].isStreaming)
 </script>
 <div class="flex max-w-full justify-center" class:bgc={isLastMemory}>
     <div class="text-neutral-200 mt-1 ml-4 mr-4 mb-1 p-2 bg-transparent flex-grow border-t-gray-900 border-opacity-30 border-transparent flexium items-start max-w-full" >
@@ -161,19 +170,17 @@
             </div>
             {#if editMode}
                 <AutoresizeArea bind:value={message} />
-            {:else}
+            {:else if md}
-                {#await ParseMarkdown(msgDisplay, character) then md} 
+                <!-- svelte-ignore a11y-click-events-have-key-events -->
-                    <!-- svelte-ignore a11y-click-events-have-key-events -->
+                <span class="text chat chattext prose prose-invert minw-0" on:click={() => {
-                    <span class="text chat chattext prose prose-invert minw-0" on:click={() => {
+                    if($DataBase.clickToEdit && idx > -1){
-                        if($DataBase.clickToEdit && idx > -1){
+                        editMode = true
-                            editMode = true
+                        msgTranslated = ""
-                            msgTranslated = ""
+                    }
-                        }
+                }}
-                    }}
+                    style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
-                        style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
+                    style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
-                        style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
+                >{@html md}</span>
-                    >{@html md}</span>
-                {/await}
             {/if}
         </span>
 

src/lib/ChatScreens/DefaultChatScreen.svelte

@@ -1,9 +1,9 @@
 <script lang="ts">
 	import Suggestion from './Suggestion.svelte';
-    import { DatabaseIcon, DicesIcon, LanguagesIcon, MenuIcon, MicOffIcon, PowerIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
+    import { DatabaseIcon, DicesIcon, LanguagesIcon, Laugh, MenuIcon, MicOffIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
     import { selectedCharID } from "../../ts/stores";
     import Chat from "./Chat.svelte";
-    import { DataBase, type Message } from "../../ts/storage/database";
+    import { DataBase, type Message, type character, type groupChat } from "../../ts/storage/database";
     import { getCharImage } from "../../ts/characters";
     import { doingChat, sendChat } from "../../ts/process/index";
     import { findCharacterbyId, messageForm, sleep } from "../../ts/util";
@@ -17,6 +17,7 @@
     import { stopTTS } from "src/ts/process/tts";
     import MainMenu from '../UI/MainMenu.svelte';
     import Help from '../Others/Help.svelte';
+    import AssetInput from './AssetInput.svelte';
 
     let messageInput:string = ''
     let messageInputTranslate:string = ''
@@ -28,6 +29,8 @@
     let rerollid = -1
     let lastCharId = -1
     let doingChatInputTranslate = false
+    let currentCharacter:character|groupChat = $DataBase.characters[$selectedCharID]
+    let toggleStickers:boolean = false
 
     async function send() {
         let selectedChar = $selectedCharID
@@ -218,6 +221,10 @@
             }
         })
     }
+
+    $: {
+        currentCharacter = $DataBase.characters[$selectedCharID]
+    }
 </script>
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div class="w-full h-full" style={customStyle} on:click={() => {
@@ -234,6 +241,12 @@
             }
         }}>
             <div class="flex items-end mt-2 mb-2 w-full">
+                {#if $DataBase.useChatSticker && currentCharacter.type !== 'group'}
+                    <div on:click={()=>{toggleStickers = !toggleStickers}}
+                            class={"ml-4 bg-gray-500 flex justify-center items-center  w-12 h-12 rounded-md hover:bg-green-500 transition-colors "+(toggleStickers ? 'text-green-500':'text-white')}>
+                            <Laugh/>
+                    </div>    
+                {/if}
                 <textarea class="text-neutral-200 p-2 min-w-0 bg-transparent input-text text-xl flex-grow ml-4 mr-2 border-gray-700 resize-none focus:bg-selected overflow-y-hidden overflow-x-hidden max-w-full"
                     bind:value={messageInput}
                     bind:this={inputEle}
@@ -299,9 +312,27 @@
                 </div>
             {/if}
             
+            {#if toggleStickers}
+                <div class="ml-4 flex flex-wrap">
+                    <AssetInput bind:currentCharacter={currentCharacter} onSelect={(additionalAsset)=>{
+                        let fileType = 'img'
+                        if(additionalAsset.length > 2 && additionalAsset[2]) {
+                            const fileExtension = additionalAsset[2]
+                            if(fileExtension === 'mp4' || fileExtension === 'webm')
+                                fileType = 'video'
+                            else if(fileExtension === 'mp3' || fileExtension === 'wav')
+                                fileType = 'audio'
+                        }
+                        messageInput += `<span class='notranslate' translate='no'>{{${fileType}::${additionalAsset[0]}}}</span> *${additionalAsset[0]} added*`
+                        updateInputSizeAll()
+                    }}/>
+                </div>    
+            {/if}
+
             {#if $DataBase.useAutoSuggestions}
                 <Suggestion messageInput={(msg)=>messageInput=msg} {send}/>
             {/if}
+            
             {#each messageForm($DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message, loadPages) as chat, i}
                 {#if chat.role === 'char'}
                     {#if $DataBase.characters[$selectedCharID].type !== 'group'}

src/lib/Setting/Pages/DisplaySettings.svelte

@@ -142,3 +142,14 @@
 <div class="flex items-center mt-2">
     <Check bind:check={$DataBase.useChatCopy} name={language.useChatCopy}/>
 </div>
+
+<div class="flex items-center mt-2">
+    <Check bind:check={$DataBase.useAdditionalAssetsPreview} name={language.useAdditionalAssetsPreview}/>
+</div>
+
+{#if $DataBase.useExperimental}
+    <div class="flex items-center mt-2">
+        <Check bind:check={$DataBase.useChatSticker} name={language.useChatSticker}/>
+        <Help key="experimental" name={language.useChatSticker}/>
+    </div>
+{/if}

src/lib/SideBars/CharConfig.svelte

@@ -16,7 +16,7 @@
     import RegexData from "./RegexData.svelte";
     import { exportChar, shareRisuHub } from "src/ts/characterCards";
     import { getElevenTTSVoices, getWebSpeechTTSVoices, getVOICEVOXVoices } from "src/ts/process/tts";
-    import { checkCharOrder } from "src/ts/storage/globalApi";
+    import { checkCharOrder, getFileSrc } from "src/ts/storage/globalApi";
     import { addGroupChar, rmCharFromGroup } from "src/ts/process/group";
   import HubUpload from "../UI/HubUpload.svelte";
 
@@ -95,6 +95,8 @@
         currentChar = currentChar
     })
 
+    let assetFileExtensions:string[] = []
+    let assetFilePath:string[] = []
 
     $: {
         if(database.characters[$selectedCharID].chaId === currentChar.data.chaId){
@@ -106,6 +108,20 @@
         emos = currentChar.data.emotionImages
         DataBase.set(database)
         loadTokenize(currentChar.data)
+
+        if(currentChar.type ==='character' && database.useAdditionalAssetsPreview){
+            if(currentChar.data.additionalAssets){
+                for(let i = 0; i < currentChar.data.additionalAssets.length; i++){
+                    if(currentChar.data.additionalAssets[i].length > 2 && currentChar.data.additionalAssets[i][2]) {
+                        assetFileExtensions[i] = currentChar.data.additionalAssets[i][2]
+                    } else 
+                        assetFileExtensions[i] = currentChar.data.additionalAssets[i][1].split('.').pop()
+                    getFileSrc(currentChar.data.additionalAssets[i][1]).then((filePath) => {
+                        assetFilePath[i] = filePath
+                    })
+                }
+            }
+        }
     }
 
     onDestroy(unsub);
@@ -615,9 +631,10 @@
                             for(const f of da){
                                 console.log(f)
                                 const img = f.data
-                                const imgp = await saveAsset(img)
                                 const name = f.name
-                                currentChar.data.additionalAssets.push([name, imgp])
+                                const extension = name.split('.').pop().toLowerCase()
+                                const imgp = await saveAsset(img,'', extension)
+                                currentChar.data.additionalAssets.push([name, imgp, extension])
                                 currentChar.data.additionalAssets = currentChar.data.additionalAssets
                             }
                         }
@@ -634,8 +651,19 @@
                 {#each currentChar.data.additionalAssets as assets, i}
                     <tr>
                         <td class="font-medium truncate">
+                            {#if assetFilePath[i] && database.useAdditionalAssetsPreview}
+                                {#if assetFileExtensions[i] === 'mp4'}
+                                <!-- svelte-ignore a11y-media-has-caption -->
+                                    <video controls class="mt-2 px-2 w-full m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
+                                {:else if assetFileExtensions[i] === 'mp3'}
+                                    <audio controls class="mt-2 px-2 w-full h-16 m-1 rounded-md" loop><source src={assetFilePath[i]} type="audio/mpeg"></audio>
+                                {:else}
+                                    <img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={assets[0]}/>
+                                {/if}
+                            {/if}
                             <input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected w-full resize-none" bind:value={currentChar.data.additionalAssets[i][0]} placeholder="..." />
                         </td>
+                        
                         <th class="font-medium cursor-pointer w-10">
                             <button class="hover:text-green-500" on:click={() => {
                                 if(currentChar.type === 'character'){

src/ts/characterCards.ts

@@ -329,7 +329,7 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
     let customScripts:customscript[] = []
     let utilityBot = false
     let sdData = defaultSdDataFunc()
-    let extAssets:[string,string][] = []
+    let extAssets:[string,string,string][] = []
 
     if(risuext){
         if(risuext.emotions){
@@ -350,8 +350,11 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
                     msg: `Loading... (Getting Assets ${i} / ${risuext.additionalAssets.length})`
                 })
                 await sleep(10)
-                const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'))
+                let fileName = ''
-                extAssets.push([risuext.additionalAssets[i][0],imgp])
+                if(risuext.additionalAssets[i].length >= 3)
+                    fileName = risuext.additionalAssets[i][2]
+                const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'), '', fileName)
+                extAssets.push([risuext.additionalAssets[i][0],imgp,fileName])
             }
         }
         bias = risuext.bias ?? bias
@@ -756,7 +759,7 @@ type CharacterCardV2 = {
                 customScripts?:customscript[]
                 utilityBot?: boolean,
                 sdData?:[string,string][],
-                additionalAssets?:[string,string][],
+                additionalAssets?:[string,string,string][],
                 backgroundHTML?:string
             }
         }

src/ts/parser.ts

@@ -63,9 +63,9 @@ async function parseAdditionalAssets(data:string, char:character, mode:'normal'|
         for(const asset of char.additionalAssets){
             const assetPath = await getFileSrc(asset[1])
             data = data.replaceAll(`{{raw::${asset[0]}}}`, assetPath).
-                    replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" />`)
+                    replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" alt="${asset[0]}"/>`)
-                    .replaceAll(`{{video::${asset[0]}}}`,`<video autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
+                    .replaceAll(`{{video::${asset[0]}}}`,`<video controls autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
-                    .replaceAll(`{{audio::${asset[0]}}}`,`<audio autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
+                    .replaceAll(`{{audio::${asset[0]}}}`,`<audio controls autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
             if(mode === 'back'){
                 data = data.replaceAll(`{{bg::${asset[0]}}}`, `<div style="width:100%;height:100%;background: linear-gradient(rgba(0, 0, 0, 0.8), rgba(0, 0, 0, 0.8)),url(${assetPath}); background-size: cover;"></div>`)
             }

src/ts/process/index.ts

@@ -409,6 +409,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
     else if(req.type === 'streaming'){
         const reader = req.result.getReader()
         const msgIndex = db.characters[selectedChar].chats[selectedChat].message.length
+        db.characters[selectedChar].chats[selectedChat].isStreaming = true
         db.characters[selectedChar].chats[selectedChat].message.push({
             role: 'char',
             data: "",
@@ -424,6 +425,8 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
                 setDatabase(db)
             }
             if(readed.done){
+                db.characters[selectedChar].chats[selectedChat].isStreaming = false
+                setDatabase(db)
                 break
             }   
         }

src/ts/storage/database.ts

@@ -342,7 +342,7 @@ export interface character{
         VOLUME_SCALE?: number
     }
     supaMemory?:boolean
-    additionalAssets?:[string, string][]
+    additionalAssets?:[string, string, string][]
     ttsReadOnlyQuoted?:boolean
     replaceGlobalNote:string
     backgroundHTML?:string
@@ -525,7 +525,9 @@ export interface Database{
             expires_in?: number
         }
     },
-    classicMaxWidth: boolean
+    classicMaxWidth: boolean,
+    useChatSticker:boolean,
+    useAdditionalAssetsPreview:boolean,
 }
 
 interface hordeConfig{
@@ -564,6 +566,7 @@ export interface Chat{
     supaMemoryData?:string
     lastMemory?:string
     suggestMessages?:string[]
+    isStreaming?:boolean
 }
 
 export interface Message{

src/ts/storage/globalApi.ts

@@ -170,7 +170,7 @@ export async function readImage(data:string) {
     }
 }
 
-export async function saveAsset(data:Uint8Array, customId:string = ''){
+export async function saveAsset(data:Uint8Array, customId:string = '', fileName:string = ''){
     let id = ''
     if(customId !== ''){
         id = customId
@@ -182,13 +182,17 @@ export async function saveAsset(data:Uint8Array, customId:string = ''){
             id = uuidv4()
         }
     }
+    let fileExtension:string = 'png'
+    if(fileName && fileName.split('.').length > 0){
+        fileExtension = fileName.split('.').pop()
+    }
     if(isTauri){
-        await writeBinaryFile(`assets/${id}.png`, data ,{dir: BaseDirectory.AppData})
+        await writeBinaryFile(`assets/${id}.${fileExtension}`, data ,{dir: BaseDirectory.AppData})
-        return `assets/${id}.png`
+        return `assets/${id}.${fileExtension}`
     }
     else{
-        await forageStorage.setItem(`assets/${id}.png`, data)
+        await forageStorage.setItem(`assets/${id}.${fileExtension}`, data)
-        return `assets/${id}.png`
+        return `assets/${id}.${fileExtension}`
     }
 }