[feat] Add support for chat stickers

This commit adds support for chat stickers by allowing users to use stickers in chat message window.
The users can toggle show additional asset list using a button.
Added Additional assets file extension data.
Additional assets list shows preview.
Optimized render when use streaming api. (prevent markdown again when message not changed)
Added controls to Video/Audio Assets

src/lang/cn.ts

@@ -304,6 +304,7 @@ export const languageChinese = {
     tags: "标签",
     copied: "已复制",
     useChatCopy: "使用聊天复制",
+    useChatSticker: "使用聊天贴纸",
     autoTranslateInput: "使用自动翻译输入",
     enterMessageForTranslateToEnglish: "输入要翻译为英语的消息",
     recent: '最新',

src/lang/en.ts

@@ -308,6 +308,7 @@ export const languageEnglish = {
     backgroundHTML: "Background Embedding",
     copied: "Copied",
     useChatCopy: "Use Chat Message Copy",
+    useChatSticker: "Use Chat Sticker",
     autoTranslateInput: "Auto Translate Input",
     enterMessageForTranslateToEnglish: "Enter Message for Translate to English",
     recent: 'Recent',

src/lang/ko.ts

@@ -280,6 +280,7 @@ export const languageKorean = {
     backgroundHTML: "백그라운드 임베딩",
     copied: "복사됨",
     useChatCopy: "채팅 메시지 복사 사용",
+    useChatSticker: "채팅 스티커 사용",
     autoTranslateInput: "입력 자동 번역",
     enterMessageForTranslateToEnglish: "영어로 번역할 메시지를 입력해주세요",
     imageCompression: "이미지 압축",

src/lib/ChatScreens/AssetInput.svelte

@@ -0,0 +1,75 @@
+<script lang="ts">
+	import { get } from 'svelte/store';
+    import { FileAudioIcon, PlusIcon } from "lucide-svelte";
+    import { DataBase, setDatabase, type character, type groupChat } from "src/ts/storage/database";
+    import { getFileSrc, saveAsset } from "src/ts/storage/globalApi";
+    import { selectMultipleFile } from "src/ts/util";
+    export let currentCharacter:character|groupChat;
+    export let onSelect:(additionalAsset:[string,string,string])=>void;
+    let assetFileExtensions:string[] = []
+    let assetFilePath:string[] = []
+
+    $:{
+        if(currentCharacter.type ==='character'){
+            if(currentCharacter.additionalAssets){
+                for(let i = 0; i < currentCharacter.additionalAssets.length; i++){
+                    // console.log('check content type ...', currentCharacter.additionalAssets[i][0], currentCharacter.additionalAssets[i][1]);
+                    if(currentCharacter.additionalAssets[i].length > 2 && currentCharacter.additionalAssets[i][2]) {
+                        assetFileExtensions[i] = currentCharacter.additionalAssets[i][2]
+                    } else {
+                        assetFileExtensions[i] = currentCharacter.additionalAssets[i][1].split('.').pop()
+                    }
+                    getFileSrc(currentCharacter.additionalAssets[i][1]).then((filePath) => {
+                        assetFilePath[i] = filePath
+                    })
+                }
+            }
+        }
+    }
+</script>
+{#if currentCharacter.type ==='character'}
+    <button class="hover:text-green-500 bg-gray-500 flex justify-center items-center w-16 h-16 m-1 rounded-md" on:click={async () => {
+        if(currentCharacter.type === 'character'){
+            const da = await selectMultipleFile(['png', 'webp', 'mp4', 'mp3', 'gif'])
+            currentCharacter.additionalAssets = currentCharacter.additionalAssets ?? []
+            if(!da){
+                return
+            }
+            for(const f of da){
+                console.log(f)
+                const img = f.data
+                const name = f.name
+                const extension = name.split('.').pop().toLowerCase()
+                const imgp = await saveAsset(img,'',extension)
+                currentCharacter.additionalAssets.push([name, imgp, extension])
+                currentCharacter = currentCharacter
+            }
+            const db = get(DataBase);
+            setDatabase(db)
+        }
+    }}>
+        <PlusIcon />
+    </button>
+    {#if currentCharacter.additionalAssets}
+        {#each currentCharacter.additionalAssets as additionalAsset, i}
+                <button on:click={()=>{
+                    onSelect(additionalAsset)
+                }}>
+                    {#if assetFilePath[i]}
+                        {#if assetFileExtensions[i] === 'mp4'}
+                            <!-- svelte-ignore a11y-media-has-caption -->
+                            <video class="w-16 h-16 m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
+                        {:else if assetFileExtensions[i] === 'mp3'}
+                            <div class='w-16 h-16 m-1 rounded-md bg-slate-500 flex flex-col justify-center items-center'>
+                                <FileAudioIcon/>
+                                <div class='w-16 px-1 text-ellipsis whitespace-nowrap overflow-hidden'>{additionalAsset[0]}</div>
+                            </div>
+                            <!-- <audio controls class="w-16 h-16 m-1 rounded-md"><source src={assetPath} type="audio/mpeg"></audio> -->
+                        {:else}
+                        <img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={additionalAsset[0]}/>
+                        {/if}
+                    {/if}
+                </button>
+        {/each}
+    {/if}
+{/if}

src/lib/ChatScreens/Chat.svelte

@@ -8,6 +8,7 @@
     import { selectedCharID } from "../../ts/stores";
     import { translate } from "../../ts/translator/translator";
     import { replacePlaceholders } from "../../ts/util";
+    
     export let message = ''
     export let name = ''
     export let isLastMemory:boolean
@@ -17,6 +18,7 @@
     export let onReroll = () => {}
     export let unReroll = () => {}
     export let character:character|groupChat|null = null
+    let md:string
     let translating = false
     let editMode = false
     let statusMessage:string = ''
@@ -24,7 +26,8 @@
 
     let msgDisplay = ''
     let msgTranslated = ''
-    let translated = false;
+    let translated = false
+
     async function rm(){
         const rm = $DataBase.askRemoval ? await alertConfirm(language.removeChat) : true
         if(rm){
@@ -53,7 +56,7 @@
         $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message = msg
     }
 
-    async function displaya(message:string){
+    async function displaya(message:string, isStreaming:boolean = false){
         if($DataBase.autoTranslate && $DataBase.translator !== ''){
             if(msgTranslated==='')
                 msgDisplay = replacePlaceholders(message, name)
@@ -64,6 +67,12 @@
         else{
             msgDisplay = replacePlaceholders(message, name)
         }
+
+        if(!md || !isStreaming || isStreaming && idx === $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message.length - 1) {
+            ParseMarkdown(msgDisplay, character, 'normal').then(mdNew=>{
+                md = mdNew
+            })
+        }
     }
 
     const setStatusMessage = (message:string, timeout:number = 0)=>{
@@ -74,7 +83,7 @@
         }, timeout)
     }
 
-    $: displaya(message)
+    $: displaya(message, $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].isStreaming)
 </script>
 <div class="flex max-w-full justify-center" class:bgc={isLastMemory}>
     <div class="text-neutral-200 mt-1 ml-4 mr-4 mb-1 p-2 bg-transparent flex-grow border-t-gray-900 border-opacity-30 border-transparent flexium items-start max-w-full" >
@@ -161,19 +170,17 @@
             </div>
             {#if editMode}
                 <AutoresizeArea bind:value={message} />
-            {:else}
+            {:else if md}
-                {#await ParseMarkdown(msgDisplay, character) then md} 
+                <!-- svelte-ignore a11y-click-events-have-key-events -->
-                    <!-- svelte-ignore a11y-click-events-have-key-events -->
+                <span class="text chat chattext prose prose-invert minw-0" on:click={() => {
-                    <span class="text chat chattext prose prose-invert minw-0" on:click={() => {
+                    if($DataBase.clickToEdit && idx > -1){
-                        if($DataBase.clickToEdit && idx > -1){
+                        editMode = true
-                            editMode = true
+                        msgTranslated = ""
-                            msgTranslated = ""
+                    }
-                        }
+                }}
-                    }}
+                    style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
-                        style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
+                    style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
-                        style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
+                >{@html md}</span>
-                    >{@html md}</span>
-                {/await}
             {/if}
         </span>
 

src/lib/ChatScreens/DefaultChatScreen.svelte

@@ -1,9 +1,9 @@
 <script lang="ts">
 	import Suggestion from './Suggestion.svelte';
-    import { DatabaseIcon, DicesIcon, LanguagesIcon, MenuIcon, MicOffIcon, PowerIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
+    import { DatabaseIcon, DicesIcon, LanguagesIcon, Laugh, MenuIcon, MicOffIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
     import { selectedCharID } from "../../ts/stores";
     import Chat from "./Chat.svelte";
-    import { DataBase, type Message } from "../../ts/storage/database";
+    import { DataBase, type Message, type character, type groupChat } from "../../ts/storage/database";
     import { getCharImage } from "../../ts/characters";
     import { doingChat, sendChat } from "../../ts/process/index";
     import { findCharacterbyId, messageForm, sleep } from "../../ts/util";
@@ -17,6 +17,7 @@
     import { stopTTS } from "src/ts/process/tts";
     import MainMenu from '../UI/MainMenu.svelte';
     import Help from '../Others/Help.svelte';
+    import AssetInput from './AssetInput.svelte';
 
     let messageInput:string = ''
     let messageInputTranslate:string = ''
@@ -28,6 +29,8 @@
     let rerollid = -1
     let lastCharId = -1
     let doingChatInputTranslate = false
+    let currentCharacter:character|groupChat = $DataBase.characters[$selectedCharID]
+    let toggleStickers:boolean = false
 
     async function send() {
         let selectedChar = $selectedCharID
@@ -218,6 +221,10 @@
             }
         })
     }
+
+    $: {
+        currentCharacter = $DataBase.characters[$selectedCharID]
+    }
 </script>
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div class="w-full h-full" style={customStyle} on:click={() => {
@@ -234,6 +241,12 @@
             }
         }}>
             <div class="flex items-end mt-2 mb-2 w-full">
+                {#if $DataBase.useChatSticker && currentCharacter.type !== 'group'}
+                    <div on:click={()=>{toggleStickers = !toggleStickers}}
+                            class={"ml-4 bg-gray-500 flex justify-center items-center  w-12 h-12 rounded-md hover:bg-green-500 transition-colors "+(toggleStickers ? 'text-green-500':'text-white')}>
+                            <Laugh/>
+                    </div>    
+                {/if}
                 <textarea class="text-neutral-200 p-2 min-w-0 bg-transparent input-text text-xl flex-grow ml-4 mr-2 border-gray-700 resize-none focus:bg-selected overflow-y-hidden overflow-x-hidden max-w-full"
                     bind:value={messageInput}
                     bind:this={inputEle}
@@ -299,9 +312,27 @@
                 </div>
             {/if}
             
+            {#if toggleStickers}
+                <div class="ml-4 flex flex-wrap">
+                    <AssetInput bind:currentCharacter={currentCharacter} onSelect={(additionalAsset)=>{
+                        let fileType = 'img'
+                        if(additionalAsset.length > 2 && additionalAsset[2]) {
+                            const fileExtension = additionalAsset[2]
+                            if(fileExtension === 'mp4' || fileExtension === 'webm')
+                                fileType = 'video'
+                            else if(fileExtension === 'mp3' || fileExtension === 'wav')
+                                fileType = 'audio'
+                        }
+                        messageInput += `<span class='notranslate' translate='no'>{{${fileType}::${additionalAsset[0]}}}</span> *${additionalAsset[0]} added*`
+                        updateInputSizeAll()
+                    }}/>
+                </div>    
+            {/if}
+
             {#if $DataBase.useAutoSuggestions}
                 <Suggestion messageInput={(msg)=>messageInput=msg} {send}/>
             {/if}
+            
             {#each messageForm($DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message, loadPages) as chat, i}
                 {#if chat.role === 'char'}
                     {#if $DataBase.characters[$selectedCharID].type !== 'group'}

src/lib/Setting/Pages/DisplaySettings.svelte

@@ -141,4 +141,12 @@
 
 <div class="flex items-center mt-2">
     <Check bind:check={$DataBase.useChatCopy} name={language.useChatCopy}/>
-</div>
+</div>
+
+
+{#if $DataBase.useExperimental}
+    <div class="flex items-center mt-2">
+        <Check bind:check={$DataBase.useChatSticker} name={language.useChatSticker}/>
+        <Help key="experimental" name={language.useChatSticker}/>
+    </div>
+{/if}

src/lib/SideBars/CharConfig.svelte

@@ -16,7 +16,7 @@
     import RegexData from "./RegexData.svelte";
     import { exportChar, shareRisuHub } from "src/ts/characterCards";
     import { getElevenTTSVoices, getWebSpeechTTSVoices, getVOICEVOXVoices } from "src/ts/process/tts";
-    import { checkCharOrder } from "src/ts/storage/globalApi";
+    import { checkCharOrder, getFileSrc } from "src/ts/storage/globalApi";
     import { addGroupChar, rmCharFromGroup } from "src/ts/process/group";
   import HubUpload from "../UI/HubUpload.svelte";
 
@@ -95,7 +95,9 @@
         currentChar = currentChar
     })
 
-    
+    let assetFileExtensions:string[] = []
+    let assetFilePath:string[] = []
+
     $: {
         if(database.characters[$selectedCharID].chaId === currentChar.data.chaId){
             database.characters[$selectedCharID] = currentChar.data
@@ -106,6 +108,20 @@
         emos = currentChar.data.emotionImages
         DataBase.set(database)
         loadTokenize(currentChar.data)
+
+        if(currentChar.type ==='character'){
+            if(currentChar.data.additionalAssets){
+                for(let i = 0; i < currentChar.data.additionalAssets.length; i++){
+                    if(currentChar.data.additionalAssets[i].length > 2 && currentChar.data.additionalAssets[i][2]) {
+                        assetFileExtensions[i] = currentChar.data.additionalAssets[i][2]
+                    } else 
+                        assetFileExtensions[i] = currentChar.data.additionalAssets[i][1].split('.').pop()
+                    getFileSrc(currentChar.data.additionalAssets[i][1]).then((filePath) => {
+                        assetFilePath[i] = filePath
+                    })
+                }
+            }
+        }
     }
 
     onDestroy(unsub);
@@ -615,9 +631,10 @@
                             for(const f of da){
                                 console.log(f)
                                 const img = f.data
-                                const imgp = await saveAsset(img)
                                 const name = f.name
-                                currentChar.data.additionalAssets.push([name, imgp])
+                                const extension = name.split('.').pop().toLowerCase()
+                                const imgp = await saveAsset(img,'', extension)
+                                currentChar.data.additionalAssets.push([name, imgp, extension])
                                 currentChar.data.additionalAssets = currentChar.data.additionalAssets
                             }
                         }
@@ -634,6 +651,16 @@
                 {#each currentChar.data.additionalAssets as assets, i}
                     <tr>
                         <td class="font-medium truncate">
+                            {#if assetFilePath[i]}
+                                {#if assetFileExtensions[i] === 'mp4'}
+                                <!-- svelte-ignore a11y-media-has-caption -->
+                                    <video controls class="mt-2 px-2 w-full m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
+                                {:else if assetFileExtensions[i] === 'mp3'}
+                                    <audio controls class="mt-2 px-2 w-full h-16 m-1 rounded-md" loop><source src={assetFilePath[i]} type="audio/mpeg"></audio>
+                                {:else}
+                                    <img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={assets[0]}/>
+                                {/if}
+                            {/if}
                             <input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected w-full resize-none" bind:value={currentChar.data.additionalAssets[i][0]} placeholder="..." />
                         </td>
                         <th class="font-medium cursor-pointer w-10">

src/ts/characterCards.ts

@@ -329,7 +329,7 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
     let customScripts:customscript[] = []
     let utilityBot = false
     let sdData = defaultSdDataFunc()
-    let extAssets:[string,string][] = []
+    let extAssets:[string,string,string][] = []
 
     if(risuext){
         if(risuext.emotions){
@@ -350,8 +350,11 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
                     msg: `Loading... (Getting Assets ${i} / ${risuext.additionalAssets.length})`
                 })
                 await sleep(10)
-                const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'))
+                let fileName = ''
-                extAssets.push([risuext.additionalAssets[i][0],imgp])
+                if(risuext.additionalAssets[i].length >= 3)
+                    fileName = risuext.additionalAssets[i][2]
+                const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'), '', fileName)
+                extAssets.push([risuext.additionalAssets[i][0],imgp,fileName])
             }
         }
         bias = risuext.bias ?? bias
@@ -756,7 +759,7 @@ type CharacterCardV2 = {
                 customScripts?:customscript[]
                 utilityBot?: boolean,
                 sdData?:[string,string][],
-                additionalAssets?:[string,string][],
+                additionalAssets?:[string,string,string][],
                 backgroundHTML?:string
             }
         }

src/ts/parser.ts

@@ -64,9 +64,9 @@ export async function ParseMarkdown(data:string, char:(character | groupChat) =
             for(const asset of char.additionalAssets){
                 const assetPath = await getFileSrc(asset[1])
                 data = data.replaceAll(`{{raw::${asset[0]}}}`, assetPath).
-                        replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" />`)
+                        replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" alt="${asset[0]}"/>`)
-                        .replaceAll(`{{video::${asset[0]}}}`,`<video autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
+                        .replaceAll(`{{video::${asset[0]}}}`,`<video controls autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
-                        .replaceAll(`{{audio::${asset[0]}}}`,`<audio autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
+                        .replaceAll(`{{audio::${asset[0]}}}`,`<audio controls autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
                 if(mode === 'back'){
                     data = data.replaceAll(`{{bg::${asset[0]}}}`, `<div style="width:100%;height:100%;background: linear-gradient(rgba(0, 0, 0, 0.8), rgba(0, 0, 0, 0.8)),url(${assetPath}); background-size: cover;"></div>`)
                 }

src/ts/process/index.ts

@@ -404,6 +404,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
     else if(req.type === 'streaming'){
         const reader = req.result.getReader()
         const msgIndex = db.characters[selectedChar].chats[selectedChat].message.length
+        db.characters[selectedChar].chats[selectedChat].isStreaming = true
         db.characters[selectedChar].chats[selectedChat].message.push({
             role: 'char',
             data: "",
@@ -419,6 +420,8 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
                 setDatabase(db)
             }
             if(readed.done){
+                db.characters[selectedChar].chats[selectedChat].isStreaming = false
+                setDatabase(db)
                 break
             }   
         }

src/ts/storage/database.ts

@@ -339,7 +339,7 @@ export interface character{
         VOLUME_SCALE?: number
     }
     supaMemory?:boolean
-    additionalAssets?:[string, string][]
+    additionalAssets?:[string, string, string][]
     ttsReadOnlyQuoted?:boolean
     replaceGlobalNote:string
     backgroundHTML?:string
@@ -522,7 +522,8 @@ export interface Database{
             expires_in?: number
         }
     },
-    classicMaxWidth: boolean
+    classicMaxWidth: boolean,
+    useChatSticker:boolean,
 }
 
 interface hordeConfig{
@@ -561,6 +562,7 @@ export interface Chat{
     supaMemoryData?:string
     lastMemory?:string
     suggestMessages?:string[]
+    isStreaming?:boolean
 }
 
 export interface Message{

src/ts/storage/globalApi.ts

@@ -170,7 +170,7 @@ export async function readImage(data:string) {
     }
 }
 
-export async function saveAsset(data:Uint8Array, customId:string = ''){
+export async function saveAsset(data:Uint8Array, customId:string = '', fileName:string = ''){
     let id = ''
     if(customId !== ''){
         id = customId
@@ -182,13 +182,17 @@ export async function saveAsset(data:Uint8Array, customId:string = ''){
             id = uuidv4()
         }
     }
+    let fileExtension:string = 'png'
+    if(fileName && fileName.split('.').length > 0){
+        fileExtension = fileName.split('.').pop()
+    }
     if(isTauri){
-        await writeBinaryFile(`assets/${id}.png`, data ,{dir: BaseDirectory.AppData})
+        await writeBinaryFile(`assets/${id}.${fileExtension}`, data ,{dir: BaseDirectory.AppData})
-        return `assets/${id}.png`
+        return `assets/${id}.${fileExtension}`
     }
     else{
-        await forageStorage.setItem(`assets/${id}.png`, data)
+        await forageStorage.setItem(`assets/${id}.${fileExtension}`, data)
-        return `assets/${id}.png`
+        return `assets/${id}.${fileExtension}`
     }
 }