mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] Add support for chat stickers

Browse Source 
This commit adds support for chat stickers by allowing users to use stickers in chat message window.
The users can toggle show additional asset list using a button.
Added Additional assets file extension data.
Additional assets list shows preview.
Optimized render when use streaming api. (prevent markdown again when message not changed)
Added controls to Video/Audio Assets
This commit is contained in:
LL
2023-06-22 07:10:18 +09:00
parent 89015ac009
commit d8a5ee5b4e
13 changed files with 200 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/lang/cn.ts
View File

@@ -304,6 +304,7 @@ export const languageChinese = {
tags: "标签",
copied: "已复制",
useChatCopy: "使用聊天复制",
useChatSticker: "使用聊天贴纸",
autoTranslateInput: "使用自动翻译输入",
enterMessageForTranslateToEnglish: "输入要翻译为英语的消息",
recent: '最新',

1
src/lang/en.ts
View File

@@ -308,6 +308,7 @@ export const languageEnglish = {
backgroundHTML: "Background Embedding",
copied: "Copied",
useChatCopy: "Use Chat Message Copy",
useChatSticker: "Use Chat Sticker",
autoTranslateInput: "Auto Translate Input",
enterMessageForTranslateToEnglish: "Enter Message for Translate to English",
recent: 'Recent',

1
src/lang/ko.ts
View File

@@ -280,6 +280,7 @@ export const languageKorean = {
backgroundHTML: "백그라운드 임베딩",
copied: "복사됨",
useChatCopy: "채팅 메시지 복사 사용",
useChatSticker: "채팅 스티커 사용",
autoTranslateInput: "입력 자동 번역",
enterMessageForTranslateToEnglish: "영어로 번역할 메시지를 입력해주세요",
imageCompression: "이미지 압축",

75
src/lib/ChatScreens/AssetInput.svelte Normal file
View File

@@ -0,0 +1,75 @@
<script lang="ts">
import { get } from 'svelte/store';
import { FileAudioIcon, PlusIcon } from "lucide-svelte";
import { DataBase, setDatabase, type character, type groupChat } from "src/ts/storage/database";
import { getFileSrc, saveAsset } from "src/ts/storage/globalApi";
import { selectMultipleFile } from "src/ts/util";
export let currentCharacter:character|groupChat;
export let onSelect:(additionalAsset:[string,string,string])=>void;
let assetFileExtensions:string[] = []
let assetFilePath:string[] = []
$:{
if(currentCharacter.type ==='character'){
if(currentCharacter.additionalAssets){
for(let i = 0; i < currentCharacter.additionalAssets.length; i++){
// console.log('check content type ...', currentCharacter.additionalAssets[i][0], currentCharacter.additionalAssets[i][1]);
if(currentCharacter.additionalAssets[i].length > 2 && currentCharacter.additionalAssets[i][2]) {
assetFileExtensions[i] = currentCharacter.additionalAssets[i][2]
} else {
assetFileExtensions[i] = currentCharacter.additionalAssets[i][1].split('.').pop()
}
getFileSrc(currentCharacter.additionalAssets[i][1]).then((filePath) => {
assetFilePath[i] = filePath
})
}
}
}
}
</script>
{#if currentCharacter.type ==='character'}
<button class="hover:text-green-500 bg-gray-500 flex justify-center items-center w-16 h-16 m-1 rounded-md" on:click={async () => {
if(currentCharacter.type === 'character'){
const da = await selectMultipleFile(['png', 'webp', 'mp4', 'mp3', 'gif'])
currentCharacter.additionalAssets = currentCharacter.additionalAssets ?? []
if(!da){
return
}
for(const f of da){
console.log(f)
const img = f.data
const name = f.name
const extension = name.split('.').pop().toLowerCase()
const imgp = await saveAsset(img,'',extension)
currentCharacter.additionalAssets.push([name, imgp, extension])
currentCharacter = currentCharacter
}
const db = get(DataBase);
setDatabase(db)
}
}}>
<PlusIcon />
</button>
{#if currentCharacter.additionalAssets}
{#each currentCharacter.additionalAssets as additionalAsset, i}
<button on:click={()=>{
onSelect(additionalAsset)
}}>
{#if assetFilePath[i]}
{#if assetFileExtensions[i] === 'mp4'}
<!-- svelte-ignore a11y-media-has-caption -->
<video class="w-16 h-16 m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
{:else if assetFileExtensions[i] === 'mp3'}
<div class='w-16 h-16 m-1 rounded-md bg-slate-500 flex flex-col justify-center items-center'>
<FileAudioIcon/>
<div class='w-16 px-1 text-ellipsis whitespace-nowrap overflow-hidden'>{additionalAsset[0]}</div>
</div>
<!-- <audio controls class="w-16 h-16 m-1 rounded-md"><source src={assetPath} type="audio/mpeg"></audio> -->
{:else}
<img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={additionalAsset[0]}/>
{/if}
{/if}
</button>
{/each}
{/if}
{/if}

39
src/lib/ChatScreens/Chat.svelte
View File

@@ -8,6 +8,7 @@
import { selectedCharID } from "../../ts/stores";
import { translate } from "../../ts/translator/translator";
import { replacePlaceholders } from "../../ts/util";
export let message = ''
export let name = ''
export let isLastMemory:boolean
@@ -17,6 +18,7 @@
export let onReroll = () => {}
export let unReroll = () => {}
export let character:character|groupChat|null = null
let md:string
let translating = false
let editMode = false
let statusMessage:string = ''
@@ -24,7 +26,8 @@
let msgDisplay = ''
let msgTranslated = ''
let translated = false;
let translated = false
async function rm(){
const rm = $DataBase.askRemoval ? await alertConfirm(language.removeChat) : true
if(rm){
@@ -53,7 +56,7 @@
$DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message = msg
}
async function displaya(message:string){
async function displaya(message:string, isStreaming:boolean = false){
if($DataBase.autoTranslate && $DataBase.translator !== ''){
if(msgTranslated==='')
msgDisplay = replacePlaceholders(message, name)
@@ -64,6 +67,12 @@
else{
msgDisplay = replacePlaceholders(message, name)
}
if(!md || !isStreaming || isStreaming && idx === $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message.length - 1) {
ParseMarkdown(msgDisplay, character, 'normal').then(mdNew=>{
md = mdNew
})
}
}
const setStatusMessage = (message:string, timeout:number = 0)=>{
@@ -74,7 +83,7 @@
}, timeout)
}
$: displaya(message)
$: displaya(message, $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].isStreaming)
</script>
<div class="flex max-w-full justify-center" class:bgc={isLastMemory}>
<div class="text-neutral-200 mt-1 ml-4 mr-4 mb-1 p-2 bg-transparent flex-grow border-t-gray-900 border-opacity-30 border-transparent flexium items-start max-w-full" >
@@ -161,19 +170,17 @@
</div>
{#if editMode}
<AutoresizeArea bind:value={message} />
{:else}
{#await ParseMarkdown(msgDisplay, character) then md}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<span class="text chat chattext prose prose-invert minw-0" on:click={() => {
if($DataBase.clickToEdit && idx > -1){
editMode = true
msgTranslated = ""
}
}}
style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
>{@html md}</span>
{/await}
{:else if md}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<span class="text chat chattext prose prose-invert minw-0" on:click={() => {
if($DataBase.clickToEdit && idx > -1){
editMode = true
msgTranslated = ""
}
}}
style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
>{@html md}</span>
{/if}
</span>

35
src/lib/ChatScreens/DefaultChatScreen.svelte
View File

@@ -1,9 +1,9 @@
<script lang="ts">
import Suggestion from './Suggestion.svelte';
import { DatabaseIcon, DicesIcon, LanguagesIcon, MenuIcon, MicOffIcon, PowerIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
import { DatabaseIcon, DicesIcon, LanguagesIcon, Laugh, MenuIcon, MicOffIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
import { selectedCharID } from "../../ts/stores";
import Chat from "./Chat.svelte";
import { DataBase, type Message } from "../../ts/storage/database";
import { DataBase, type Message, type character, type groupChat } from "../../ts/storage/database";
import { getCharImage } from "../../ts/characters";
import { doingChat, sendChat } from "../../ts/process/index";
import { findCharacterbyId, messageForm, sleep } from "../../ts/util";
@@ -17,6 +17,7 @@
import { stopTTS } from "src/ts/process/tts";
import MainMenu from '../UI/MainMenu.svelte';
import Help from '../Others/Help.svelte';
import AssetInput from './AssetInput.svelte';
let messageInput:string = ''
let messageInputTranslate:string = ''
@@ -28,6 +29,8 @@
let rerollid = -1
let lastCharId = -1
let doingChatInputTranslate = false
let currentCharacter:character|groupChat = $DataBase.characters[$selectedCharID]
let toggleStickers:boolean = false
async function send() {
let selectedChar = $selectedCharID
@@ -218,6 +221,10 @@
}
})
}
$: {
currentCharacter = $DataBase.characters[$selectedCharID]
}
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="w-full h-full" style={customStyle} on:click={() => {
@@ -234,6 +241,12 @@
}
}}>
<div class="flex items-end mt-2 mb-2 w-full">
{#if $DataBase.useChatSticker && currentCharacter.type !== 'group'}
<div on:click={()=>{toggleStickers = !toggleStickers}}
class={"ml-4 bg-gray-500 flex justify-center items-center w-12 h-12 rounded-md hover:bg-green-500 transition-colors "+(toggleStickers ? 'text-green-500':'text-white')}>
<Laugh/>
</div>
{/if}
<textarea class="text-neutral-200 p-2 min-w-0 bg-transparent input-text text-xl flex-grow ml-4 mr-2 border-gray-700 resize-none focus:bg-selected overflow-y-hidden overflow-x-hidden max-w-full"
bind:value={messageInput}
bind:this={inputEle}
@@ -299,9 +312,27 @@
</div>
{/if}
{#if toggleStickers}
<div class="ml-4 flex flex-wrap">
<AssetInput bind:currentCharacter={currentCharacter} onSelect={(additionalAsset)=>{
let fileType = 'img'
if(additionalAsset.length > 2 && additionalAsset[2]) {
const fileExtension = additionalAsset[2]
if(fileExtension === 'mp4' || fileExtension === 'webm')
fileType = 'video'
else if(fileExtension === 'mp3' || fileExtension === 'wav')
fileType = 'audio'
}
messageInput += `<span class='notranslate' translate='no'>{{${fileType}::${additionalAsset[0]}}}</span> *${additionalAsset[0]} added*`
updateInputSizeAll()
}}/>
</div>
{/if}
{#if $DataBase.useAutoSuggestions}
<Suggestion messageInput={(msg)=>messageInput=msg} {send}/>
{/if}
{#each messageForm($DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message, loadPages) as chat, i}
{#if chat.role === 'char'}
{#if $DataBase.characters[$selectedCharID].type !== 'group'}

10
src/lib/Setting/Pages/DisplaySettings.svelte
View File

@@ -141,4 +141,12 @@
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.useChatCopy} name={language.useChatCopy}/>
</div>
</div>
{#if $DataBase.useExperimental}
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.useChatSticker} name={language.useChatSticker}/>
<Help key="experimental" name={language.useChatSticker}/>
</div>
{/if}

35
src/lib/SideBars/CharConfig.svelte
View File

@@ -16,7 +16,7 @@
import RegexData from "./RegexData.svelte";
import { exportChar, shareRisuHub } from "src/ts/characterCards";
import { getElevenTTSVoices, getWebSpeechTTSVoices, getVOICEVOXVoices } from "src/ts/process/tts";
import { checkCharOrder } from "src/ts/storage/globalApi";
import { checkCharOrder, getFileSrc } from "src/ts/storage/globalApi";
import { addGroupChar, rmCharFromGroup } from "src/ts/process/group";
import HubUpload from "../UI/HubUpload.svelte";
@@ -95,7 +95,9 @@
currentChar = currentChar
})
let assetFileExtensions:string[] = []
let assetFilePath:string[] = []
$: {
if(database.characters[$selectedCharID].chaId === currentChar.data.chaId){
database.characters[$selectedCharID] = currentChar.data
@@ -106,6 +108,20 @@
emos = currentChar.data.emotionImages
DataBase.set(database)
loadTokenize(currentChar.data)
if(currentChar.type ==='character'){
if(currentChar.data.additionalAssets){
for(let i = 0; i < currentChar.data.additionalAssets.length; i++){
if(currentChar.data.additionalAssets[i].length > 2 && currentChar.data.additionalAssets[i][2]) {
assetFileExtensions[i] = currentChar.data.additionalAssets[i][2]
} else
assetFileExtensions[i] = currentChar.data.additionalAssets[i][1].split('.').pop()
getFileSrc(currentChar.data.additionalAssets[i][1]).then((filePath) => {
assetFilePath[i] = filePath
})
}
}
}
}
onDestroy(unsub);
@@ -615,9 +631,10 @@
for(const f of da){
console.log(f)
const img = f.data
const imgp = await saveAsset(img)
const name = f.name
currentChar.data.additionalAssets.push([name, imgp])
const extension = name.split('.').pop().toLowerCase()
const imgp = await saveAsset(img,'', extension)
currentChar.data.additionalAssets.push([name, imgp, extension])
currentChar.data.additionalAssets = currentChar.data.additionalAssets
}
}
@@ -634,6 +651,16 @@
{#each currentChar.data.additionalAssets as assets, i}
<tr>
<td class="font-medium truncate">
{#if assetFilePath[i]}
{#if assetFileExtensions[i] === 'mp4'}
<!-- svelte-ignore a11y-media-has-caption -->
<video controls class="mt-2 px-2 w-full m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
{:else if assetFileExtensions[i] === 'mp3'}
<audio controls class="mt-2 px-2 w-full h-16 m-1 rounded-md" loop><source src={assetFilePath[i]} type="audio/mpeg"></audio>
{:else}
<img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={assets[0]}/>
{/if}
{/if}
<input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected w-full resize-none" bind:value={currentChar.data.additionalAssets[i][0]} placeholder="..." />
</td>
<th class="font-medium cursor-pointer w-10">

11
src/ts/characterCards.ts
View File

@@ -329,7 +329,7 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
let customScripts:customscript[] = []
let utilityBot = false
let sdData = defaultSdDataFunc()
let extAssets:[string,string][] = []
let extAssets:[string,string,string][] = []
if(risuext){
if(risuext.emotions){
@@ -350,8 +350,11 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
msg: `Loading... (Getting Assets ${i} / ${risuext.additionalAssets.length})`
})
await sleep(10)
const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'))
extAssets.push([risuext.additionalAssets[i][0],imgp])
let fileName = ''
if(risuext.additionalAssets[i].length >= 3)
fileName = risuext.additionalAssets[i][2]
const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'), '', fileName)
extAssets.push([risuext.additionalAssets[i][0],imgp,fileName])
}
}
bias = risuext.bias ?? bias
@@ -756,7 +759,7 @@ type CharacterCardV2 = {
customScripts?:customscript[]
utilityBot?: boolean,
sdData?:[string,string][],
additionalAssets?:[string,string][],
additionalAssets?:[string,string,string][],
backgroundHTML?:string
}
}

6
src/ts/parser.ts
View File

@@ -64,9 +64,9 @@ export async function ParseMarkdown(data:string, char:(character | groupChat) =
for(const asset of char.additionalAssets){
const assetPath = await getFileSrc(asset[1])
data = data.replaceAll(`{{raw::${asset[0]}}}`, assetPath).
replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" />`)
.replaceAll(`{{video::${asset[0]}}}`,`<video autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
.replaceAll(`{{audio::${asset[0]}}}`,`<audio autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" alt="${asset[0]}"/>`)
.replaceAll(`{{video::${asset[0]}}}`,`<video controls autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
.replaceAll(`{{audio::${asset[0]}}}`,`<audio controls autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
if(mode === 'back'){
data = data.replaceAll(`{{bg::${asset[0]}}}`, `<div style="width:100%;height:100%;background: linear-gradient(rgba(0, 0, 0, 0.8), rgba(0, 0, 0, 0.8)),url(${assetPath}); background-size: cover;"></div>`)
}

3
src/ts/process/index.ts
View File

@@ -404,6 +404,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
else if(req.type === 'streaming'){
const reader = req.result.getReader()
const msgIndex = db.characters[selectedChar].chats[selectedChat].message.length
db.characters[selectedChar].chats[selectedChat].isStreaming = true
db.characters[selectedChar].chats[selectedChat].message.push({
role: 'char',
data: "",
@@ -419,6 +420,8 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
setDatabase(db)
}
if(readed.done){
db.characters[selectedChar].chats[selectedChat].isStreaming = false
setDatabase(db)
break
}
}

6
src/ts/storage/database.ts
View File

@@ -339,7 +339,7 @@ export interface character{
VOLUME_SCALE?: number
}
supaMemory?:boolean
additionalAssets?:[string, string][]
additionalAssets?:[string, string, string][]
ttsReadOnlyQuoted?:boolean
replaceGlobalNote:string
backgroundHTML?:string
@@ -522,7 +522,8 @@ export interface Database{
expires_in?: number
}
},
classicMaxWidth: boolean
classicMaxWidth: boolean,
useChatSticker:boolean,
}
interface hordeConfig{
@@ -561,6 +562,7 @@ export interface Chat{
supaMemoryData?:string
lastMemory?:string
suggestMessages?:string[]
isStreaming?:boolean
}
export interface Message{

14
src/ts/storage/globalApi.ts
View File

@@ -170,7 +170,7 @@ export async function readImage(data:string) {
}
}
export async function saveAsset(data:Uint8Array, customId:string = ''){
export async function saveAsset(data:Uint8Array, customId:string = '', fileName:string = ''){
let id = ''
if(customId !== ''){
id = customId
@@ -182,13 +182,17 @@ export async function saveAsset(data:Uint8Array, customId:string = ''){
id = uuidv4()
}
}
let fileExtension:string = 'png'
if(fileName && fileName.split('.').length > 0){
fileExtension = fileName.split('.').pop()
}
if(isTauri){
await writeBinaryFile(`assets/${id}.png`, data ,{dir: BaseDirectory.AppData})
return `assets/${id}.png`
await writeBinaryFile(`assets/${id}.${fileExtension}`, data ,{dir: BaseDirectory.AppData})
return `assets/${id}.${fileExtension}`
}
else{
await forageStorage.setItem(`assets/${id}.png`, data)
return `assets/${id}.png`
await forageStorage.setItem(`assets/${id}.${fileExtension}`, data)
return `assets/${id}.${fileExtension}`
}
}