mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] Add support for chat stickers (#178)

Browse Source 
# PR Checklist
- [x] Did you check if it works normally in all models? *ignore this
when it dosen't uses models*
- [x] Did you check if it works normally in all of web, local and node
hosted versions? if it dosen't, did you blocked it in those versions?
- [x] Did you added a type def?

# Description
- This commit adds support for chat stickers by allowing users to use
stickers in chat message window.
- The users can toggle show additional asset list using a button.
- Added Additional assets file extension data.
- Added option for additional assets preview.
- Optimized render when use streaming api. (prevent markdown again when
message not changed)
Added controls to Video/Audio Assets


[Video11.webm](https://github.com/kwaroran/RisuAI/assets/11344967/8980282f-d001-4afc-a6d9-6f6369b4cd4c)


[Video12.webm](https://github.com/kwaroran/RisuAI/assets/11344967/e0e153b8-eba6-48a5-b47d-d2dd287f32fd)
This commit is contained in:
kwaroran
2023-06-25 23:07:44 +09:00
committed by GitHub
parent 747ee3ea28 5adc992174
commit ca1a0afe99
13 changed files with 208 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/lang/cn.ts
View File

@@ -304,6 +304,8 @@ export const languageChinese = {
tags: "标签",
copied: "已复制",
useChatCopy: "使用聊天复制",
useChatSticker: "使用聊天贴纸",
useAdditionalAssetsPreview: "使用附加资源预览",
autoTranslateInput: "使用自动翻译输入",
enterMessageForTranslateToEnglish: "输入要翻译为英语的消息",
recent: '最新',

2
src/lang/en.ts
View File

@@ -308,6 +308,8 @@ export const languageEnglish = {
backgroundHTML: "Background Embedding",
copied: "Copied",
useChatCopy: "Use Chat Message Copy",
useChatSticker: "Use Chat Sticker",
useAdditionalAssetsPreview: "Use Additional Assets Preview",
autoTranslateInput: "Auto Translate Input",
enterMessageForTranslateToEnglish: "Enter Message for Translate to English",
recent: 'Recent',

2
src/lang/ko.ts
View File

@@ -280,6 +280,8 @@ export const languageKorean = {
backgroundHTML: "백그라운드 임베딩",
copied: "복사됨",
useChatCopy: "채팅 메시지 복사 사용",
useChatSticker: "채팅 스티커 사용",
useAdditionalAssetsPreview: "추가 에셋 미리보기 사용",
autoTranslateInput: "입력 자동 번역",
enterMessageForTranslateToEnglish: "영어로 번역할 메시지를 입력해주세요",
imageCompression: "이미지 압축",

75
src/lib/ChatScreens/AssetInput.svelte Normal file
View File

@@ -0,0 +1,75 @@
<script lang="ts">
import { get } from 'svelte/store';
import { FileAudioIcon, PlusIcon } from "lucide-svelte";
import { DataBase, setDatabase, type character, type groupChat } from "src/ts/storage/database";
import { getFileSrc, saveAsset } from "src/ts/storage/globalApi";
import { selectMultipleFile } from "src/ts/util";
export let currentCharacter:character|groupChat;
export let onSelect:(additionalAsset:[string,string,string])=>void;
let assetFileExtensions:string[] = []
let assetFilePath:string[] = []
$:{
if(currentCharacter.type ==='character'){
if(currentCharacter.additionalAssets){
for(let i = 0; i < currentCharacter.additionalAssets.length; i++){
// console.log('check content type ...', currentCharacter.additionalAssets[i][0], currentCharacter.additionalAssets[i][1]);
if(currentCharacter.additionalAssets[i].length > 2 && currentCharacter.additionalAssets[i][2]) {
assetFileExtensions[i] = currentCharacter.additionalAssets[i][2]
} else {
assetFileExtensions[i] = currentCharacter.additionalAssets[i][1].split('.').pop()
}
getFileSrc(currentCharacter.additionalAssets[i][1]).then((filePath) => {
assetFilePath[i] = filePath
})
}
}
}
}
</script>
{#if currentCharacter.type ==='character'}
<button class="hover:text-green-500 bg-gray-500 flex justify-center items-center w-16 h-16 m-1 rounded-md" on:click={async () => {
if(currentCharacter.type === 'character'){
const da = await selectMultipleFile(['png', 'webp', 'mp4', 'mp3', 'gif'])
currentCharacter.additionalAssets = currentCharacter.additionalAssets ?? []
if(!da){
return
}
for(const f of da){
console.log(f)
const img = f.data
const name = f.name
const extension = name.split('.').pop().toLowerCase()
const imgp = await saveAsset(img,'',extension)
currentCharacter.additionalAssets.push([name, imgp, extension])
currentCharacter = currentCharacter
}
const db = get(DataBase);
setDatabase(db)
}
}}>
<PlusIcon />
</button>
{#if currentCharacter.additionalAssets}
{#each currentCharacter.additionalAssets as additionalAsset, i}
<button on:click={()=>{
onSelect(additionalAsset)
}}>
{#if assetFilePath[i]}
{#if assetFileExtensions[i] === 'mp4'}
<!-- svelte-ignore a11y-media-has-caption -->
<video class="w-16 h-16 m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
{:else if assetFileExtensions[i] === 'mp3'}
<div class='w-16 h-16 m-1 rounded-md bg-slate-500 flex flex-col justify-center items-center'>
<FileAudioIcon/>
<div class='w-16 px-1 text-ellipsis whitespace-nowrap overflow-hidden'>{additionalAsset[0]}</div>
</div>
<!-- <audio controls class="w-16 h-16 m-1 rounded-md"><source src={assetPath} type="audio/mpeg"></audio> -->
{:else}
<img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={additionalAsset[0]}/>
{/if}
{/if}
</button>
{/each}
{/if}
{/if}

39
src/lib/ChatScreens/Chat.svelte
View File

@@ -8,6 +8,7 @@
import { selectedCharID } from "../../ts/stores";
import { translate } from "../../ts/translator/translator";
import { replacePlaceholders } from "../../ts/util";
export let message = ''
export let name = ''
export let isLastMemory:boolean
@@ -17,6 +18,7 @@
export let onReroll = () => {}
export let unReroll = () => {}
export let character:character|groupChat|null = null
let md:string
let translating = false
let editMode = false
let statusMessage:string = ''
@@ -24,7 +26,8 @@
let msgDisplay = ''
let msgTranslated = ''
let translated = false;
let translated = false
async function rm(){
const rm = $DataBase.askRemoval ? await alertConfirm(language.removeChat) : true
if(rm){
@@ -53,7 +56,7 @@
$DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message = msg
}
async function displaya(message:string){
async function displaya(message:string, isStreaming:boolean = false){
if($DataBase.autoTranslate && $DataBase.translator !== ''){
if(msgTranslated==='')
msgDisplay = replacePlaceholders(message, name)
@@ -64,6 +67,12 @@
else{
msgDisplay = replacePlaceholders(message, name)
}
if(!md || !isStreaming || isStreaming && idx === $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message.length - 1) {
ParseMarkdown(msgDisplay, character, 'normal').then(mdNew=>{
md = mdNew
})
}
}
const setStatusMessage = (message:string, timeout:number = 0)=>{
@@ -74,7 +83,7 @@
}, timeout)
}
$: displaya(message)
$: displaya(message, $DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].isStreaming)
</script>
<div class="flex max-w-full justify-center" class:bgc={isLastMemory}>
<div class="text-neutral-200 mt-1 ml-4 mr-4 mb-1 p-2 bg-transparent flex-grow border-t-gray-900 border-opacity-30 border-transparent flexium items-start max-w-full" >
@@ -161,19 +170,17 @@
</div>
{#if editMode}
<AutoresizeArea bind:value={message} />
{:else}
{#await ParseMarkdown(msgDisplay, character) then md}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<span class="text chat chattext prose prose-invert minw-0" on:click={() => {
if($DataBase.clickToEdit && idx > -1){
editMode = true
msgTranslated = ""
}
}}
style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
>{@html md}</span>
{/await}
{:else if md}
<!-- svelte-ignore a11y-click-events-have-key-events -->
<span class="text chat chattext prose prose-invert minw-0" on:click={() => {
if($DataBase.clickToEdit && idx > -1){
editMode = true
msgTranslated = ""
}
}}
style:font-size="{0.875 * ($DataBase.zoomsize / 100)}rem"
style:line-height="{1.25 * ($DataBase.zoomsize / 100)}rem"
>{@html md}</span>
{/if}
</span>

35
src/lib/ChatScreens/DefaultChatScreen.svelte
View File

@@ -1,9 +1,9 @@
<script lang="ts">
import Suggestion from './Suggestion.svelte';
import { DatabaseIcon, DicesIcon, LanguagesIcon, MenuIcon, MicOffIcon, PowerIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
import { DatabaseIcon, DicesIcon, LanguagesIcon, Laugh, MenuIcon, MicOffIcon, RefreshCcwIcon, ReplyIcon, Send } from "lucide-svelte";
import { selectedCharID } from "../../ts/stores";
import Chat from "./Chat.svelte";
import { DataBase, type Message } from "../../ts/storage/database";
import { DataBase, type Message, type character, type groupChat } from "../../ts/storage/database";
import { getCharImage } from "../../ts/characters";
import { doingChat, sendChat } from "../../ts/process/index";
import { findCharacterbyId, messageForm, sleep } from "../../ts/util";
@@ -17,6 +17,7 @@
import { stopTTS } from "src/ts/process/tts";
import MainMenu from '../UI/MainMenu.svelte';
import Help from '../Others/Help.svelte';
import AssetInput from './AssetInput.svelte';
let messageInput:string = ''
let messageInputTranslate:string = ''
@@ -28,6 +29,8 @@
let rerollid = -1
let lastCharId = -1
let doingChatInputTranslate = false
let currentCharacter:character|groupChat = $DataBase.characters[$selectedCharID]
let toggleStickers:boolean = false
async function send() {
let selectedChar = $selectedCharID
@@ -218,6 +221,10 @@
}
})
}
$: {
currentCharacter = $DataBase.characters[$selectedCharID]
}
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="w-full h-full" style={customStyle} on:click={() => {
@@ -234,6 +241,12 @@
}
}}>
<div class="flex items-end mt-2 mb-2 w-full">
{#if $DataBase.useChatSticker && currentCharacter.type !== 'group'}
<div on:click={()=>{toggleStickers = !toggleStickers}}
class={"ml-4 bg-gray-500 flex justify-center items-center w-12 h-12 rounded-md hover:bg-green-500 transition-colors "+(toggleStickers ? 'text-green-500':'text-white')}>
<Laugh/>
</div>
{/if}
<textarea class="text-neutral-200 p-2 min-w-0 bg-transparent input-text text-xl flex-grow ml-4 mr-2 border-gray-700 resize-none focus:bg-selected overflow-y-hidden overflow-x-hidden max-w-full"
bind:value={messageInput}
bind:this={inputEle}
@@ -299,9 +312,27 @@
</div>
{/if}
{#if toggleStickers}
<div class="ml-4 flex flex-wrap">
<AssetInput bind:currentCharacter={currentCharacter} onSelect={(additionalAsset)=>{
let fileType = 'img'
if(additionalAsset.length > 2 && additionalAsset[2]) {
const fileExtension = additionalAsset[2]
if(fileExtension === 'mp4' || fileExtension === 'webm')
fileType = 'video'
else if(fileExtension === 'mp3' || fileExtension === 'wav')
fileType = 'audio'
}
messageInput += `<span class='notranslate' translate='no'>{{${fileType}::${additionalAsset[0]}}}</span> *${additionalAsset[0]} added*`
updateInputSizeAll()
}}/>
</div>
{/if}
{#if $DataBase.useAutoSuggestions}
<Suggestion messageInput={(msg)=>messageInput=msg} {send}/>
{/if}
{#each messageForm($DataBase.characters[$selectedCharID].chats[$DataBase.characters[$selectedCharID].chatPage].message, loadPages) as chat, i}
{#if chat.role === 'char'}
{#if $DataBase.characters[$selectedCharID].type !== 'group'}

11
src/lib/Setting/Pages/DisplaySettings.svelte
View File

@@ -142,3 +142,14 @@
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.useChatCopy} name={language.useChatCopy}/>
</div>
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.useAdditionalAssetsPreview} name={language.useAdditionalAssetsPreview}/>
</div>
{#if $DataBase.useExperimental}
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.useChatSticker} name={language.useChatSticker}/>
<Help key="experimental" name={language.useChatSticker}/>
</div>
{/if}

34
src/lib/SideBars/CharConfig.svelte
View File

@@ -16,7 +16,7 @@
import RegexData from "./RegexData.svelte";
import { exportChar, shareRisuHub } from "src/ts/characterCards";
import { getElevenTTSVoices, getWebSpeechTTSVoices, getVOICEVOXVoices } from "src/ts/process/tts";
import { checkCharOrder } from "src/ts/storage/globalApi";
import { checkCharOrder, getFileSrc } from "src/ts/storage/globalApi";
import { addGroupChar, rmCharFromGroup } from "src/ts/process/group";
import HubUpload from "../UI/HubUpload.svelte";
@@ -95,6 +95,8 @@
currentChar = currentChar
})
let assetFileExtensions:string[] = []
let assetFilePath:string[] = []
$: {
if(database.characters[$selectedCharID].chaId === currentChar.data.chaId){
@@ -106,6 +108,20 @@
emos = currentChar.data.emotionImages
DataBase.set(database)
loadTokenize(currentChar.data)
if(currentChar.type ==='character' && database.useAdditionalAssetsPreview){
if(currentChar.data.additionalAssets){
for(let i = 0; i < currentChar.data.additionalAssets.length; i++){
if(currentChar.data.additionalAssets[i].length > 2 && currentChar.data.additionalAssets[i][2]) {
assetFileExtensions[i] = currentChar.data.additionalAssets[i][2]
} else
assetFileExtensions[i] = currentChar.data.additionalAssets[i][1].split('.').pop()
getFileSrc(currentChar.data.additionalAssets[i][1]).then((filePath) => {
assetFilePath[i] = filePath
})
}
}
}
}
onDestroy(unsub);
@@ -615,9 +631,10 @@
for(const f of da){
console.log(f)
const img = f.data
const imgp = await saveAsset(img)
const name = f.name
currentChar.data.additionalAssets.push([name, imgp])
const extension = name.split('.').pop().toLowerCase()
const imgp = await saveAsset(img,'', extension)
currentChar.data.additionalAssets.push([name, imgp, extension])
currentChar.data.additionalAssets = currentChar.data.additionalAssets
}
}
@@ -634,8 +651,19 @@
{#each currentChar.data.additionalAssets as assets, i}
<tr>
<td class="font-medium truncate">
{#if assetFilePath[i] && database.useAdditionalAssetsPreview}
{#if assetFileExtensions[i] === 'mp4'}
<!-- svelte-ignore a11y-media-has-caption -->
<video controls class="mt-2 px-2 w-full m-1 rounded-md"><source src={assetFilePath[i]} type="video/mp4"></video>
{:else if assetFileExtensions[i] === 'mp3'}
<audio controls class="mt-2 px-2 w-full h-16 m-1 rounded-md" loop><source src={assetFilePath[i]} type="audio/mpeg"></audio>
{:else}
<img src={assetFilePath[i]} class="w-16 h-16 m-1 rounded-md" alt={assets[0]}/>
{/if}
{/if}
<input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected w-full resize-none" bind:value={currentChar.data.additionalAssets[i][0]} placeholder="..." />
</td>
<th class="font-medium cursor-pointer w-10">
<button class="hover:text-green-500" on:click={() => {
if(currentChar.type === 'character'){

11
src/ts/characterCards.ts
View File

@@ -329,7 +329,7 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
let customScripts:customscript[] = []
let utilityBot = false
let sdData = defaultSdDataFunc()
let extAssets:[string,string][] = []
let extAssets:[string,string,string][] = []
if(risuext){
if(risuext.emotions){
@@ -350,8 +350,11 @@ async function importSpecv2(card:CharacterCardV2, img?:Uint8Array, mode?:'hub'|'
msg: `Loading... (Getting Assets ${i} / ${risuext.additionalAssets.length})`
})
await sleep(10)
const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'))
extAssets.push([risuext.additionalAssets[i][0],imgp])
let fileName = ''
if(risuext.additionalAssets[i].length >= 3)
fileName = risuext.additionalAssets[i][2]
const imgp = await saveAsset(mode === 'hub' ? (await getHubResources(risuext.additionalAssets[i][1])) :Buffer.from(risuext.additionalAssets[i][1], 'base64'), '', fileName)
extAssets.push([risuext.additionalAssets[i][0],imgp,fileName])
}
}
bias = risuext.bias ?? bias
@@ -756,7 +759,7 @@ type CharacterCardV2 = {
customScripts?:customscript[]
utilityBot?: boolean,
sdData?:[string,string][],
additionalAssets?:[string,string][],
additionalAssets?:[string,string,string][],
backgroundHTML?:string
}
}

6
src/ts/parser.ts
View File

@@ -63,9 +63,9 @@ async function parseAdditionalAssets(data:string, char:character, mode:'normal'|
for(const asset of char.additionalAssets){
const assetPath = await getFileSrc(asset[1])
data = data.replaceAll(`{{raw::${asset[0]}}}`, assetPath).
replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" />`)
.replaceAll(`{{video::${asset[0]}}}`,`<video autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
.replaceAll(`{{audio::${asset[0]}}}`,`<audio autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
replaceAll(`{{img::${asset[0]}}}`,`<img src="${assetPath}" alt="${asset[0]}"/>`)
.replaceAll(`{{video::${asset[0]}}}`,`<video controls autoplay loop><source src="${assetPath}" type="video/mp4"></video>`)
.replaceAll(`{{audio::${asset[0]}}}`,`<audio controls autoplay loop><source src="${assetPath}" type="audio/mpeg"></audio>`)
if(mode === 'back'){
data = data.replaceAll(`{{bg::${asset[0]}}}`, `<div style="width:100%;height:100%;background: linear-gradient(rgba(0, 0, 0, 0.8), rgba(0, 0, 0, 0.8)),url(${assetPath}); background-size: cover;"></div>`)
}

3
src/ts/process/index.ts
View File

@@ -409,6 +409,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
else if(req.type === 'streaming'){
const reader = req.result.getReader()
const msgIndex = db.characters[selectedChar].chats[selectedChat].message.length
db.characters[selectedChar].chats[selectedChat].isStreaming = true
db.characters[selectedChar].chats[selectedChat].message.push({
role: 'char',
data: "",
@@ -424,6 +425,8 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
setDatabase(db)
}
if(readed.done){
db.characters[selectedChar].chats[selectedChat].isStreaming = false
setDatabase(db)
break
}
}

7
src/ts/storage/database.ts
View File

@@ -342,7 +342,7 @@ export interface character{
VOLUME_SCALE?: number
}
supaMemory?:boolean
additionalAssets?:[string, string][]
additionalAssets?:[string, string, string][]
ttsReadOnlyQuoted?:boolean
replaceGlobalNote:string
backgroundHTML?:string
@@ -525,7 +525,9 @@ export interface Database{
expires_in?: number
}
},
classicMaxWidth: boolean
classicMaxWidth: boolean,
useChatSticker:boolean,
useAdditionalAssetsPreview:boolean,
}
interface hordeConfig{
@@ -564,6 +566,7 @@ export interface Chat{
supaMemoryData?:string
lastMemory?:string
suggestMessages?:string[]
isStreaming?:boolean
}
export interface Message{

14
src/ts/storage/globalApi.ts
View File

@@ -170,7 +170,7 @@ export async function readImage(data:string) {
}
}
export async function saveAsset(data:Uint8Array, customId:string = ''){
export async function saveAsset(data:Uint8Array, customId:string = '', fileName:string = ''){
let id = ''
if(customId !== ''){
id = customId
@@ -182,13 +182,17 @@ export async function saveAsset(data:Uint8Array, customId:string = ''){
id = uuidv4()
}
}
let fileExtension:string = 'png'
if(fileName && fileName.split('.').length > 0){
fileExtension = fileName.split('.').pop()
}
if(isTauri){
await writeBinaryFile(`assets/${id}.png`, data ,{dir: BaseDirectory.AppData})
return `assets/${id}.png`
await writeBinaryFile(`assets/${id}.${fileExtension}`, data ,{dir: BaseDirectory.AppData})
return `assets/${id}.${fileExtension}`
}
else{
await forageStorage.setItem(`assets/${id}.png`, data)
return `assets/${id}.png`
await forageStorage.setItem(`assets/${id}.${fileExtension}`, data)
return `assets/${id}.${fileExtension}`
}
}