Merge branch 'main' of https://github.com/kwaroran/RisuAI

Signed-off-by: hashcoko <hashcoko@gmail.com>

package.json

@@ -15,80 +15,80 @@
   },
   "dependencies": {
     "@adobe/css-tools": "4.3.1",
-    "@aws-crypto/sha256-js": "^5.1.0",
+    "@aws-crypto/sha256-js": "^5.2.0",
     "@dqbd/tiktoken": "^1.0.7",
-    "@mlc-ai/web-tokenizers": "^0.1.0",
+    "@mlc-ai/web-tokenizers": "^0.1.2",
-    "@smithy/protocol-http": "^3.0.7",
+    "@smithy/protocol-http": "^3.0.10",
-    "@smithy/signature-v4": "^2.0.11",
+    "@smithy/signature-v4": "^2.0.16",
     "@tauri-apps/api": "1.4.0",
-    "@types/marked": "^5.0.1",
+    "@types/marked": "^5.0.2",
-    "@xenova/transformers": "^2.5.0",
+    "@xenova/transformers": "^2.9.0",
     "blueimp-md5": "^2.19.0",
     "body-parser": "^1.20.2",
     "buffer": "^6.0.3",
-    "core-js": "^3.31.1",
+    "core-js": "^3.33.3",
     "cors": "^2.8.5",
-    "dompurify": "^3.0.5",
+    "dompurify": "^3.0.6",
     "exifr": "^7.1.3",
     "express": "^4.18.2",
-    "fflate": "^0.8.0",
+    "fflate": "^0.8.1",
     "gpt-3-encoder": "^1.1.4",
     "gpt3-tokenizer": "^1.1.5",
     "html-to-image": "^1.11.11",
-    "isomorphic-dompurify": "^1.8.0",
+    "isomorphic-dompurify": "^1.9.0",
     "jszip": "^3.10.1",
-    "libsodium-wrappers-sumo": "^0.7.11",
+    "libsodium-wrappers-sumo": "^0.7.13",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
     "lucide-svelte": "^0.292.0",
-    "marked": "^5.1.1",
+    "marked": "^5.1.2",
     "ml-distance": "^4.0.1",
     "mobile-drag-drop": "3.0.0-rc.0",
-    "msgpackr": "^1.9.5",
+    "msgpackr": "^1.9.9",
-    "node-html-parser": "^6.1.5",
+    "node-html-parser": "^6.1.11",
     "peerjs": "^1.5.1",
     "png-chunk-text": "^1.0.0",
     "png-chunks-encode": "^1.0.0",
     "png-chunks-extract": "^1.0.0",
     "pngjs": "^7.0.0",
-    "rollup": "^3.26.3",
+    "rollup": "^3.29.4",
     "showdown": "^2.1.0",
     "sortablejs": "^1.15.0",
     "three": "^0.154.0",
     "tippy.js": "^6.3.7",
-    "uuid": "^9.0.0",
+    "uuid": "^9.0.1",
-    "wasmoon": "^1.15.0",
+    "wasmoon": "^1.15.1",
     "web-streams-polyfill": "^3.2.1",
     "yuso": "^0.1.3"
   },
   "devDependencies": {
-    "@sveltejs/vite-plugin-svelte": "^2.4.2",
+    "@sveltejs/vite-plugin-svelte": "^2.5.3",
-    "@tailwindcss/typography": "^0.5.9",
+    "@tailwindcss/typography": "^0.5.10",
     "@tauri-apps/cli": "1.4.0",
     "@tsconfig/svelte": "^3.0.0",
-    "@types/blueimp-md5": "^2.18.0",
+    "@types/blueimp-md5": "^2.18.2",
-    "@types/dompurify": "^3.0.2",
+    "@types/dompurify": "^3.0.5",
-    "@types/libsodium-wrappers-sumo": "^0.7.5",
+    "@types/libsodium-wrappers-sumo": "^0.7.8",
-    "@types/lodash": "^4.14.195",
+    "@types/lodash": "^4.14.202",
-    "@types/lodash.clonedeep": "^4.5.7",
+    "@types/lodash.clonedeep": "^4.5.9",
-    "@types/lodash.isequal": "^4.5.6",
+    "@types/lodash.isequal": "^4.5.8",
-    "@types/node": "^18.16.19",
+    "@types/node": "^18.18.13",
-    "@types/pngjs": "^6.0.1",
+    "@types/pngjs": "^6.0.4",
-    "@types/showdown": "^2.0.1",
+    "@types/showdown": "^2.0.6",
-    "@types/sortablejs": "^1.15.1",
+    "@types/sortablejs": "^1.15.7",
     "@types/three": "^0.154.0",
-    "@types/uuid": "^9.0.2",
+    "@types/uuid": "^9.0.7",
-    "@types/wicg-file-system-access": "^2020.9.6",
+    "@types/wicg-file-system-access": "^2020.9.8",
-    "autoprefixer": "^10.4.14",
+    "autoprefixer": "^10.4.16",
     "internal-ip": "^7.0.0",
-    "postcss": "^8.4.26",
+    "postcss": "^8.4.31",
-    "svelte": "^4.1.0",
+    "svelte": "^4.2.7",
-    "svelte-check": "^3.4.6",
+    "svelte-check": "^3.6.2",
-    "svelte-preprocess": "^5.0.4",
+    "svelte-preprocess": "^5.1.1",
-    "tailwindcss": "^3.3.3",
+    "tailwindcss": "^3.3.5",
-    "tslib": "^2.6.0",
+    "tslib": "^2.6.2",
-    "typescript": "^5.1.6",
+    "typescript": "^5.3.2",
-    "vite": "^4.4.5",
+    "vite": "^4.5.0",
     "vite-plugin-top-level-await": "^1.3.1",
     "vite-plugin-wasm": "^3.2.2"
   }

src-tauri/tauri.conf.json

@@ -8,7 +8,7 @@
   },
   "package": {
     "productName": "RisuAI",
-    "version": "1.60.1"
+    "version": "1.61.0"
   },
   "tauri": {
     "allowlist": {

src/lang/en.ts

@@ -450,6 +450,6 @@ export const languageEnglish = {
     imgGenPrompt: "Image Generation Prompt",
     imgGenNegatives: "Image Generation Negative Prompt",
     imgGenInstructions: "Image Generation Instructions",
-    usePlainFetchWarn: "Please disable this option when using NovelAI, as it can cause CORS errors when using NovelAI."
+    usePlainFetchWarn: "Please disable this option when using NovelAI, as it can cause CORS errors when using NovelAI.",
-
+    translationPrompt: "Translation Prompt",
 }

src/lib/ChatScreens/Chat.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-    import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon } from "lucide-svelte";
+    import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon, Volume2Icon } from "lucide-svelte";
     import { ParseMarkdown, type simpleCharacterArgument } from "../../ts/parser";
     import AutoresizeArea from "../UI/GUI/TextAreaResizable.svelte";
     import { alertConfirm, alertError } from "../../ts/alert";
@@ -10,6 +10,7 @@
     import { risuChatParser } from "src/ts/process/scripts";
     import { get } from "svelte/store";
   import { isEqual } from "lodash";
+  import { sayTTS } from "src/ts/process/tts";
     export let message = ''
     export let name = ''
     export let largePortrait = false
@@ -145,6 +146,12 @@
                         </button>    
                     {/if}
                     {#if idx > -1}
+                        <button class="ml-2 hover:text-green-500 transition-colors" on:click={()=>{
+                            return sayTTS(null, message)
+                        }}>
+                            <Volume2Icon size={20}/>
+                        </button>
+
                         <button class={"ml-2 hover:text-green-500 transition-colors "+(editMode?'text-green-400':'')} on:click={() => {
                             if(!editMode){
                                 editMode = true

src/lib/ChatScreens/DefaultChatScreen.svelte

@@ -8,7 +8,7 @@
     import { doingChat, sendChat } from "../../ts/process/index";
     import { findCharacterbyId, messageForm, sleep } from "../../ts/util";
     import { language } from "../../lang";
-    import { translate } from "../../ts/translator/translator";
+    import { isExpTranslator, translate } from "../../ts/translator/translator";
     import { alertError, alertNormal, alertWait } from "../../ts/alert";
     import sendSound from '../../etc/send.mp3'
     import {cloneDeep} from 'lodash'
@@ -251,7 +251,33 @@
 
     $: updateInputSizeAll()
 
-    function updateInputTransateMessage(reverse: boolean) {
+    async function updateInputTransateMessage(reverse: boolean) {
+        if(isExpTranslator()){
+            if(!reverse){
+                messageInputTranslate = ''
+                return
+            }
+            if(messageInputTranslate === '') {
+                messageInput = ''
+                return
+            }
+            const lastMessageInputTranslate = messageInputTranslate
+            await sleep(1500)
+            if(lastMessageInputTranslate === messageInputTranslate){
+                console.log(lastMessageInputTranslate === messageInputTranslate)
+                console.log(lastMessageInputTranslate, messageInputTranslate)
+                translate(reverse ? messageInputTranslate : messageInput, reverse).then((translatedMessage) => {
+                    if(translatedMessage){
+                        if(reverse)
+                            messageInput = translatedMessage
+                        else
+                            messageInputTranslate = translatedMessage
+                    }
+                })
+            }
+            return
+
+        }
         if(reverse && messageInputTranslate === '') {
             messageInput = ''
             return

src/lib/Setting/Pages/BotSettings.svelte

@@ -149,6 +149,7 @@
         <OptionInput value="gpt35_0613">GPT-3.5 0613</OptionInput>
         <OptionInput value="gpt4_0613">GPT-4 0613</OptionInput>
         <OptionInput value="claude-2.1">claude-2.1</OptionInput>
+        <OptionInput value="claude-2.0">claude-2.0</OptionInput>
         <OptionInput value="claude-2">claude-2</OptionInput>
         <OptionInput value="claude-v1.3">claude-v1.3</OptionInput>
         <OptionInput value="claude-v1.3-100k">claude-v1.3-100k</OptionInput>

src/lib/Setting/Pages/LanguageSettings.svelte

@@ -9,6 +9,7 @@
     import { downloadFile } from "src/ts/storage/globalApi";
     import { languageEnglish } from "src/lang/en";
     import TextInput from "src/lib/UI/GUI/TextInput.svelte";
+    import TextAreaInput from "src/lib/UI/GUI/TextAreaInput.svelte";
     let langChanged = false
 
 </script>
@@ -56,7 +57,7 @@
     <SelectInput className="mt-2 mb-4" bind:value={$DataBase.translatorType}>
         <OptionInput value="google" >Google</OptionInput>
         <OptionInput value="deepl" >DeepL</OptionInput>
-        <OptionInput value="submodel" >Use submodel</OptionInput>
+        <OptionInput value="llm" >Ax. Model</OptionInput>
     </SelectInput>
 
     {#if $DataBase.translatorType === 'deepl'}
@@ -75,6 +76,11 @@
         <span class="text-draculared text-xs mb-2">Translation sends a lot of requests at once, so don't use it at reverse proxy.</span>
     {/if}
 
+    {#if $DataBase.translatorType === 'llm'}
+        <span class="text-textcolor mt-4">{language.translationPrompt}</span>
+        <TextAreaInput bind:value={$DataBase.translatorPrompt} placeholder={"You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation."}/>
+    {/if}
+
 
     <div class="flex items-center mt-2">
         <Check bind:check={$DataBase.autoTranslate} name={language.autoTranslation}/>

src/lib/Setting/Pages/OtherBotSettings.svelte

@@ -146,10 +146,15 @@
     <span class="text-textcolor mt-2">VOICEVOX URL</span>
     <TextInput size="sm" marginBottom bind:value={$DataBase.voicevoxUrl}/>
     
+    <span class="text-textcolor">OpenAI Key</span>
+    <TextInput size="sm" marginBottom bind:value={$DataBase.openAIKey}/>
+
     <span class="text-textcolor mt-2">NovelAI API key</span>
     <TextInput size="sm" marginBottom placeholder="pst-..." bind:value={$DataBase.NAIApiKey}/>
     
-    
+    <span class="text-textcolor">Huggingface Key</span>
+    <TextInput size="sm" marginBottom bind:value={$DataBase.huggingfaceKey} placeholder="hf_..."/>
+
 </Arcodion>
 
 <Arcodion name={language.emotionImage} styled>

src/lib/SideBars/CharConfig.svelte

@@ -519,6 +519,7 @@
             <OptionInput value="VOICEVOX">VOICEVOX</OptionInput>
             <OptionInput value="openai">OpenAI</OptionInput>
             <OptionInput value="novelai">NovelAI</OptionInput>
+            <OptionInput value="huggingface">Huggingface</OptionInput>
         </SelectInput>
         
 
@@ -604,7 +605,6 @@
             </SelectInput>
         {/if}
         {#if currentChar.data.ttsMode === 'openai'}
-        <span class="text-textcolor">OpenAI TTS uses your OpenAI key on the chat model section</span>
             <SelectInput className="mb-4 mt-2" bind:value={currentChar.data.oaiVoice}>
                 <OptionInput value="">Unset</OptionInput>
                 {#each oaiVoices as voice}
@@ -612,7 +612,19 @@
                 {/each}
             </SelectInput>
         {/if}
-        {#if currentChar.data.ttsMode === 'webspeech' || currentChar.data.ttsMode === 'elevenlab' || currentChar.data.ttsMode === 'VOICEVOX' || currentChar.data.ttsMode === 'novelai'}
+        {#if currentChar.data.ttsMode === 'huggingface'}
+            <span class="text-textcolor">Model</span>
+            <TextInput additionalClass="mb-4 mt-2" bind:value={currentChar.data.hfTTS.model} />
+
+            <span class="text-textcolor">Language</span>
+            <TextInput additionalClass="mb-4 mt-2" bind:value={currentChar.data.hfTTS.language} placeholder="en" />
+        {/if}
+        {#if currentChar.data.ttsMode === 'webspeech' ||
+            currentChar.data.ttsMode === 'elevenlab' ||
+            currentChar.data.ttsMode === 'VOICEVOX' ||
+            currentChar.data.ttsMode === 'huggingface' ||
+            currentChar.data.ttsMode === 'openai' ||
+            currentChar.data.ttsMode === 'novelai'}
             <div class="flex items-center mt-2">
                 <Check bind:check={currentChar.data.ttsReadOnlyQuoted} name={language.ttsReadOnlyQuoted}/>
             </div>

src/ts/characters.ts

@@ -301,6 +301,10 @@ export function characterFormatUpdate(index:number|character){
             depth: 0,
             prompt: ''
         }
+        cha.hfTTS ??= {
+            model: '',
+            language: 'en'
+        }
         if(!cha.newGenData){
             cha = updateInlayScreen(cha)
         }

src/ts/hotkey.ts

@@ -70,7 +70,7 @@ function changeToPreset(num:number){
         let db = get(DataBase)
         let pres = db.botPresets
         if(pres.length > num){
-            alertToast(`Changed to Preset ${num+1} || Preset name : ${pres[num].name}`)
+            alertToast(`Changed to Preset: ${pres[num].name}`)
             changeToPreset2(num)
         }
     }

src/ts/plugins/embedscript.ts

@@ -1,5 +1,6 @@
 import { get } from 'svelte/store'
 import type { ScriptMode } from '../process/scripts'
+//@ts-ignore
 import WorkerUrl from './embedworker?worker&url'
 import { DataBase, type Chat, type character, type Message } from '../storage/database'
 import { selectedCharID } from '../stores'

src/ts/process/embedding/transformers.ts

@@ -52,4 +52,11 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
     let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
     let result = await extractor(text, { pooling: 'mean', normalize: true });
     return result?.data ?? null;
+}
+
+export const runTTS = async (text: string) => {
+    let speaker_embeddings = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
+    let synthesizer = await pipeline('text-to-speech', 'Xenova/speecht5_tts', { local_files_only: true });
+    let out = await synthesizer(text, { speaker_embeddings });
+    return out
 }

src/ts/process/tts.ts

@@ -1,13 +1,21 @@
 import { get } from "svelte/store";
 import { alertError } from "../alert";
 import { DataBase, type character } from "../storage/database";
-import { translateVox } from "../translator/translator";
+import { runTranslator, translateVox } from "../translator/translator";
 import { globalFetch } from "../storage/globalApi";
 import { language } from "src/lang";
+import { getCurrentCharacter, sleep } from "../util";
 
 let sourceNode:AudioBufferSourceNode = null
 
 export async function sayTTS(character:character,text:string) {
+    if(!character){
+        const v = getCurrentCharacter()
+        if(v.type === 'group'){
+            return
+        }
+        character = v
+    }
 
     let db = get(DataBase)
     text = text.replace(/\*/g,'')
@@ -162,6 +170,48 @@ export async function sayTTS(character:character,text:string) {
             }
             break;
         }
+        case 'huggingface': {
+            while(true){
+                if(character.hfTTS.language !== 'en'){
+                    text = await runTranslator(text, false, 'en', character.hfTTS.language)
+                }
+                const audioContext = new AudioContext();
+                const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
+                    method: 'POST',
+                    headers: {
+                        "Authorization": "Bearer " + db.huggingfaceKey,
+                        "Content-Type": "application/json",
+                    },
+                    body: JSON.stringify({
+                        inputs: text,
+                    })
+                });
+    
+                if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
+                    const json = await response.json()
+                    if(json.estimated_time){
+                        await sleep(json.estimated_time * 1000)
+                        continue
+                    }
+                }
+                else if(response.status >= 400){
+                    alertError(language.errors.httpError + `${await response.text()}`)
+                    return
+                }
+                else if (response.status === 200) {
+                    const audioBuffer = await response.arrayBuffer();
+                    audioContext.decodeAudioData(audioBuffer, (decodedData) => {
+                        const sourceNode = audioContext.createBufferSource();
+                        sourceNode.buffer = decodedData;
+                        sourceNode.connect(audioContext.destination);
+                        sourceNode.start();
+                    });
+                } else {
+                    alertError("Error fetching or decoding audio data");
+                }
+                return
+            }
+        }
     }
 }
 

src/ts/storage/database.ts

@@ -15,7 +15,7 @@ import type { OobaChatCompletionRequestParams } from '../model/ooba';
 
 export const DataBase = writable({} as any as Database)
 export const loadedStore = writable(false)
-export let appVer = "1.60.1"
+export let appVer = "1.61.0"
 export let webAppSubVer = ''
 
 export function setDatabase(data:Database){
@@ -350,6 +350,7 @@ export function setDatabase(data:Database){
     data.generationSeed ??= -1
     data.newOAIHandle ??= true
     data.gptVisionQuality ??= 'low'
+    data.huggingfaceKey ??= ''
     data.reverseProxyOobaArgs ??= {
         mode: 'instruct'
     }
@@ -522,7 +523,7 @@ export interface Database{
     mancerHeader:string
     emotionProcesser:'submodel'|'embedding',
     showMenuChatList?:boolean,
-    translatorType:'google'|'deepl'|'submodel'|'none',
+    translatorType:'google'|'deepl'|'none'|'llm',
     NAIadventure?:boolean,
     NAIappendName?:boolean,
     deeplOptions:{
@@ -542,8 +543,9 @@ export interface Database{
     reverseProxyOobaArgs: OobaChatCompletionRequestParams
     tpo?:boolean
     automark?:boolean
-
+    huggingfaceKey:string
     allowAllExtentionFiles?:boolean
+    translatorPrompt:string
 }
 
 export interface customscript{
@@ -652,6 +654,10 @@ export interface character{
     largePortrait?:boolean
     lorePlus?:boolean
     inlayViewScreen?:boolean
+    hfTTS?: {
+        model: string
+        language: string
+    }
 }
 
 

src/ts/translator/translator.ts

@@ -3,8 +3,8 @@ import { translatorPlugin } from "../plugins/plugins"
 import { DataBase } from "../storage/database"
 import { globalFetch } from "../storage/globalApi"
 import { alertError } from "../alert"
-import type { OpenAIChat } from "../process"
 import { requestChatData } from "../process/request"
+import { doingChat } from "../process"
 
 let cache={
     origin: [''],
@@ -35,7 +35,7 @@ export async function translate(text:string, reverse:boolean) {
     return runTranslator(text, reverse, db.translator,db.aiModel.startsWith('novellist') ? 'ja' : 'en')
 }
 
-async function runTranslator(text:string, reverse:boolean, from:string,target:'en'|'ja') {
+export async function runTranslator(text:string, reverse:boolean, from:string,target:string) {
     const arg = {
 
         from: reverse ? from : target,
@@ -100,19 +100,23 @@ async function runTranslator(text:string, reverse:boolean, from:string,target:'e
 
 async function translateMain(text:string, arg:{from:string, to:string, host:string}){
     let db = get(DataBase)
-
+    if(db.translatorType === 'llm'){
+        const tr = db.translator || 'en'
+        return translateLLM(text, {to: tr})
+    }
     if(db.translatorType === 'deepl'){
-        //deepl raise error 525 because of cloudflare
+        const body = {
+            text: [text],
+            source_lang: arg.from.toLocaleUpperCase(),
+            target_lang: arg.to.toLocaleUpperCase(),
+        }
         let url = db.deeplOptions.freeApi ? "https://api-free.deepl.com/v2/translate" : "https://api.deepl.com/v2/translate"
         const f = await globalFetch(url, {
             headers: {
                 "Authorization": "DeepL-Auth-Key " + db.deeplOptions.key,
+                "Content-Type": "application/json"
             },
-            body: {
+            body: body
-                text: text,
-                source_lang: arg.from.toLocaleUpperCase(),
-                target_lang: arg.to.toLocaleUpperCase(),
-            }
         })
 
         if(!f.ok){
@@ -197,9 +201,23 @@ async function jaTrans(text:string) {
     return await runTranslator(text, true, 'en','ja')
 }
 
-
+export function isExpTranslator(){
+    const db = get(DataBase)
+    return db.translatorType === 'llm' || db.translatorType === 'deepl'
+}
 
 export async function translateHTML(html: string, reverse:boolean): Promise<string> {
+    let db = get(DataBase)
+    let DoingChat = get(doingChat)
+    if(DoingChat){
+        if(isExpTranslator()){
+            return html
+        }
+    }
+    if(db.translatorType === 'llm'){
+        const tr = db.translator || 'en'
+        return translateLLM(html, {to: tr})
+    }
     const dom = new DOMParser().parseFromString(html, 'text/html');
     console.log(html)
 
@@ -254,4 +272,35 @@ export async function translateHTML(html: string, reverse:boolean): Promise<stri
     // console.log(translatedHTML)
     // Return the translated HTML, excluding the outer <body> tags if needed
     return translatedHTML
+}
+
+let llmCache = new Map<string, string>()
+async function translateLLM(text:string, arg:{to:string}){
+    if(llmCache.has(text)){
+        return llmCache.get(text)
+    }
+    const db = get(DataBase)
+    let prompt = db.translatorPrompt || `You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation.`
+    prompt = prompt.replace('{{slot}}', arg.to)
+    const rq = await requestChatData({
+        formated: [
+            {
+                'role': 'system',
+                'content': prompt
+            },
+            {
+                'role': 'user',
+                'content': text
+            }
+        ],
+        bias: {},
+        useStreaming: false,
+    }, 'submodel')
+
+    if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
+        alertError(`${rq.result}`)
+        return text
+    }
+    llmCache.set(text, rq.result)
+    return rq.result
 }

src/ts/util.ts

@@ -365,4 +365,10 @@ export async function decryptBuffer(data:Uint8Array, keys:string){
     )
 
     return result
+}
+
+export function getCurrentCharacter(){
+    const db = get(DataBase)
+    const selectedChar = get(selectedCharID)
+    return db.characters[selectedChar]
 }

version.json

@@ -1 +1 @@
-{"version":"1.60.1"}
+{"version":"1.61.0"}