From 6ae8264f8b01f22b4f4bfa1b97c45b9fe9288b4a Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Tue, 28 Nov 2023 03:36:08 +0900
Subject: [PATCH] [feat] hf tts translation

---
 src/lib/ChatScreens/Chat.svelte          |  9 ++++++++-
 src/ts/characters.ts                     |  2 +-
 src/ts/plugins/embedscript.ts            |  1 +
 src/ts/process/embedding/transformers.ts |  7 +++++++
 src/ts/process/tts.ts                    | 14 ++++++++++++--
 src/ts/translator/translator.ts          |  2 +-
 src/ts/util.ts                           |  6 ++++++
 7 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/src/lib/ChatScreens/Chat.svelte b/src/lib/ChatScreens/Chat.svelte
index 52b19c72..35b18ccf 100644
--- a/src/lib/ChatScreens/Chat.svelte
+++ b/src/lib/ChatScreens/Chat.svelte
@@ -1,5 +1,5 @@
 <script lang="ts">
-    import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon } from "lucide-svelte";
+    import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon, Volume2Icon } from "lucide-svelte";
     import { ParseMarkdown, type simpleCharacterArgument } from "../../ts/parser";
     import AutoresizeArea from "../UI/GUI/TextAreaResizable.svelte";
     import { alertConfirm, alertError } from "../../ts/alert";
@@ -10,6 +10,7 @@
     import { risuChatParser } from "src/ts/process/scripts";
     import { get } from "svelte/store";
   import { isEqual } from "lodash";
+  import { sayTTS } from "src/ts/process/tts";
     export let message = ''
     export let name = ''
     export let largePortrait = false
@@ -145,6 +146,12 @@
                         </button>    
                     {/if}
                     {#if idx > -1}
+                        <button class="ml-2 hover:text-green-500 transition-colors" on:click={()=>{
+                            return sayTTS(null, message)
+                        }}>
+                            <Volume2Icon size={20}/>
+                        </button>
+
                         <button class={"ml-2 hover:text-green-500 transition-colors "+(editMode?'text-green-400':'')} on:click={() => {
                             if(!editMode){
                                 editMode = true
diff --git a/src/ts/characters.ts b/src/ts/characters.ts
index 7fcc7ed7..0755a5c8 100644
--- a/src/ts/characters.ts
+++ b/src/ts/characters.ts
@@ -301,7 +301,7 @@ export function characterFormatUpdate(index:number|character){
             depth: 0,
             prompt: ''
         }
-        cha.hfTTS = {
+        cha.hfTTS ??= {
             model: '',
             language: 'en'
         }
diff --git a/src/ts/plugins/embedscript.ts b/src/ts/plugins/embedscript.ts
index 002daaf1..4af50ec8 100644
--- a/src/ts/plugins/embedscript.ts
+++ b/src/ts/plugins/embedscript.ts
@@ -1,5 +1,6 @@
 import { get } from 'svelte/store'
 import type { ScriptMode } from '../process/scripts'
+//@ts-ignore
 import WorkerUrl from './embedworker?worker&url'
 import { DataBase, type Chat, type character, type Message } from '../storage/database'
 import { selectedCharID } from '../stores'
diff --git a/src/ts/process/embedding/transformers.ts b/src/ts/process/embedding/transformers.ts
index c8523e37..cac64bda 100644
--- a/src/ts/process/embedding/transformers.ts
+++ b/src/ts/process/embedding/transformers.ts
@@ -53,3 +53,10 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
     let result = await extractor(text, { pooling: 'mean', normalize: true });
     return result?.data ?? null;
 }
+
+export const runTTS = async (text: string) => {
+    let speaker_embeddings = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
+    let synthesizer = await pipeline('text-to-speech', 'Xenova/speecht5_tts', { local_files_only: true });
+    let out = await synthesizer(text, { speaker_embeddings });
+    return out
+}
\ No newline at end of file
diff --git a/src/ts/process/tts.ts b/src/ts/process/tts.ts
index cdb83356..b05a51f1 100644
--- a/src/ts/process/tts.ts
+++ b/src/ts/process/tts.ts
@@ -1,14 +1,21 @@
 import { get } from "svelte/store";
 import { alertError } from "../alert";
 import { DataBase, type character } from "../storage/database";
-import { translateVox } from "../translator/translator";
+import { runTranslator, translateVox } from "../translator/translator";
 import { globalFetch } from "../storage/globalApi";
 import { language } from "src/lang";
-import { sleep } from "../util";
+import { getCurrentCharacter, sleep } from "../util";
 
 let sourceNode:AudioBufferSourceNode = null
 
 export async function sayTTS(character:character,text:string) {
+    if(!character){
+        const v = getCurrentCharacter()
+        if(v.type === 'group'){
+            return
+        }
+        character = v
+    }
 
     let db = get(DataBase)
     text = text.replace(/\*/g,'')
@@ -164,6 +171,9 @@ export async function sayTTS(character:character,text:string) {
         }
         case 'huggingface': {
             while(true){
+                if(character.hfTTS.language !== 'en'){
+                    text = await runTranslator(text, false, 'en', character.hfTTS.language)
+                }
                 const audioContext = new AudioContext();
                 const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
                     method: 'POST',
diff --git a/src/ts/translator/translator.ts b/src/ts/translator/translator.ts
index 8432a36f..9b7de9eb 100644
--- a/src/ts/translator/translator.ts
+++ b/src/ts/translator/translator.ts
@@ -33,7 +33,7 @@ export async function translate(text:string, reverse:boolean) {
     return runTranslator(text, reverse, db.translator,db.aiModel.startsWith('novellist') ? 'ja' : 'en')
 }
 
-async function runTranslator(text:string, reverse:boolean, from:string,target:'en'|'ja') {
+export async function runTranslator(text:string, reverse:boolean, from:string,target:string) {
     const arg = {
 
         from: reverse ? from : target,
diff --git a/src/ts/util.ts b/src/ts/util.ts
index 9b951437..a3825d37 100644
--- a/src/ts/util.ts
+++ b/src/ts/util.ts
@@ -365,4 +365,10 @@ export async function decryptBuffer(data:Uint8Array, keys:string){
     )
 
     return result
+}
+
+export function getCurrentCharacter(){
+    const db = get(DataBase)
+    const selectedChar = get(selectedCharID)
+    return db.characters[selectedChar]
 }
\ No newline at end of file