Merge branch 'main' of https://github.com/kwaroran/RisuAI

Signed-off-by: hashcoko <hashcoko@gmail.com>

src/ts/process/embedding/transformers.ts

@@ -52,4 +52,11 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
     let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
     let result = await extractor(text, { pooling: 'mean', normalize: true });
     return result?.data ?? null;
+}
+
+export const runTTS = async (text: string) => {
+    let speaker_embeddings = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
+    let synthesizer = await pipeline('text-to-speech', 'Xenova/speecht5_tts', { local_files_only: true });
+    let out = await synthesizer(text, { speaker_embeddings });
+    return out
 }

src/ts/process/tts.ts

@@ -1,13 +1,21 @@
 import { get } from "svelte/store";
 import { alertError } from "../alert";
 import { DataBase, type character } from "../storage/database";
-import { translateVox } from "../translator/translator";
+import { runTranslator, translateVox } from "../translator/translator";
 import { globalFetch } from "../storage/globalApi";
 import { language } from "src/lang";
+import { getCurrentCharacter, sleep } from "../util";
 
 let sourceNode:AudioBufferSourceNode = null
 
 export async function sayTTS(character:character,text:string) {
+    if(!character){
+        const v = getCurrentCharacter()
+        if(v.type === 'group'){
+            return
+        }
+        character = v
+    }
 
     let db = get(DataBase)
     text = text.replace(/\*/g,'')
@@ -162,6 +170,48 @@ export async function sayTTS(character:character,text:string) {
             }
             break;
         }
+        case 'huggingface': {
+            while(true){
+                if(character.hfTTS.language !== 'en'){
+                    text = await runTranslator(text, false, 'en', character.hfTTS.language)
+                }
+                const audioContext = new AudioContext();
+                const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
+                    method: 'POST',
+                    headers: {
+                        "Authorization": "Bearer " + db.huggingfaceKey,
+                        "Content-Type": "application/json",
+                    },
+                    body: JSON.stringify({
+                        inputs: text,
+                    })
+                });
+    
+                if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
+                    const json = await response.json()
+                    if(json.estimated_time){
+                        await sleep(json.estimated_time * 1000)
+                        continue
+                    }
+                }
+                else if(response.status >= 400){
+                    alertError(language.errors.httpError + `${await response.text()}`)
+                    return
+                }
+                else if (response.status === 200) {
+                    const audioBuffer = await response.arrayBuffer();
+                    audioContext.decodeAudioData(audioBuffer, (decodedData) => {
+                        const sourceNode = audioContext.createBufferSource();
+                        sourceNode.buffer = decodedData;
+                        sourceNode.connect(audioContext.destination);
+                        sourceNode.start();
+                    });
+                } else {
+                    alertError("Error fetching or decoding audio data");
+                }
+                return
+            }
+        }
     }
 }