[feat] add hf tts

src/ts/characters.ts

@@ -301,6 +301,10 @@ export function characterFormatUpdate(index:number|character){
             depth: 0,
             prompt: ''
         }
+        cha.hfTTS = {
+            model: '',
+            language: 'en'
+        }
         if(!cha.newGenData){
             cha = updateInlayScreen(cha)
         }

src/ts/process/embedding/transformers.ts

@@ -52,4 +52,4 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
     let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
     let result = await extractor(text, { pooling: 'mean', normalize: true });
     return result?.data ?? null;
-}
+}

src/ts/process/tts.ts

@@ -4,6 +4,7 @@ import { DataBase, type character } from "../storage/database";
 import { translateVox } from "../translator/translator";
 import { globalFetch } from "../storage/globalApi";
 import { language } from "src/lang";
+import { sleep } from "../util";
 
 let sourceNode:AudioBufferSourceNode = null
 
@@ -161,6 +162,45 @@ export async function sayTTS(character:character,text:string) {
             }
             break;
         }
+        case 'huggingface': {
+            while(true){
+                const audioContext = new AudioContext();
+                const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
+                    method: 'POST',
+                    headers: {
+                        "Authorization": "Bearer " + db.huggingfaceKey,
+                        "Content-Type": "application/json",
+                    },
+                    body: JSON.stringify({
+                        inputs: text,
+                    })
+                });
+    
+                if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
+                    const json = await response.json()
+                    if(json.estimated_time){
+                        await sleep(json.estimated_time * 1000)
+                        continue
+                    }
+                }
+                else if(response.status >= 400){
+                    alertError(language.errors.httpError + `${await response.text()}`)
+                    return
+                }
+                else if (response.status === 200) {
+                    const audioBuffer = await response.arrayBuffer();
+                    audioContext.decodeAudioData(audioBuffer, (decodedData) => {
+                        const sourceNode = audioContext.createBufferSource();
+                        sourceNode.buffer = decodedData;
+                        sourceNode.connect(audioContext.destination);
+                        sourceNode.start();
+                    });
+                } else {
+                    alertError("Error fetching or decoding audio data");
+                }
+                return
+            }
+        }
     }
 }
 

src/ts/storage/database.ts

@@ -347,6 +347,7 @@ export function setDatabase(data:Database){
     data.generationSeed ??= -1
     data.newOAIHandle ??= true
     data.gptVisionQuality ??= 'low'
+    data.huggingfaceKey ??= ''
     data.reverseProxyOobaArgs ??= {
         mode: 'instruct'
     }
@@ -538,7 +539,7 @@ export interface Database{
     reverseProxyOobaArgs: OobaChatCompletionRequestParams
     tpo?:boolean
     automark?:boolean
-
+    huggingfaceKey:string
     allowAllExtentionFiles?:boolean
 }
 
@@ -648,6 +649,10 @@ export interface character{
     largePortrait?:boolean
     lorePlus?:boolean
     inlayViewScreen?:boolean
+    hfTTS?: {
+        model: string
+        language: string
+    }
 }