diff --git a/src/lib/Setting/Pages/OtherBotSettings.svelte b/src/lib/Setting/Pages/OtherBotSettings.svelte index 44a542ac..71314d32 100644 --- a/src/lib/Setting/Pages/OtherBotSettings.svelte +++ b/src/lib/Setting/Pages/OtherBotSettings.svelte @@ -347,6 +347,9 @@ Huggingface Key + fish-speech API Key + + {/if} diff --git a/src/lib/SideBars/CharConfig.svelte b/src/lib/SideBars/CharConfig.svelte index e5b5fae9..dd754bce 100644 --- a/src/lib/SideBars/CharConfig.svelte +++ b/src/lib/SideBars/CharConfig.svelte @@ -82,7 +82,7 @@ } - const unsub = DataBase.subscribe((v) => { + const unsub = DataBase.subscribe(async (v) => { database = v const cha = (v.characters[$selectedCharID]) if(!cha){ @@ -117,6 +117,7 @@ } } + }) let assetFileExtensions:string[] = [] @@ -184,10 +185,56 @@ }; } + let fishSpeechModels:{ + _id:string, + title:string, + description:string + }[] = [] + + $: if (currentChar.data.ttsMode === 'fishspeech' && (currentChar.data as character).fishSpeechConfig === undefined) { + (currentChar.data as character).fishSpeechConfig = { + model: { + _id: '', + title: '', + description: '' + }, + chunk_length: 200, + normalize: false, + }; + } + $: { if(currentChar.type === 'group' && ($CharConfigSubMenu === 4 || $CharConfigSubMenu === 5)){ $CharConfigSubMenu = 0 } + + } + + async function getFishSpeechModels() { + try { + const res = await fetch(`https://api.fish.audio/model?self=true`, { + headers: { + 'Authorization': `Bearer ${$DataBase.fishSpeechKey}` + } + }); + const data = await res.json(); + console.log(data.items); + console.log(currentChar.data) + + if (Array.isArray(data.items)) { + fishSpeechModels = data.items.map((item) => ({ + _id: item._id || '', + title: item.title || '', + description: item.description || '' + })); + } else { + console.error('Expected an array of items, but received:', data.items); + fishSpeechModels = []; + } + } catch (error) { + console.error('Error fetching fish speech models:', error); + fishSpeechModels = []; + } } @@ -665,6 +712,7 @@ Huggingface VITS GPT-SoVITS + fish-speech @@ -877,6 +925,31 @@ Cut 4 (Split by English periods) Cut 5 (Split by various punctuation marks) + {:else if currentChar.data.ttsMode === 'fishspeech'} + {#await getFishSpeechModels()} + Loading... + {:then} + Model + + Not selected + {#each fishSpeechModels as model} + +
+ {model.title} + {model.description} +
+
+ {/each} +
+ {:catch} + An error occurred while fetching the models. + {/await} + + Chunk Length + + + Normalize + {/if} {#if currentChar.data.ttsMode}
diff --git a/src/ts/process/tts.ts b/src/ts/process/tts.ts index 18b39caf..9b82082c 100644 --- a/src/ts/process/tts.ts +++ b/src/ts/process/tts.ts @@ -311,6 +311,49 @@ export async function sayTTS(character:character,text:string) { throw new Error(text); } } + case 'fishspeech':{ + if (character.fishSpeechConfig.model._id === ''){ + throw new Error('FishSpeech Model is not selected') + } + const audioContext = new AudioContext(); + + const body = { + text: text, + reference_id: character.fishSpeechConfig.model._id, + chunk_length: character.fishSpeechConfig.chunk_length, + normalize: character.fishSpeechConfig.normalize, + format: 'mp3', + mp3_bitrate: 192, + } + + + console.log(body) + + const response = await globalFetch(`https://api.fish.audio/v1/tts`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${db.fishSpeechKey}` + }, + body: body, + rawResponse: true, + }) + console.log(response) + + if (response.ok) { + const audioBuffer = response.data.buffer; + audioContext.decodeAudioData(audioBuffer, (decodedData) => { + const sourceNode = audioContext.createBufferSource(); + sourceNode.buffer = decodedData; + sourceNode.connect(audioContext.destination); + sourceNode.start(); + }); + } else { + const textBuffer: Uint8Array = response.data.buffer + const text = Buffer.from(textBuffer).toString('utf-8') + throw new Error(text); + } + } } } catch (error) { alertError(`TTS Error: ${error}`) diff --git a/src/ts/storage/database.ts b/src/ts/storage/database.ts index 35bbdd6f..aa7cf35b 100644 --- a/src/ts/storage/database.ts +++ b/src/ts/storage/database.ts @@ -351,6 +351,7 @@ export function setDatabase(data:Database){ data.newOAIHandle ??= true data.gptVisionQuality ??= 'low' data.huggingfaceKey ??= '' + data.fishSpeechKey ??= '' data.statistics ??= {} data.reverseProxyOobaArgs ??= { mode: 'instruct' @@ -651,6 +652,7 @@ export interface Database{ tpo?:boolean automark?:boolean huggingfaceKey:string + fishSpeechKey:string allowAllExtentionFiles?:boolean translatorPrompt:string translatorMaxResponse:number @@ -863,6 +865,16 @@ export interface character{ top_k?:number text_split_method?: "cut0" | "cut1" | "cut2" | "cut3" | "cut4" | "cut5" } + fishSpeechConfig?:{ + model?: { + _id:string + title:string + description:string + }, + chunk_length:number, + normalize:boolean, + + } supaMemory?:boolean additionalAssets?:[string, string, string][] ttsReadOnlyQuoted?:boolean