Add model selection for VitsModel
This commit is contained in:
24
public/sw.js
24
public/sw.js
@@ -7,7 +7,13 @@ self.addEventListener('fetch', (event) => {
|
|||||||
try {
|
try {
|
||||||
switch (path[2]){
|
switch (path[2]){
|
||||||
case "check":{
|
case "check":{
|
||||||
event.respondWith(checkCache(url))
|
let targetUrl = url
|
||||||
|
const headers = event.request.headers
|
||||||
|
const headerUrl = headers.get('x-register-url')
|
||||||
|
if(headerUrl){
|
||||||
|
targetUrl.pathname = decodeURIComponent(headerUrl)
|
||||||
|
}
|
||||||
|
event.respondWith(checkCache(targetUrl))
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
case "img": {
|
case "img": {
|
||||||
@@ -15,20 +21,20 @@ self.addEventListener('fetch', (event) => {
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
case "register": {
|
case "register": {
|
||||||
let targerUrl = url
|
let targetUrl = url
|
||||||
const headers = event.request.headers
|
const headers = event.request.headers
|
||||||
const headerUrl = headers.get('x-register-url')
|
const headerUrl = headers.get('x-register-url')
|
||||||
if(headerUrl){
|
if(headerUrl){
|
||||||
targerUrl = new URL(headerUrl)
|
targetUrl.pathname = decodeURIComponent(headerUrl)
|
||||||
}
|
}
|
||||||
const noContentType = headers.get('x-no-content-type') === 'true'
|
const noContentType = headers.get('x-no-content-type') === 'true'
|
||||||
event.respondWith(
|
event.respondWith(
|
||||||
registerCache(targerUrl, event.request.arrayBuffer(), noContentType)
|
registerCache(targetUrl, event.request.arrayBuffer(), noContentType)
|
||||||
)
|
)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
case "init":{
|
case "init":{
|
||||||
event.respondWith(new Response("true"))
|
event.respondWith(new Response("v2"))
|
||||||
}
|
}
|
||||||
default: {
|
default: {
|
||||||
event.respondWith(new Response(
|
event.respondWith(new Response(
|
||||||
@@ -74,9 +80,11 @@ async function check(){
|
|||||||
async function registerCache(urlr, buffer, noContentType = false){
|
async function registerCache(urlr, buffer, noContentType = false){
|
||||||
const cache = await caches.open('risuCache')
|
const cache = await caches.open('risuCache')
|
||||||
const url = new URL(urlr)
|
const url = new URL(urlr)
|
||||||
let path = url.pathname.split('/')
|
if(!noContentType){
|
||||||
path[2] = 'img'
|
let path = url.pathname.split('/')
|
||||||
url.pathname = path.join('/')
|
path[2] = 'img'
|
||||||
|
url.pathname = path.join('/')
|
||||||
|
}
|
||||||
const buf = new Uint8Array(await buffer)
|
const buf = new Uint8Array(await buffer)
|
||||||
let headers = {
|
let headers = {
|
||||||
"cache-control": "max-age=604800",
|
"cache-control": "max-age=604800",
|
||||||
|
|||||||
@@ -484,4 +484,5 @@ export const languageEnglish = {
|
|||||||
chatAsOriginalOnSystem: "Send as original role",
|
chatAsOriginalOnSystem: "Send as original role",
|
||||||
exportAsDataset: "Export Save as Dataset",
|
exportAsDataset: "Export Save as Dataset",
|
||||||
editTranslationDisplay: "Edit Translation Display",
|
editTranslationDisplay: "Edit Translation Display",
|
||||||
|
selectModel: "Select Model",
|
||||||
}
|
}
|
||||||
@@ -29,6 +29,7 @@
|
|||||||
import TriggerList from "./Scripts/TriggerList.svelte";
|
import TriggerList from "./Scripts/TriggerList.svelte";
|
||||||
import CheckInput from "../UI/GUI/CheckInput.svelte";
|
import CheckInput from "../UI/GUI/CheckInput.svelte";
|
||||||
import { updateInlayScreen } from "src/ts/process/inlayScreen";
|
import { updateInlayScreen } from "src/ts/process/inlayScreen";
|
||||||
|
import { registerOnnxModel } from "src/ts/process/embedding/transformers";
|
||||||
|
|
||||||
|
|
||||||
let subMenu = 0
|
let subMenu = 0
|
||||||
@@ -626,6 +627,19 @@
|
|||||||
<span class="text-textcolor">Language</span>
|
<span class="text-textcolor">Language</span>
|
||||||
<TextInput additionalClass="mb-4 mt-2" bind:value={currentChar.data.hfTTS.language} placeholder="en" />
|
<TextInput additionalClass="mb-4 mt-2" bind:value={currentChar.data.hfTTS.language} placeholder="en" />
|
||||||
{/if}
|
{/if}
|
||||||
|
{#if currentChar.data.ttsMode === 'vits'}
|
||||||
|
{#if currentChar.data.vits}
|
||||||
|
<span class="text-textcolor">{currentChar.data.vits.name ?? 'Unnamed VitsModel'}</span>
|
||||||
|
{:else}
|
||||||
|
<span class="text-textcolor">No Model</span>
|
||||||
|
{/if}
|
||||||
|
<Button on:click={async () => {
|
||||||
|
const model = await registerOnnxModel()
|
||||||
|
if(model && currentChar.type === 'character'){
|
||||||
|
currentChar.data.vits = model
|
||||||
|
}
|
||||||
|
}}>{language.selectModel}</Button>
|
||||||
|
{/if}
|
||||||
{#if currentChar.data.ttsMode}
|
{#if currentChar.data.ttsMode}
|
||||||
<div class="flex items-center mt-2">
|
<div class="flex items-center mt-2">
|
||||||
<Check bind:check={currentChar.data.ttsReadOnlyQuoted} name={language.ttsReadOnlyQuoted}/>
|
<Check bind:check={currentChar.data.ttsReadOnlyQuoted} name={language.ttsReadOnlyQuoted}/>
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
import {env, AutoTokenizer, pipeline, VitsModel, type SummarizationOutput, type TextGenerationConfig, type TextGenerationOutput, FeatureExtractionPipeline, TextToAudioPipeline } from '@xenova/transformers';
|
import {env, AutoTokenizer, pipeline, type SummarizationOutput, type TextGenerationConfig, type TextGenerationOutput, FeatureExtractionPipeline, TextToAudioPipeline } from '@xenova/transformers';
|
||||||
|
import { unzip } from 'fflate';
|
||||||
|
import { loadAsset, saveAsset } from 'src/ts/storage/globalApi';
|
||||||
|
import { selectSingleFile } from 'src/ts/util';
|
||||||
|
import { v4 } from 'uuid';
|
||||||
|
|
||||||
env.localModelPath = "https://sv.risuai.xyz/transformers/"
|
env.localModelPath = "/transformers/"
|
||||||
|
env.remoteHost = "https://sv.risuai.xyz/transformers/"
|
||||||
|
|
||||||
export const runTransformers = async (baseText:string, model:string,config:TextGenerationConfig = {}) => {
|
export const runTransformers = async (baseText:string, model:string,config:TextGenerationConfig = {}) => {
|
||||||
let text = baseText
|
let text = baseText
|
||||||
@@ -61,11 +66,49 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
|
|||||||
|
|
||||||
let synthesizer:TextToAudioPipeline = null
|
let synthesizer:TextToAudioPipeline = null
|
||||||
let lastSynth:string = null
|
let lastSynth:string = null
|
||||||
export const runVITS = async (text: string, model:string = 'Xenova/mms-tts-eng') => {
|
|
||||||
|
export interface OnnxModelFiles {
|
||||||
|
files: {[key:string]:string},
|
||||||
|
id: string,
|
||||||
|
name?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export const runVITS = async (text: string, modelData:string|OnnxModelFiles = 'Xenova/mms-tts-eng') => {
|
||||||
const {WaveFile} = await import('wavefile')
|
const {WaveFile} = await import('wavefile')
|
||||||
if((!synthesizer) || (lastSynth !== model)){
|
if(modelData === null){
|
||||||
lastSynth = model
|
return
|
||||||
synthesizer = await pipeline('text-to-speech', model);
|
}
|
||||||
|
if(typeof modelData === 'string'){
|
||||||
|
if((!synthesizer) || (lastSynth !== modelData)){
|
||||||
|
lastSynth = modelData
|
||||||
|
synthesizer = await pipeline('text-to-speech', modelData);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
if((!synthesizer) || (lastSynth !== modelData.id)){
|
||||||
|
const files = modelData.files
|
||||||
|
const keys = Object.keys(files)
|
||||||
|
for(const key of keys){
|
||||||
|
const hasCache:boolean = (await (await fetch("/sw/check/", {
|
||||||
|
headers: {
|
||||||
|
'x-register-url': encodeURIComponent(key)
|
||||||
|
}
|
||||||
|
})).json()).able
|
||||||
|
|
||||||
|
if(!hasCache){
|
||||||
|
await fetch("/sw/register/", {
|
||||||
|
method: "POST",
|
||||||
|
body: await loadAsset(files[key]),
|
||||||
|
headers: {
|
||||||
|
'x-register-url': encodeURIComponent(key),
|
||||||
|
'x-no-content-type': 'true'
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lastSynth = modelData.id
|
||||||
|
synthesizer = await pipeline('text-to-speech', modelData.id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let out = await synthesizer(text, {});
|
let out = await synthesizer(text, {});
|
||||||
const wav = new WaveFile();
|
const wav = new WaveFile();
|
||||||
@@ -77,4 +120,52 @@ export const runVITS = async (text: string, model:string = 'Xenova/mms-tts-eng')
|
|||||||
sourceNode.connect(audioContext.destination);
|
sourceNode.connect(audioContext.destination);
|
||||||
sourceNode.start();
|
sourceNode.start();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
export const registerOnnxModel = async ():Promise<OnnxModelFiles> => {
|
||||||
|
const id = v4().replace(/-/g, '')
|
||||||
|
|
||||||
|
const modelFile = await selectSingleFile(['zip'])
|
||||||
|
|
||||||
|
if(!modelFile){
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const unziped = await new Promise((res, rej) => {unzip(modelFile.data, {
|
||||||
|
filter: (file) => {
|
||||||
|
return file.name.endsWith('.onnx') || file.size < 10_000_000 || file.name.includes('.git')
|
||||||
|
}
|
||||||
|
}, (err, unzipped) => {
|
||||||
|
if(err){
|
||||||
|
rej(err)
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
res(unzipped)
|
||||||
|
}
|
||||||
|
})})
|
||||||
|
|
||||||
|
console.log(unziped)
|
||||||
|
|
||||||
|
let fileIdMapped:{[key:string]:string} = {}
|
||||||
|
|
||||||
|
const keys = Object.keys(unziped)
|
||||||
|
for(let i = 0; i < keys.length; i++){
|
||||||
|
const key = keys[i]
|
||||||
|
const file = unziped[key]
|
||||||
|
const fid = await saveAsset(file)
|
||||||
|
let url = key
|
||||||
|
if(url.startsWith('/')){
|
||||||
|
url = url.substring(1)
|
||||||
|
}
|
||||||
|
url = '/transformers/' + id +'/' + url
|
||||||
|
fileIdMapped[url] = fid
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
files: fileIdMapped,
|
||||||
|
name: modelFile.name,
|
||||||
|
id: id,
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,211 +5,171 @@ import { runTranslator, translateVox } from "../translator/translator";
|
|||||||
import { globalFetch } from "../storage/globalApi";
|
import { globalFetch } from "../storage/globalApi";
|
||||||
import { language } from "src/lang";
|
import { language } from "src/lang";
|
||||||
import { getCurrentCharacter, sleep } from "../util";
|
import { getCurrentCharacter, sleep } from "../util";
|
||||||
import { runVITS } from "./embedding/transformers";
|
import { registerOnnxModel, runVITS } from "./embedding/transformers";
|
||||||
|
|
||||||
let sourceNode:AudioBufferSourceNode = null
|
let sourceNode:AudioBufferSourceNode = null
|
||||||
|
|
||||||
export async function sayTTS(character:character,text:string) {
|
export async function sayTTS(character:character,text:string) {
|
||||||
if(!character){
|
try {
|
||||||
const v = getCurrentCharacter()
|
if(!character){
|
||||||
if(v.type === 'group'){
|
const v = getCurrentCharacter()
|
||||||
return
|
if(v.type === 'group'){
|
||||||
}
|
return
|
||||||
character = v
|
|
||||||
}
|
|
||||||
|
|
||||||
let db = get(DataBase)
|
|
||||||
text = text.replace(/\*/g,'')
|
|
||||||
|
|
||||||
if(character.ttsReadOnlyQuoted){
|
|
||||||
const matches = text.match(/"(.*?)"/g)
|
|
||||||
if(matches && matches.length > 0){
|
|
||||||
text = matches.map(match => match.slice(1, -1)).join("");
|
|
||||||
}
|
|
||||||
else{
|
|
||||||
text = ''
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
switch(character.ttsMode){
|
|
||||||
case "webspeech":{
|
|
||||||
if(speechSynthesis && SpeechSynthesisUtterance){
|
|
||||||
const utterThis = new SpeechSynthesisUtterance(text);
|
|
||||||
const voices = speechSynthesis.getVoices();
|
|
||||||
let voiceIndex = 0
|
|
||||||
for(let i=0;i<voices.length;i++){
|
|
||||||
if(voices[i].name === character.ttsSpeech){
|
|
||||||
voiceIndex = i
|
|
||||||
}
|
|
||||||
}
|
|
||||||
utterThis.voice = voices[voiceIndex]
|
|
||||||
const speak = speechSynthesis.speak(utterThis)
|
|
||||||
}
|
}
|
||||||
break
|
character = v
|
||||||
}
|
}
|
||||||
case "elevenlab": {
|
|
||||||
const audioContext = new AudioContext();
|
let db = get(DataBase)
|
||||||
const da = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${character.ttsSpeech}`, {
|
text = text.replace(/\*/g,'')
|
||||||
body: JSON.stringify({
|
|
||||||
text: text
|
if(character.ttsReadOnlyQuoted){
|
||||||
}),
|
const matches = text.match(/"(.*?)"/g)
|
||||||
method: "POST",
|
if(matches && matches.length > 0){
|
||||||
headers: {
|
text = matches.map(match => match.slice(1, -1)).join("");
|
||||||
"Content-Type": "application/json",
|
|
||||||
'xi-api-key': db.elevenLabKey || undefined
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if(da.status >= 200 && da.status < 300){
|
|
||||||
const audioBuffer = await audioContext.decodeAudioData(await da.arrayBuffer())
|
|
||||||
sourceNode = audioContext.createBufferSource();
|
|
||||||
sourceNode.buffer = audioBuffer;
|
|
||||||
sourceNode.connect(audioContext.destination);
|
|
||||||
sourceNode.start();
|
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
alertError(await da.text())
|
text = ''
|
||||||
}
|
}
|
||||||
break
|
|
||||||
}
|
}
|
||||||
case "VOICEVOX": {
|
|
||||||
const jpText = await translateVox(text)
|
switch(character.ttsMode){
|
||||||
const audioContext = new AudioContext();
|
case "webspeech":{
|
||||||
const query = await fetch(`${db.voicevoxUrl}/audio_query?text=${jpText}&speaker=${character.ttsSpeech}`, {
|
if(speechSynthesis && SpeechSynthesisUtterance){
|
||||||
method: 'POST',
|
const utterThis = new SpeechSynthesisUtterance(text);
|
||||||
headers: { "Content-Type": "application/json"},
|
const voices = speechSynthesis.getVoices();
|
||||||
})
|
let voiceIndex = 0
|
||||||
if (query.status == 200){
|
for(let i=0;i<voices.length;i++){
|
||||||
const queryJson = await query.json();
|
if(voices[i].name === character.ttsSpeech){
|
||||||
const bodyData = {
|
voiceIndex = i
|
||||||
accent_phrases: queryJson.accent_phrases,
|
}
|
||||||
speedScale: character.voicevoxConfig.SPEED_SCALE,
|
}
|
||||||
pitchScale: character.voicevoxConfig.PITCH_SCALE,
|
utterThis.voice = voices[voiceIndex]
|
||||||
volumeScale: character.voicevoxConfig.VOLUME_SCALE,
|
const speak = speechSynthesis.speak(utterThis)
|
||||||
intonationScale: character.voicevoxConfig.INTONATION_SCALE,
|
|
||||||
prePhonemeLength: queryJson.prePhonemeLength,
|
|
||||||
postPhonemeLength: queryJson.postPhonemeLength,
|
|
||||||
outputSamplingRate: queryJson.outputSamplingRate,
|
|
||||||
outputStereo: queryJson.outputStereo,
|
|
||||||
kana: queryJson.kana,
|
|
||||||
}
|
}
|
||||||
const getVoice = await fetch(`${db.voicevoxUrl}/synthesis?speaker=${character.ttsSpeech}`, {
|
break
|
||||||
|
}
|
||||||
|
case "elevenlab": {
|
||||||
|
const audioContext = new AudioContext();
|
||||||
|
const da = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${character.ttsSpeech}`, {
|
||||||
|
body: JSON.stringify({
|
||||||
|
text: text
|
||||||
|
}),
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
'xi-api-key': db.elevenLabKey || undefined
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if(da.status >= 200 && da.status < 300){
|
||||||
|
const audioBuffer = await audioContext.decodeAudioData(await da.arrayBuffer())
|
||||||
|
sourceNode = audioContext.createBufferSource();
|
||||||
|
sourceNode.buffer = audioBuffer;
|
||||||
|
sourceNode.connect(audioContext.destination);
|
||||||
|
sourceNode.start();
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
alertError(await da.text())
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
case "VOICEVOX": {
|
||||||
|
const jpText = await translateVox(text)
|
||||||
|
const audioContext = new AudioContext();
|
||||||
|
const query = await fetch(`${db.voicevoxUrl}/audio_query?text=${jpText}&speaker=${character.ttsSpeech}`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { "Content-Type": "application/json"},
|
headers: { "Content-Type": "application/json"},
|
||||||
body: JSON.stringify(bodyData),
|
|
||||||
})
|
})
|
||||||
if (getVoice.status == 200 && getVoice.headers.get('content-type') === 'audio/wav'){
|
if (query.status == 200){
|
||||||
const audioBuffer = await audioContext.decodeAudioData(await getVoice.arrayBuffer())
|
const queryJson = await query.json();
|
||||||
sourceNode = audioContext.createBufferSource();
|
const bodyData = {
|
||||||
sourceNode.buffer = audioBuffer;
|
accent_phrases: queryJson.accent_phrases,
|
||||||
sourceNode.connect(audioContext.destination);
|
speedScale: character.voicevoxConfig.SPEED_SCALE,
|
||||||
sourceNode.start();
|
pitchScale: character.voicevoxConfig.PITCH_SCALE,
|
||||||
}
|
volumeScale: character.voicevoxConfig.VOLUME_SCALE,
|
||||||
}
|
intonationScale: character.voicevoxConfig.INTONATION_SCALE,
|
||||||
break
|
prePhonemeLength: queryJson.prePhonemeLength,
|
||||||
}
|
postPhonemeLength: queryJson.postPhonemeLength,
|
||||||
case 'openai':{
|
outputSamplingRate: queryJson.outputSamplingRate,
|
||||||
const key = db.openAIKey
|
outputStereo: queryJson.outputStereo,
|
||||||
const res = await globalFetch('https://api.openai.com/v1/audio/speech', {
|
kana: queryJson.kana,
|
||||||
method: 'POST',
|
}
|
||||||
headers: {
|
const getVoice = await fetch(`${db.voicevoxUrl}/synthesis?speaker=${character.ttsSpeech}`, {
|
||||||
'Content-Type': 'application/json',
|
method: 'POST',
|
||||||
'Authorization': 'Bearer ' + key,
|
headers: { "Content-Type": "application/json"},
|
||||||
},
|
body: JSON.stringify(bodyData),
|
||||||
body: {
|
|
||||||
model: 'tts-1',
|
|
||||||
input: text,
|
|
||||||
voice: character.oaiVoice,
|
|
||||||
|
|
||||||
},
|
|
||||||
rawResponse: true,
|
|
||||||
})
|
|
||||||
const dat = res.data
|
|
||||||
|
|
||||||
if(res.ok){
|
|
||||||
try {
|
|
||||||
const audio = Buffer.from(dat).buffer
|
|
||||||
const audioContext = new AudioContext();
|
|
||||||
const audioBuffer = await audioContext.decodeAudioData(audio)
|
|
||||||
sourceNode = audioContext.createBufferSource();
|
|
||||||
sourceNode.buffer = audioBuffer;
|
|
||||||
sourceNode.connect(audioContext.destination);
|
|
||||||
sourceNode.start();
|
|
||||||
} catch (error) {
|
|
||||||
alertError(language.errors.httpError + `${error}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else{
|
|
||||||
if(dat.error && dat.error.message){
|
|
||||||
alertError((language.errors.httpError + `${dat.error.message}`))
|
|
||||||
}
|
|
||||||
else{
|
|
||||||
alertError((language.errors.httpError + `${Buffer.from(res.data).toString()}`))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
|
|
||||||
}
|
|
||||||
case 'novelai': {
|
|
||||||
const audioContext = new AudioContext();
|
|
||||||
if(text === ''){
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
const encodedText = encodeURIComponent(text);
|
|
||||||
const encodedSeed = encodeURIComponent(character.naittsConfig.voice);
|
|
||||||
|
|
||||||
const url = `https://api.novelai.net/ai/generate-voice?text=${encodedText}&voice=-1&seed=${encodedSeed}&opus=false&version=${character.naittsConfig.version}`;
|
|
||||||
|
|
||||||
const response = await globalFetch(url, {
|
|
||||||
method: 'GET',
|
|
||||||
headers: {
|
|
||||||
"Authorization": "Bearer " + db.NAIApiKey,
|
|
||||||
},
|
|
||||||
rawResponse: true
|
|
||||||
});
|
|
||||||
|
|
||||||
if (response.ok) {
|
|
||||||
const audioBuffer = response.data.buffer;
|
|
||||||
audioContext.decodeAudioData(audioBuffer, (decodedData) => {
|
|
||||||
const sourceNode = audioContext.createBufferSource();
|
|
||||||
sourceNode.buffer = decodedData;
|
|
||||||
sourceNode.connect(audioContext.destination);
|
|
||||||
sourceNode.start();
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
alertError("Error fetching or decoding audio data");
|
|
||||||
}
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
case 'huggingface': {
|
|
||||||
while(true){
|
|
||||||
if(character.hfTTS.language !== 'en'){
|
|
||||||
text = await runTranslator(text, false, 'en', character.hfTTS.language)
|
|
||||||
}
|
|
||||||
const audioContext = new AudioContext();
|
|
||||||
const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
"Authorization": "Bearer " + db.huggingfaceKey,
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
inputs: text,
|
|
||||||
})
|
})
|
||||||
});
|
if (getVoice.status == 200 && getVoice.headers.get('content-type') === 'audio/wav'){
|
||||||
|
const audioBuffer = await audioContext.decodeAudioData(await getVoice.arrayBuffer())
|
||||||
if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
|
sourceNode = audioContext.createBufferSource();
|
||||||
const json = await response.json()
|
sourceNode.buffer = audioBuffer;
|
||||||
if(json.estimated_time){
|
sourceNode.connect(audioContext.destination);
|
||||||
await sleep(json.estimated_time * 1000)
|
sourceNode.start();
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if(response.status >= 400){
|
break
|
||||||
alertError(language.errors.httpError + `${await response.text()}`)
|
}
|
||||||
return
|
case 'openai':{
|
||||||
|
const key = db.openAIKey
|
||||||
|
const res = await globalFetch('https://api.openai.com/v1/audio/speech', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'Authorization': 'Bearer ' + key,
|
||||||
|
},
|
||||||
|
body: {
|
||||||
|
model: 'tts-1',
|
||||||
|
input: text,
|
||||||
|
voice: character.oaiVoice,
|
||||||
|
|
||||||
|
},
|
||||||
|
rawResponse: true,
|
||||||
|
})
|
||||||
|
const dat = res.data
|
||||||
|
|
||||||
|
if(res.ok){
|
||||||
|
try {
|
||||||
|
const audio = Buffer.from(dat).buffer
|
||||||
|
const audioContext = new AudioContext();
|
||||||
|
const audioBuffer = await audioContext.decodeAudioData(audio)
|
||||||
|
sourceNode = audioContext.createBufferSource();
|
||||||
|
sourceNode.buffer = audioBuffer;
|
||||||
|
sourceNode.connect(audioContext.destination);
|
||||||
|
sourceNode.start();
|
||||||
|
} catch (error) {
|
||||||
|
alertError(language.errors.httpError + `${error}`)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else if (response.status === 200) {
|
else{
|
||||||
const audioBuffer = await response.arrayBuffer();
|
if(dat.error && dat.error.message){
|
||||||
|
alertError((language.errors.httpError + `${dat.error.message}`))
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
alertError((language.errors.httpError + `${Buffer.from(res.data).toString()}`))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
}
|
||||||
|
case 'novelai': {
|
||||||
|
const audioContext = new AudioContext();
|
||||||
|
if(text === ''){
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
const encodedText = encodeURIComponent(text);
|
||||||
|
const encodedSeed = encodeURIComponent(character.naittsConfig.voice);
|
||||||
|
|
||||||
|
const url = `https://api.novelai.net/ai/generate-voice?text=${encodedText}&voice=-1&seed=${encodedSeed}&opus=false&version=${character.naittsConfig.version}`;
|
||||||
|
|
||||||
|
const response = await globalFetch(url, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
"Authorization": "Bearer " + db.NAIApiKey,
|
||||||
|
},
|
||||||
|
rawResponse: true
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const audioBuffer = response.data.buffer;
|
||||||
audioContext.decodeAudioData(audioBuffer, (decodedData) => {
|
audioContext.decodeAudioData(audioBuffer, (decodedData) => {
|
||||||
const sourceNode = audioContext.createBufferSource();
|
const sourceNode = audioContext.createBufferSource();
|
||||||
sourceNode.buffer = decodedData;
|
sourceNode.buffer = decodedData;
|
||||||
@@ -219,12 +179,56 @@ export async function sayTTS(character:character,text:string) {
|
|||||||
} else {
|
} else {
|
||||||
alertError("Error fetching or decoding audio data");
|
alertError("Error fetching or decoding audio data");
|
||||||
}
|
}
|
||||||
return
|
break;
|
||||||
}
|
}
|
||||||
}
|
case 'huggingface': {
|
||||||
case 'vits':{
|
while(true){
|
||||||
await runVITS(text)
|
if(character.hfTTS.language !== 'en'){
|
||||||
}
|
text = await runTranslator(text, false, 'en', character.hfTTS.language)
|
||||||
|
}
|
||||||
|
const audioContext = new AudioContext();
|
||||||
|
const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
"Authorization": "Bearer " + db.huggingfaceKey,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
inputs: text,
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
|
||||||
|
const json = await response.json()
|
||||||
|
if(json.estimated_time){
|
||||||
|
await sleep(json.estimated_time * 1000)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if(response.status >= 400){
|
||||||
|
alertError(language.errors.httpError + `${await response.text()}`)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
else if (response.status === 200) {
|
||||||
|
const audioBuffer = await response.arrayBuffer();
|
||||||
|
audioContext.decodeAudioData(audioBuffer, (decodedData) => {
|
||||||
|
const sourceNode = audioContext.createBufferSource();
|
||||||
|
sourceNode.buffer = decodedData;
|
||||||
|
sourceNode.connect(audioContext.destination);
|
||||||
|
sourceNode.start();
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
alertError("Error fetching or decoding audio data");
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case 'vits':{
|
||||||
|
await runVITS(text, character.vits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
alertError(`TTS Error: ${error}`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -695,7 +695,8 @@ export interface character{
|
|||||||
hfTTS?: {
|
hfTTS?: {
|
||||||
model: string
|
model: string
|
||||||
language: string
|
language: string
|
||||||
}
|
},
|
||||||
|
vits?: OnnxModelFiles
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1115,6 +1116,7 @@ export function setPreset(db:Database, newPres: botPreset){
|
|||||||
|
|
||||||
import { encode as encodeMsgpack, decode as decodeMsgpack } from "msgpackr";
|
import { encode as encodeMsgpack, decode as decodeMsgpack } from "msgpackr";
|
||||||
import * as fflate from "fflate";
|
import * as fflate from "fflate";
|
||||||
|
import type { OnnxModelFiles } from '../process/embedding/transformers';
|
||||||
|
|
||||||
export async function downloadPreset(id:number){
|
export async function downloadPreset(id:number){
|
||||||
saveCurrentPreset()
|
saveCurrentPreset()
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import { checkOldDomain, checkUpdate } from "../update";
|
|||||||
import { botMakerMode, selectedCharID } from "../stores";
|
import { botMakerMode, selectedCharID } from "../stores";
|
||||||
import { Body, ResponseType, fetch as TauriFetch } from "@tauri-apps/api/http";
|
import { Body, ResponseType, fetch as TauriFetch } from "@tauri-apps/api/http";
|
||||||
import { loadPlugins } from "../plugins/plugins";
|
import { loadPlugins } from "../plugins/plugins";
|
||||||
import { alertConfirm, alertError } from "../alert";
|
import { alertConfirm, alertError, alertNormal } from "../alert";
|
||||||
import { checkDriverInit, syncDrive } from "../drive/drive";
|
import { checkDriverInit, syncDrive } from "../drive/drive";
|
||||||
import { hasher } from "../parser";
|
import { hasher } from "../parser";
|
||||||
import { characterURLImport, hubURL } from "../characterCards";
|
import { characterURLImport, hubURL } from "../characterCards";
|
||||||
@@ -231,6 +231,15 @@ export async function saveAsset(data:Uint8Array, customId:string = '', fileName:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function loadAsset(id:string){
|
||||||
|
if(isTauri){
|
||||||
|
return await readBinaryFile(id,{dir: BaseDirectory.AppData})
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
return await forageStorage.getItem(id) as Uint8Array
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let lastSave = ''
|
let lastSave = ''
|
||||||
|
|
||||||
export async function saveDb(){
|
export async function saveDb(){
|
||||||
@@ -369,6 +378,7 @@ export async function loadData() {
|
|||||||
throw "Your save file is corrupted"
|
throw "Your save file is corrupted"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
await registerSw()
|
||||||
await checkUpdate()
|
await checkUpdate()
|
||||||
await changeFullscreen()
|
await changeFullscreen()
|
||||||
|
|
||||||
@@ -432,15 +442,7 @@ export async function loadData() {
|
|||||||
}
|
}
|
||||||
if(navigator.serviceWorker && (!Capacitor.isNativePlatform())){
|
if(navigator.serviceWorker && (!Capacitor.isNativePlatform())){
|
||||||
usingSw = true
|
usingSw = true
|
||||||
await navigator.serviceWorker.register("/sw.js", {
|
await registerSw()
|
||||||
scope: "/"
|
|
||||||
});
|
|
||||||
|
|
||||||
await sleep(100)
|
|
||||||
const da = await fetch('/sw/init')
|
|
||||||
if(!(da.status >= 200 && da.status < 300)){
|
|
||||||
location.reload()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
usingSw = false
|
usingSw = false
|
||||||
@@ -792,6 +794,20 @@ export async function globalFetch(url:string, arg:{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function registerSw() {
|
||||||
|
await navigator.serviceWorker.register("/sw.js", {
|
||||||
|
scope: "/"
|
||||||
|
});
|
||||||
|
await sleep(100)
|
||||||
|
const da = await fetch('/sw/init')
|
||||||
|
if(!(da.status >= 200 && da.status < 300)){
|
||||||
|
location.reload()
|
||||||
|
}
|
||||||
|
else{
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const re = /\\/g
|
const re = /\\/g
|
||||||
function getBasename(data:string){
|
function getBasename(data:string){
|
||||||
const splited = data.replace(re, '/').split('/')
|
const splited = data.replace(re, '/').split('/')
|
||||||
@@ -833,6 +849,13 @@ export function getUnpargeables(db:Database, uptype:'basename'|'pure' = 'basenam
|
|||||||
addUnparge(em[1])
|
addUnparge(em[1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if(cha.vits){
|
||||||
|
const keys = Object.keys(cha.vits.files)
|
||||||
|
for(const key of keys){
|
||||||
|
const vit = cha.vits.files[key]
|
||||||
|
addUnparge(vit)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1044,7 +1067,7 @@ async function pargeChunks(){
|
|||||||
const assets = await readDir('assets', {dir: BaseDirectory.AppData})
|
const assets = await readDir('assets', {dir: BaseDirectory.AppData})
|
||||||
for(const asset of assets){
|
for(const asset of assets){
|
||||||
const n = getBasename(asset.name)
|
const n = getBasename(asset.name)
|
||||||
if(unpargeable.includes(n) || (!n.endsWith('png'))){
|
if(unpargeable.includes(n)){
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
await removeFile(asset.path)
|
await removeFile(asset.path)
|
||||||
@@ -1054,8 +1077,11 @@ async function pargeChunks(){
|
|||||||
else{
|
else{
|
||||||
const indexes = await forageStorage.keys()
|
const indexes = await forageStorage.keys()
|
||||||
for(const asset of indexes){
|
for(const asset of indexes){
|
||||||
|
if(!asset.startsWith('assets/')){
|
||||||
|
continue
|
||||||
|
}
|
||||||
const n = getBasename(asset)
|
const n = getBasename(asset)
|
||||||
if(unpargeable.includes(n) || (!asset.endsWith(".png"))){
|
if(unpargeable.includes(n)){
|
||||||
}
|
}
|
||||||
else{
|
else{
|
||||||
await forageStorage.removeItem(asset)
|
await forageStorage.removeItem(asset)
|
||||||
|
|||||||
Reference in New Issue
Block a user