mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/kwaroran/RisuAI

Browse Source 
Signed-off-by: hashcoko <hashcoko@gmail.com>
This commit is contained in:
hashcoko
2023-11-28 13:25:05 +09:00
parent 70adcedc46 c50039889b
commit e1f567cc8a
19 changed files with 803 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/ts/characters.ts
View File

@@ -301,6 +301,10 @@ export function characterFormatUpdate(index:number|character){
depth: 0,
prompt: ''
}
cha.hfTTS ??= {
model: '',
language: 'en'
}
if(!cha.newGenData){
cha = updateInlayScreen(cha)
}

2
src/ts/hotkey.ts
View File

@@ -70,7 +70,7 @@ function changeToPreset(num:number){
let db = get(DataBase)
let pres = db.botPresets
if(pres.length > num){
alertToast(`Changed to Preset ${num+1} || Preset name : ${pres[num].name}`)
alertToast(`Changed to Preset: ${pres[num].name}`)
changeToPreset2(num)
}
}

1
src/ts/plugins/embedscript.ts
View File

@@ -1,5 +1,6 @@
import { get } from 'svelte/store'
import type { ScriptMode } from '../process/scripts'
//@ts-ignore
import WorkerUrl from './embedworker?worker&url'
import { DataBase, type Chat, type character, type Message } from '../storage/database'
import { selectedCharID } from '../stores'

7
src/ts/process/embedding/transformers.ts
View File

@@ -52,4 +52,11 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
let result = await extractor(text, { pooling: 'mean', normalize: true });
return result?.data ?? null;
}
export const runTTS = async (text: string) => {
let speaker_embeddings = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
let synthesizer = await pipeline('text-to-speech', 'Xenova/speecht5_tts', { local_files_only: true });
let out = await synthesizer(text, { speaker_embeddings });
return out
}

52
src/ts/process/tts.ts
View File

@@ -1,13 +1,21 @@
import { get } from "svelte/store";
import { alertError } from "../alert";
import { DataBase, type character } from "../storage/database";
import { translateVox } from "../translator/translator";
import { runTranslator, translateVox } from "../translator/translator";
import { globalFetch } from "../storage/globalApi";
import { language } from "src/lang";
import { getCurrentCharacter, sleep } from "../util";
let sourceNode:AudioBufferSourceNode = null
export async function sayTTS(character:character,text:string) {
if(!character){
const v = getCurrentCharacter()
if(v.type === 'group'){
return
}
character = v
}
let db = get(DataBase)
text = text.replace(/\*/g,'')
@@ -162,6 +170,48 @@ export async function sayTTS(character:character,text:string) {
}
break;
}
case 'huggingface': {
while(true){
if(character.hfTTS.language !== 'en'){
text = await runTranslator(text, false, 'en', character.hfTTS.language)
}
const audioContext = new AudioContext();
const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
method: 'POST',
headers: {
"Authorization": "Bearer " + db.huggingfaceKey,
"Content-Type": "application/json",
},
body: JSON.stringify({
inputs: text,
})
});
if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
const json = await response.json()
if(json.estimated_time){
await sleep(json.estimated_time * 1000)
continue
}
}
else if(response.status >= 400){
alertError(language.errors.httpError + `${await response.text()}`)
return
}
else if (response.status === 200) {
const audioBuffer = await response.arrayBuffer();
audioContext.decodeAudioData(audioBuffer, (decodedData) => {
const sourceNode = audioContext.createBufferSource();
sourceNode.buffer = decodedData;
sourceNode.connect(audioContext.destination);
sourceNode.start();
});
} else {
alertError("Error fetching or decoding audio data");
}
return
}
}
}
}

12
src/ts/storage/database.ts
View File

@@ -15,7 +15,7 @@ import type { OobaChatCompletionRequestParams } from '../model/ooba';
export const DataBase = writable({} as any as Database)
export const loadedStore = writable(false)
export let appVer = "1.60.1"
export let appVer = "1.61.0"
export let webAppSubVer = ''
export function setDatabase(data:Database){
@@ -350,6 +350,7 @@ export function setDatabase(data:Database){
data.generationSeed ??= -1
data.newOAIHandle ??= true
data.gptVisionQuality ??= 'low'
data.huggingfaceKey ??= ''
data.reverseProxyOobaArgs ??= {
mode: 'instruct'
}
@@ -522,7 +523,7 @@ export interface Database{
mancerHeader:string
emotionProcesser:'submodel'|'embedding',
showMenuChatList?:boolean,
translatorType:'google'|'deepl'|'submodel'|'none',
translatorType:'google'|'deepl'|'none'|'llm',
NAIadventure?:boolean,
NAIappendName?:boolean,
deeplOptions:{
@@ -542,8 +543,9 @@ export interface Database{
reverseProxyOobaArgs: OobaChatCompletionRequestParams
tpo?:boolean
automark?:boolean
huggingfaceKey:string
allowAllExtentionFiles?:boolean
translatorPrompt:string
}
export interface customscript{
@@ -652,6 +654,10 @@ export interface character{
largePortrait?:boolean
lorePlus?:boolean
inlayViewScreen?:boolean
hfTTS?: {
model: string
language: string
}
}

69
src/ts/translator/translator.ts
View File

@@ -3,8 +3,8 @@ import { translatorPlugin } from "../plugins/plugins"
import { DataBase } from "../storage/database"
import { globalFetch } from "../storage/globalApi"
import { alertError } from "../alert"
import type { OpenAIChat } from "../process"
import { requestChatData } from "../process/request"
import { doingChat } from "../process"
let cache={
origin: [''],
@@ -35,7 +35,7 @@ export async function translate(text:string, reverse:boolean) {
return runTranslator(text, reverse, db.translator,db.aiModel.startsWith('novellist') ? 'ja' : 'en')
}
async function runTranslator(text:string, reverse:boolean, from:string,target:'en'|'ja') {
export async function runTranslator(text:string, reverse:boolean, from:string,target:string) {
const arg = {
from: reverse ? from : target,
@@ -100,19 +100,23 @@ async function runTranslator(text:string, reverse:boolean, from:string,target:'e
async function translateMain(text:string, arg:{from:string, to:string, host:string}){
let db = get(DataBase)
if(db.translatorType === 'llm'){
const tr = db.translator || 'en'
return translateLLM(text, {to: tr})
}
if(db.translatorType === 'deepl'){
//deepl raise error 525 because of cloudflare
const body = {
text: [text],
source_lang: arg.from.toLocaleUpperCase(),
target_lang: arg.to.toLocaleUpperCase(),
}
let url = db.deeplOptions.freeApi ? "https://api-free.deepl.com/v2/translate" : "https://api.deepl.com/v2/translate"
const f = await globalFetch(url, {
headers: {
"Authorization": "DeepL-Auth-Key " + db.deeplOptions.key,
"Content-Type": "application/json"
},
body: {
text: text,
source_lang: arg.from.toLocaleUpperCase(),
target_lang: arg.to.toLocaleUpperCase(),
}
body: body
})
if(!f.ok){
@@ -197,9 +201,23 @@ async function jaTrans(text:string) {
return await runTranslator(text, true, 'en','ja')
}
export function isExpTranslator(){
const db = get(DataBase)
return db.translatorType === 'llm' || db.translatorType === 'deepl'
}
export async function translateHTML(html: string, reverse:boolean): Promise<string> {
let db = get(DataBase)
let DoingChat = get(doingChat)
if(DoingChat){
if(isExpTranslator()){
return html
}
}
if(db.translatorType === 'llm'){
const tr = db.translator || 'en'
return translateLLM(html, {to: tr})
}
const dom = new DOMParser().parseFromString(html, 'text/html');
console.log(html)
@@ -254,4 +272,35 @@ export async function translateHTML(html: string, reverse:boolean): Promise<stri
// console.log(translatedHTML)
// Return the translated HTML, excluding the outer <body> tags if needed
return translatedHTML
}
let llmCache = new Map<string, string>()
async function translateLLM(text:string, arg:{to:string}){
if(llmCache.has(text)){
return llmCache.get(text)
}
const db = get(DataBase)
let prompt = db.translatorPrompt || `You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation.`
prompt = prompt.replace('{{slot}}', arg.to)
const rq = await requestChatData({
formated: [
{
'role': 'system',
'content': prompt
},
{
'role': 'user',
'content': text
}
],
bias: {},
useStreaming: false,
}, 'submodel')
if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
alertError(`${rq.result}`)
return text
}
llmCache.set(text, rq.result)
return rq.result
}

6
src/ts/util.ts
View File

@@ -365,4 +365,10 @@ export async function decryptBuffer(data:Uint8Array, keys:string){
)
return result
}
export function getCurrentCharacter(){
const db = get(DataBase)
const selectedChar = get(selectedCharID)
return db.characters[selectedChar]
}