mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/kwaroran/RisuAI

Browse Source 
Signed-off-by: hashcoko <hashcoko@gmail.com>
This commit is contained in:
hashcoko
2023-11-28 13:25:05 +09:00
parent 70adcedc46 c50039889b
commit e1f567cc8a
19 changed files with 803 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
package.json
View File

@@ -15,80 +15,80 @@
},
"dependencies": {
"@adobe/css-tools": "4.3.1",
"@aws-crypto/sha256-js": "^5.1.0",
"@aws-crypto/sha256-js": "^5.2.0",
"@dqbd/tiktoken": "^1.0.7",
"@mlc-ai/web-tokenizers": "^0.1.0",
"@smithy/protocol-http": "^3.0.7",
"@smithy/signature-v4": "^2.0.11",
"@mlc-ai/web-tokenizers": "^0.1.2",
"@smithy/protocol-http": "^3.0.10",
"@smithy/signature-v4": "^2.0.16",
"@tauri-apps/api": "1.4.0",
"@types/marked": "^5.0.1",
"@xenova/transformers": "^2.5.0",
"@types/marked": "^5.0.2",
"@xenova/transformers": "^2.9.0",
"blueimp-md5": "^2.19.0",
"body-parser": "^1.20.2",
"buffer": "^6.0.3",
"core-js": "^3.31.1",
"core-js": "^3.33.3",
"cors": "^2.8.5",
"dompurify": "^3.0.5",
"dompurify": "^3.0.6",
"exifr": "^7.1.3",
"express": "^4.18.2",
"fflate": "^0.8.0",
"fflate": "^0.8.1",
"gpt-3-encoder": "^1.1.4",
"gpt3-tokenizer": "^1.1.5",
"html-to-image": "^1.11.11",
"isomorphic-dompurify": "^1.8.0",
"isomorphic-dompurify": "^1.9.0",
"jszip": "^3.10.1",
"libsodium-wrappers-sumo": "^0.7.11",
"libsodium-wrappers-sumo": "^0.7.13",
"localforage": "^1.10.0",
"lodash": "^4.17.21",
"lucide-svelte": "^0.292.0",
"marked": "^5.1.1",
"marked": "^5.1.2",
"ml-distance": "^4.0.1",
"mobile-drag-drop": "3.0.0-rc.0",
"msgpackr": "^1.9.5",
"node-html-parser": "^6.1.5",
"msgpackr": "^1.9.9",
"node-html-parser": "^6.1.11",
"peerjs": "^1.5.1",
"png-chunk-text": "^1.0.0",
"png-chunks-encode": "^1.0.0",
"png-chunks-extract": "^1.0.0",
"pngjs": "^7.0.0",
"rollup": "^3.26.3",
"rollup": "^3.29.4",
"showdown": "^2.1.0",
"sortablejs": "^1.15.0",
"three": "^0.154.0",
"tippy.js": "^6.3.7",
"uuid": "^9.0.0",
"wasmoon": "^1.15.0",
"uuid": "^9.0.1",
"wasmoon": "^1.15.1",
"web-streams-polyfill": "^3.2.1",
"yuso": "^0.1.3"
},
"devDependencies": {
"@sveltejs/vite-plugin-svelte": "^2.4.2",
"@tailwindcss/typography": "^0.5.9",
"@sveltejs/vite-plugin-svelte": "^2.5.3",
"@tailwindcss/typography": "^0.5.10",
"@tauri-apps/cli": "1.4.0",
"@tsconfig/svelte": "^3.0.0",
"@types/blueimp-md5": "^2.18.0",
"@types/dompurify": "^3.0.2",
"@types/libsodium-wrappers-sumo": "^0.7.5",
"@types/lodash": "^4.14.195",
"@types/lodash.clonedeep": "^4.5.7",
"@types/lodash.isequal": "^4.5.6",
"@types/node": "^18.16.19",
"@types/pngjs": "^6.0.1",
"@types/showdown": "^2.0.1",
"@types/sortablejs": "^1.15.1",
"@types/blueimp-md5": "^2.18.2",
"@types/dompurify": "^3.0.5",
"@types/libsodium-wrappers-sumo": "^0.7.8",
"@types/lodash": "^4.14.202",
"@types/lodash.clonedeep": "^4.5.9",
"@types/lodash.isequal": "^4.5.8",
"@types/node": "^18.18.13",
"@types/pngjs": "^6.0.4",
"@types/showdown": "^2.0.6",
"@types/sortablejs": "^1.15.7",
"@types/three": "^0.154.0",
"@types/uuid": "^9.0.2",
"@types/wicg-file-system-access": "^2020.9.6",
"autoprefixer": "^10.4.14",
"@types/uuid": "^9.0.7",
"@types/wicg-file-system-access": "^2020.9.8",
"autoprefixer": "^10.4.16",
"internal-ip": "^7.0.0",
"postcss": "^8.4.26",
"svelte": "^4.1.0",
"svelte-check": "^3.4.6",
"svelte-preprocess": "^5.0.4",
"tailwindcss": "^3.3.3",
"tslib": "^2.6.0",
"typescript": "^5.1.6",
"vite": "^4.4.5",
"postcss": "^8.4.31",
"svelte": "^4.2.7",
"svelte-check": "^3.6.2",
"svelte-preprocess": "^5.1.1",
"tailwindcss": "^3.3.5",
"tslib": "^2.6.2",
"typescript": "^5.3.2",
"vite": "^4.5.0",
"vite-plugin-top-level-await": "^1.3.1",
"vite-plugin-wasm": "^3.2.2"
}

1076
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

2
src-tauri/tauri.conf.json
View File

@@ -8,7 +8,7 @@
},
"package": {
"productName": "RisuAI",
"version": "1.60.1"
"version": "1.61.0"
},
"tauri": {
"allowlist": {

4
src/lang/en.ts
View File

@@ -450,6 +450,6 @@ export const languageEnglish = {
imgGenPrompt: "Image Generation Prompt",
imgGenNegatives: "Image Generation Negative Prompt",
imgGenInstructions: "Image Generation Instructions",
usePlainFetchWarn: "Please disable this option when using NovelAI, as it can cause CORS errors when using NovelAI."
usePlainFetchWarn: "Please disable this option when using NovelAI, as it can cause CORS errors when using NovelAI.",
translationPrompt: "Translation Prompt",
}

9
src/lib/ChatScreens/Chat.svelte
View File

@@ -1,5 +1,5 @@
<script lang="ts">
import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon } from "lucide-svelte";
import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon, Volume2Icon } from "lucide-svelte";
import { ParseMarkdown, type simpleCharacterArgument } from "../../ts/parser";
import AutoresizeArea from "../UI/GUI/TextAreaResizable.svelte";
import { alertConfirm, alertError } from "../../ts/alert";
@@ -10,6 +10,7 @@
import { risuChatParser } from "src/ts/process/scripts";
import { get } from "svelte/store";
import { isEqual } from "lodash";
import { sayTTS } from "src/ts/process/tts";
export let message = ''
export let name = ''
export let largePortrait = false
@@ -145,6 +146,12 @@
</button>
{/if}
{#if idx > -1}
<button class="ml-2 hover:text-green-500 transition-colors" on:click={()=>{
return sayTTS(null, message)
}}>
<Volume2Icon size={20}/>
</button>
<button class={"ml-2 hover:text-green-500 transition-colors "+(editMode?'text-green-400':'')} on:click={() => {
if(!editMode){
editMode = true

30
src/lib/ChatScreens/DefaultChatScreen.svelte
View File

@@ -8,7 +8,7 @@
import { doingChat, sendChat } from "../../ts/process/index";
import { findCharacterbyId, messageForm, sleep } from "../../ts/util";
import { language } from "../../lang";
import { translate } from "../../ts/translator/translator";
import { isExpTranslator, translate } from "../../ts/translator/translator";
import { alertError, alertNormal, alertWait } from "../../ts/alert";
import sendSound from '../../etc/send.mp3'
import {cloneDeep} from 'lodash'
@@ -251,7 +251,33 @@
$: updateInputSizeAll()
function updateInputTransateMessage(reverse: boolean) {
async function updateInputTransateMessage(reverse: boolean) {
if(isExpTranslator()){
if(!reverse){
messageInputTranslate = ''
return
}
if(messageInputTranslate === '') {
messageInput = ''
return
}
const lastMessageInputTranslate = messageInputTranslate
await sleep(1500)
if(lastMessageInputTranslate === messageInputTranslate){
console.log(lastMessageInputTranslate === messageInputTranslate)
console.log(lastMessageInputTranslate, messageInputTranslate)
translate(reverse ? messageInputTranslate : messageInput, reverse).then((translatedMessage) => {
if(translatedMessage){
if(reverse)
messageInput = translatedMessage
else
messageInputTranslate = translatedMessage
}
})
}
return
}
if(reverse && messageInputTranslate === '') {
messageInput = ''
return

1
src/lib/Setting/Pages/BotSettings.svelte
View File

@@ -149,6 +149,7 @@
<OptionInput value="gpt35_0613">GPT-3.5 0613</OptionInput>
<OptionInput value="gpt4_0613">GPT-4 0613</OptionInput>
<OptionInput value="claude-2.1">claude-2.1</OptionInput>
<OptionInput value="claude-2.0">claude-2.0</OptionInput>
<OptionInput value="claude-2">claude-2</OptionInput>
<OptionInput value="claude-v1.3">claude-v1.3</OptionInput>
<OptionInput value="claude-v1.3-100k">claude-v1.3-100k</OptionInput>

8
src/lib/Setting/Pages/LanguageSettings.svelte
View File

@@ -9,6 +9,7 @@
import { downloadFile } from "src/ts/storage/globalApi";
import { languageEnglish } from "src/lang/en";
import TextInput from "src/lib/UI/GUI/TextInput.svelte";
import TextAreaInput from "src/lib/UI/GUI/TextAreaInput.svelte";
let langChanged = false
</script>
@@ -56,7 +57,7 @@
<SelectInput className="mt-2 mb-4" bind:value={$DataBase.translatorType}>
<OptionInput value="google" >Google</OptionInput>
<OptionInput value="deepl" >DeepL</OptionInput>
<OptionInput value="submodel" >Use submodel</OptionInput>
<OptionInput value="llm" >Ax. Model</OptionInput>
</SelectInput>
{#if $DataBase.translatorType === 'deepl'}
@@ -75,6 +76,11 @@
<span class="text-draculared text-xs mb-2">Translation sends a lot of requests at once, so don't use it at reverse proxy.</span>
{/if}
{#if $DataBase.translatorType === 'llm'}
<span class="text-textcolor mt-4">{language.translationPrompt}</span>
<TextAreaInput bind:value={$DataBase.translatorPrompt} placeholder={"You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation."}/>
{/if}
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.autoTranslate} name={language.autoTranslation}/>

7
src/lib/Setting/Pages/OtherBotSettings.svelte
View File

@@ -146,10 +146,15 @@
<span class="text-textcolor mt-2">VOICEVOX URL</span>
<TextInput size="sm" marginBottom bind:value={$DataBase.voicevoxUrl}/>
<span class="text-textcolor">OpenAI Key</span>
<TextInput size="sm" marginBottom bind:value={$DataBase.openAIKey}/>
<span class="text-textcolor mt-2">NovelAI API key</span>
<TextInput size="sm" marginBottom placeholder="pst-..." bind:value={$DataBase.NAIApiKey}/>
<span class="text-textcolor">Huggingface Key</span>
<TextInput size="sm" marginBottom bind:value={$DataBase.huggingfaceKey} placeholder="hf_..."/>
</Arcodion>
<Arcodion name={language.emotionImage} styled>

16
src/lib/SideBars/CharConfig.svelte
View File

@@ -519,6 +519,7 @@
<OptionInput value="VOICEVOX">VOICEVOX</OptionInput>
<OptionInput value="openai">OpenAI</OptionInput>
<OptionInput value="novelai">NovelAI</OptionInput>
<OptionInput value="huggingface">Huggingface</OptionInput>
</SelectInput>
@@ -604,7 +605,6 @@
</SelectInput>
{/if}
{#if currentChar.data.ttsMode === 'openai'}
<span class="text-textcolor">OpenAI TTS uses your OpenAI key on the chat model section</span>
<SelectInput className="mb-4 mt-2" bind:value={currentChar.data.oaiVoice}>
<OptionInput value="">Unset</OptionInput>
{#each oaiVoices as voice}
@@ -612,7 +612,19 @@
{/each}
</SelectInput>
{/if}
{#if currentChar.data.ttsMode === 'webspeech' || currentChar.data.ttsMode === 'elevenlab' || currentChar.data.ttsMode === 'VOICEVOX' || currentChar.data.ttsMode === 'novelai'}
{#if currentChar.data.ttsMode === 'huggingface'}
<span class="text-textcolor">Model</span>
<TextInput additionalClass="mb-4 mt-2" bind:value={currentChar.data.hfTTS.model} />
<span class="text-textcolor">Language</span>
<TextInput additionalClass="mb-4 mt-2" bind:value={currentChar.data.hfTTS.language} placeholder="en" />
{/if}
{#if currentChar.data.ttsMode === 'webspeech' ||
currentChar.data.ttsMode === 'elevenlab' ||
currentChar.data.ttsMode === 'VOICEVOX' ||
currentChar.data.ttsMode === 'huggingface' ||
currentChar.data.ttsMode === 'openai' ||
currentChar.data.ttsMode === 'novelai'}
<div class="flex items-center mt-2">
<Check bind:check={currentChar.data.ttsReadOnlyQuoted} name={language.ttsReadOnlyQuoted}/>
</div>

4
src/ts/characters.ts
View File

@@ -301,6 +301,10 @@ export function characterFormatUpdate(index:number|character){
depth: 0,
prompt: ''
}
cha.hfTTS ??= {
model: '',
language: 'en'
}
if(!cha.newGenData){
cha = updateInlayScreen(cha)
}

2
src/ts/hotkey.ts
View File

@@ -70,7 +70,7 @@ function changeToPreset(num:number){
let db = get(DataBase)
let pres = db.botPresets
if(pres.length > num){
alertToast(`Changed to Preset ${num+1} || Preset name : ${pres[num].name}`)
alertToast(`Changed to Preset: ${pres[num].name}`)
changeToPreset2(num)
}
}

1
src/ts/plugins/embedscript.ts
View File

@@ -1,5 +1,6 @@
import { get } from 'svelte/store'
import type { ScriptMode } from '../process/scripts'
//@ts-ignore
import WorkerUrl from './embedworker?worker&url'
import { DataBase, type Chat, type character, type Message } from '../storage/database'
import { selectedCharID } from '../stores'

7
src/ts/process/embedding/transformers.ts
View File

@@ -52,4 +52,11 @@ export const runEmbedding = async (text: string):Promise<Float32Array> => {
let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
let result = await extractor(text, { pooling: 'mean', normalize: true });
return result?.data ?? null;
}
export const runTTS = async (text: string) => {
let speaker_embeddings = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/speaker_embeddings.bin';
let synthesizer = await pipeline('text-to-speech', 'Xenova/speecht5_tts', { local_files_only: true });
let out = await synthesizer(text, { speaker_embeddings });
return out
}

52
src/ts/process/tts.ts
View File

@@ -1,13 +1,21 @@
import { get } from "svelte/store";
import { alertError } from "../alert";
import { DataBase, type character } from "../storage/database";
import { translateVox } from "../translator/translator";
import { runTranslator, translateVox } from "../translator/translator";
import { globalFetch } from "../storage/globalApi";
import { language } from "src/lang";
import { getCurrentCharacter, sleep } from "../util";
let sourceNode:AudioBufferSourceNode = null
export async function sayTTS(character:character,text:string) {
if(!character){
const v = getCurrentCharacter()
if(v.type === 'group'){
return
}
character = v
}
let db = get(DataBase)
text = text.replace(/\*/g,'')
@@ -162,6 +170,48 @@ export async function sayTTS(character:character,text:string) {
}
break;
}
case 'huggingface': {
while(true){
if(character.hfTTS.language !== 'en'){
text = await runTranslator(text, false, 'en', character.hfTTS.language)
}
const audioContext = new AudioContext();
const response = await fetch(`https://api-inference.huggingface.co/models/${character.hfTTS.model}`, {
method: 'POST',
headers: {
"Authorization": "Bearer " + db.huggingfaceKey,
"Content-Type": "application/json",
},
body: JSON.stringify({
inputs: text,
})
});
if(response.status === 503 && response.headers.get('content-type') === 'application/json'){
const json = await response.json()
if(json.estimated_time){
await sleep(json.estimated_time * 1000)
continue
}
}
else if(response.status >= 400){
alertError(language.errors.httpError + `${await response.text()}`)
return
}
else if (response.status === 200) {
const audioBuffer = await response.arrayBuffer();
audioContext.decodeAudioData(audioBuffer, (decodedData) => {
const sourceNode = audioContext.createBufferSource();
sourceNode.buffer = decodedData;
sourceNode.connect(audioContext.destination);
sourceNode.start();
});
} else {
alertError("Error fetching or decoding audio data");
}
return
}
}
}
}

12
src/ts/storage/database.ts
View File

@@ -15,7 +15,7 @@ import type { OobaChatCompletionRequestParams } from '../model/ooba';
export const DataBase = writable({} as any as Database)
export const loadedStore = writable(false)
export let appVer = "1.60.1"
export let appVer = "1.61.0"
export let webAppSubVer = ''
export function setDatabase(data:Database){
@@ -350,6 +350,7 @@ export function setDatabase(data:Database){
data.generationSeed ??= -1
data.newOAIHandle ??= true
data.gptVisionQuality ??= 'low'
data.huggingfaceKey ??= ''
data.reverseProxyOobaArgs ??= {
mode: 'instruct'
}
@@ -522,7 +523,7 @@ export interface Database{
mancerHeader:string
emotionProcesser:'submodel'|'embedding',
showMenuChatList?:boolean,
translatorType:'google'|'deepl'|'submodel'|'none',
translatorType:'google'|'deepl'|'none'|'llm',
NAIadventure?:boolean,
NAIappendName?:boolean,
deeplOptions:{
@@ -542,8 +543,9 @@ export interface Database{
reverseProxyOobaArgs: OobaChatCompletionRequestParams
tpo?:boolean
automark?:boolean
huggingfaceKey:string
allowAllExtentionFiles?:boolean
translatorPrompt:string
}
export interface customscript{
@@ -652,6 +654,10 @@ export interface character{
largePortrait?:boolean
lorePlus?:boolean
inlayViewScreen?:boolean
hfTTS?: {
model: string
language: string
}
}

69
src/ts/translator/translator.ts
View File

@@ -3,8 +3,8 @@ import { translatorPlugin } from "../plugins/plugins"
import { DataBase } from "../storage/database"
import { globalFetch } from "../storage/globalApi"
import { alertError } from "../alert"
import type { OpenAIChat } from "../process"
import { requestChatData } from "../process/request"
import { doingChat } from "../process"
let cache={
origin: [''],
@@ -35,7 +35,7 @@ export async function translate(text:string, reverse:boolean) {
return runTranslator(text, reverse, db.translator,db.aiModel.startsWith('novellist') ? 'ja' : 'en')
}
async function runTranslator(text:string, reverse:boolean, from:string,target:'en'|'ja') {
export async function runTranslator(text:string, reverse:boolean, from:string,target:string) {
const arg = {
from: reverse ? from : target,
@@ -100,19 +100,23 @@ async function runTranslator(text:string, reverse:boolean, from:string,target:'e
async function translateMain(text:string, arg:{from:string, to:string, host:string}){
let db = get(DataBase)
if(db.translatorType === 'llm'){
const tr = db.translator || 'en'
return translateLLM(text, {to: tr})
}
if(db.translatorType === 'deepl'){
//deepl raise error 525 because of cloudflare
const body = {
text: [text],
source_lang: arg.from.toLocaleUpperCase(),
target_lang: arg.to.toLocaleUpperCase(),
}
let url = db.deeplOptions.freeApi ? "https://api-free.deepl.com/v2/translate" : "https://api.deepl.com/v2/translate"
const f = await globalFetch(url, {
headers: {
"Authorization": "DeepL-Auth-Key " + db.deeplOptions.key,
"Content-Type": "application/json"
},
body: {
text: text,
source_lang: arg.from.toLocaleUpperCase(),
target_lang: arg.to.toLocaleUpperCase(),
}
body: body
})
if(!f.ok){
@@ -197,9 +201,23 @@ async function jaTrans(text:string) {
return await runTranslator(text, true, 'en','ja')
}
export function isExpTranslator(){
const db = get(DataBase)
return db.translatorType === 'llm' || db.translatorType === 'deepl'
}
export async function translateHTML(html: string, reverse:boolean): Promise<string> {
let db = get(DataBase)
let DoingChat = get(doingChat)
if(DoingChat){
if(isExpTranslator()){
return html
}
}
if(db.translatorType === 'llm'){
const tr = db.translator || 'en'
return translateLLM(html, {to: tr})
}
const dom = new DOMParser().parseFromString(html, 'text/html');
console.log(html)
@@ -254,4 +272,35 @@ export async function translateHTML(html: string, reverse:boolean): Promise<stri
// console.log(translatedHTML)
// Return the translated HTML, excluding the outer <body> tags if needed
return translatedHTML
}
let llmCache = new Map<string, string>()
async function translateLLM(text:string, arg:{to:string}){
if(llmCache.has(text)){
return llmCache.get(text)
}
const db = get(DataBase)
let prompt = db.translatorPrompt || `You are a translator. translate the following html or text into {{slot}}. do not output anything other than the translation.`
prompt = prompt.replace('{{slot}}', arg.to)
const rq = await requestChatData({
formated: [
{
'role': 'system',
'content': prompt
},
{
'role': 'user',
'content': text
}
],
bias: {},
useStreaming: false,
}, 'submodel')
if(rq.type === 'fail' || rq.type === 'streaming' || rq.type === 'multiline'){
alertError(`${rq.result}`)
return text
}
llmCache.set(text, rq.result)
return rq.result
}

6
src/ts/util.ts
View File

@@ -365,4 +365,10 @@ export async function decryptBuffer(data:Uint8Array, keys:string){
)
return result
}
export function getCurrentCharacter(){
const db = get(DataBase)
const selectedChar = get(selectedCharID)
return db.characters[selectedChar]
}

2
version.json
View File

@@ -1 +1 @@
{"version":"1.60.1"}
{"version":"1.61.0"}