Add Novel AI Diffusion and TTS (#260)

Signed-off-by: hashcoko <hashcoko@gmail.com>

# PR Checklist
- [ V ] Did you check if it works normally in all models? *ignore this
when it dosen't uses models*
- [ V ] Did you check if it works normally in all of web, local and node
hosted versions? if it dosen't, did you blocked it in those versions?
- [ V ] Did you added a type def?

# Description

package.json

@@ -36,6 +36,7 @@
     "gpt3-tokenizer": "^1.1.5",
     "html-to-image": "^1.11.11",
     "isomorphic-dompurify": "^1.8.0",
+    "jszip": "^3.10.1",
     "libsodium-wrappers-sumo": "^0.7.11",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
@@ -91,4 +92,4 @@
     "vite-plugin-top-level-await": "^1.3.1",
     "vite-plugin-wasm": "^3.2.2"
   }
-}
+}

server/node/server.cjs

@@ -61,7 +61,6 @@ const reverseProxyFunc = async (req, res, next) => {
             headers: header,
             body: JSON.stringify(req.body)
         });
-
         // get response body as stream
         const originalBody = originalResponse.body;
         // get response headers

src/lib/Setting/Pages/OtherBotSettings.svelte

@@ -2,12 +2,16 @@
     import Check from "src/lib/UI/GUI/CheckInput.svelte";
     import { language } from "src/lang";
     import Help from "src/lib/Others/Help.svelte";
+    import { selectSingleFile } from "src/ts/util";
     import { DataBase } from "src/ts/storage/database";
     import { isTauri } from "src/ts/storage/globalApi";
     import NumberInput from "src/lib/UI/GUI/NumberInput.svelte";
     import TextInput from "src/lib/UI/GUI/TextInput.svelte";
     import SelectInput from "src/lib/UI/GUI/SelectInput.svelte";
     import OptionInput from "src/lib/UI/GUI/OptionInput.svelte";
+    import SliderInput from "src/lib/UI/GUI/SliderInput.svelte";
+    import Button from "src/lib/UI/GUI/Button.svelte";
+    import { convertToBase64 } from "src/ts/process/uinttobase64";
 
 </script>
 <h2 class="mb-2 text-2xl font-bold mt-2">{language.otherBots}</h2>
@@ -18,6 +22,7 @@
 <SelectInput className="mt-2 mb-4" bind:value={$DataBase.sdProvider}>
     <OptionInput value="" >None</OptionInput>
     <OptionInput value="webui" >Stable Diffusion WebUI</OptionInput>
+    <OptionInput value="novelai" >Novel AI</OptionInput>
     <!-- TODO -->
     <!-- <OptionInput value="runpod" >Runpod Serverless</OptionInput> -->
 </SelectInput>
@@ -56,6 +61,65 @@
     {/if}
 {/if}
 
+{#if $DataBase.sdProvider === 'novelai'}
+    <span class="text-textcolor mt-2">Novel AI {language.providerURL}</span>
+    <TextInput size="sm" marginBottom placeholder="https://api.novelai.net" bind:value={$DataBase.NAIImgUrl}/>
+    <span class="text-textcolor">API Key</span>
+    <TextInput size="sm" marginBottom placeholder="pst-..." bind:value={$DataBase.NAIApiKey}/>
+
+    <span class="text-textcolor">Model</span>
+    <TextInput size="sm" marginBottom placeholder="nai-diffusion-3" bind:value={$DataBase.NAIImgModel}/>
+
+    <span class="text-textcolor">Enable I2I</span>
+    <Check bind:check={$DataBase.NAII2I}/>
+
+
+    {#if $DataBase.NAII2I}
+
+        <span class="text-textcolor">strength</span>
+        <SliderInput min={0} max={0.99} step={0.01} bind:value={$DataBase.NAIImgConfig.strength}/>
+        <span class="text-textcolor2 mb-6 text-sm">{$DataBase.NAIImgConfig.strength}</span>
+        <span class="text-textcolor">noise</span>
+        <SliderInput min={0} max={0.99} step={0.01} bind:value={$DataBase.NAIImgConfig.noise}/>
+        <span class="text-textcolor2 mb-6 text-sm">{$DataBase.NAIImgConfig.noise}</span>
+
+        <span class="text-textcolor">base image</span>
+        <TextInput size="sm" marginBottom placeholder="If empty, a profile picture is sent." bind:value={$DataBase.NAIImgConfig.image}/>
+        <span class="text-textcolor">If empty, a profile picture is sent.</span>
+        
+        <Button on:click={async () => {
+            const img = await selectSingleFile([
+                'jpg',
+                'jpeg',
+                'png',
+                'webp'
+            ])
+            if(!img){
+                return null
+            }
+            const base64 = await convertToBase64(img.data)
+            $DataBase.NAIImgConfig.image = base64
+        }}>Select Image</Button>
+    {/if}
+
+    <span class="text-textcolor">Width</span>
+    <NumberInput size="sm" marginBottom min={0} max={2048} bind:value={$DataBase.NAIImgConfig.width}/>
+    <span class="text-textcolor">Height</span>
+    <NumberInput size="sm" marginBottom min={0} max={2048} bind:value={$DataBase.NAIImgConfig.height}/>
+    <span class="text-textcolor">Sampler</span>
+    <TextInput size="sm" marginBottom bind:value={$DataBase.NAIImgConfig.sampler}/>
+    <span class="text-textcolor">steps</span>
+    <NumberInput size="sm" marginBottom min={0} max={2048} bind:value={$DataBase.NAIImgConfig.steps}/>
+    <span class="text-textcolor">CFG scale</span>
+    <NumberInput size="sm" marginBottom min={0} max={2048} bind:value={$DataBase.NAIImgConfig.scale}/>
+
+    {#if !$DataBase.NAII2I}
+        <span class="text-textcolor">Use SMEA</span>
+        <Check bind:check={$DataBase.NAIImgConfig.sm}/>
+        <span class="text-textcolor">Use DYN</span>
+        <Check bind:check={$DataBase.NAIImgConfig.sm_dyn}/>
+    {/if}
+{/if}
 
 <span class="text-textcolor mt-4 text-lg font-bold">TTS</span>
 <span class="text-textcolor mt-2">ElevenLabs API key</span>
@@ -64,6 +128,9 @@
 <span class="text-textcolor mt-2">VOICEVOX URL</span>
 <TextInput size="sm" marginBottom bind:value={$DataBase.voicevoxUrl}/>
 
+<span class="text-textcolor mt-2">NovelAI API key</span>
+<TextInput size="sm" marginBottom placeholder="pst-..." bind:value={$DataBase.NAIApiKey}/>
+
 <span class="text-textcolor mt-4 text-lg font-bold">{language.emotionImage}</span>
 
 <span class="text-textcolor mt-2">{language.emotionMethod}</span>

src/lib/SideBars/CharConfig.svelte

@@ -15,7 +15,7 @@
     import Help from "../Others/Help.svelte";
     import RegexData from "./Scripts/RegexData.svelte";
     import { exportChar, shareRisuHub } from "src/ts/characterCards";
-    import { getElevenTTSVoices, getWebSpeechTTSVoices, getVOICEVOXVoices, oaiVoices } from "src/ts/process/tts";
+    import { getElevenTTSVoices, getWebSpeechTTSVoices, getVOICEVOXVoices, oaiVoices, getNovelAIVoices, FixNAITTS } from "src/ts/process/tts";
     import { checkCharOrder, getFileSrc } from "src/ts/storage/globalApi";
     import { addGroupChar, rmCharFromGroup } from "src/ts/process/group";
     import RealmUpload from "../UI/Realm/RealmUpload.svelte";
@@ -27,6 +27,7 @@
     import OptionInput from "../UI/GUI/OptionInput.svelte";
     import RegexList from "./Scripts/RegexList.svelte";
     import TriggerList from "./Scripts/TriggerList.svelte";
+    
 
     let subMenu = 0
     let openHubUpload = false
@@ -130,12 +131,20 @@
                 }
             }
         }
+        
     }
 
     onDestroy(unsub);
 
 
     $:licensed = (currentChar.type === 'character') ? currentChar.data.license : ''
+    $: if (currentChar.data.ttsMode === 'novelai' && (currentChar.data as character).naittsConfig === undefined) {
+        (currentChar.data as character).naittsConfig = {
+            customvoice: false,
+            voice: 'Aini',
+            version: 'v2'
+        };
+    }
 </script>
 
 {#if licensed !== 'private'}
@@ -542,6 +551,7 @@
             <OptionInput value="webspeech">Web Speech</OptionInput>
             <OptionInput value="VOICEVOX">VOICEVOX</OptionInput>
             <OptionInput value="openai">OpenAI</OptionInput>
+            <OptionInput value="novelai">NovelAI</OptionInput>
         </SelectInput>
         
 
@@ -600,6 +610,31 @@
                 <span class="text-textcolor">Intonation scale</span>
                 <NumberInput size={"sm"} marginBottom bind:value={currentChar.data.voicevoxConfig.INTONATION_SCALE}/>
                 <span class="text-sm mb-2 text-textcolor2">To use VOICEVOX, you need to run a colab and put the localtunnel URL in "Settings → Other Bots". https://colab.research.google.com/drive/1tyeXJSklNfjW-aZJAib1JfgOMFarAwze</span>
+        {:else if currentChar.data.ttsMode === 'novelai'}
+            <span class="text-textcolor">Custom Voice Seed</span>
+            <Check bind:check={currentChar.data.naittsConfig.customvoice}/>
+            {#if !currentChar.data.naittsConfig.customvoice}
+                <span class="text-textcolor">Voice</span>
+                <SelectInput className="mb-4 mt-2" bind:value={currentChar.data.naittsConfig.voice}>
+                    {#await getNovelAIVoices() then voices}
+                        {#each voices as voiceGroup}
+                            <optgroup label={voiceGroup.gender} class="bg-darkbg appearance-none">
+                                {#each voiceGroup.voices as voice}
+                                    <OptionInput value={voice} selected={currentChar.data.naittsConfig.voice === voice}>{voice}</OptionInput>
+                                {/each}
+                            </optgroup>
+                        {/each}
+                    {/await}
+                </SelectInput>
+            {:else}
+                <span class="text-textcolor">Voice</span>
+                <TextInput size={"sm"} bind:value={currentChar.data.naittsConfig.voice}/>
+            {/if}
+            <span class="text-textcolor">Version</span>
+            <SelectInput className="mb-4 mt-2" bind:value={currentChar.data.naittsConfig.version}>
+                <OptionInput value="v1">v1</OptionInput>
+                <OptionInput value="v2">v2</OptionInput>
+            </SelectInput>
         {/if}
         {#if currentChar.data.ttsMode === 'openai'}
         <span class="text-textcolor">OpenAI TTS uses your OpenAI key on the chat model section</span>
@@ -610,7 +645,7 @@
                 {/each}
             </SelectInput>
         {/if}
-        {#if currentChar.data.ttsMode === 'webspeech' || currentChar.data.ttsMode === 'elevenlab' || currentChar.data.ttsMode === 'VOICEVOX'}
+        {#if currentChar.data.ttsMode === 'webspeech' || currentChar.data.ttsMode === 'elevenlab' || currentChar.data.ttsMode === 'VOICEVOX' || currentChar.data.ttsMode === 'novelai'}
             <div class="flex items-center mt-2">
                 <Check bind:check={currentChar.data.ttsReadOnlyQuoted} name={language.ttsReadOnlyQuoted}/>
             </div>

src/ts/process/generateSeed.ts

@@ -0,0 +1,9 @@
+export function generateRandomSeed(length) {
+    let result = '';
+    const characters = '0123456789';
+    const charactersLength = characters.length;
+    for (let i = 0; i < length; i++) {
+        result += characters.charAt(Math.floor(Math.random() * charactersLength));
+    }
+    return result;
+}

src/ts/process/processzip.ts

@@ -0,0 +1,15 @@
+import JSZip from "jszip";
+
+export async function processZip(dataArray: Uint8Array): Promise<string> {
+    const blob = new Blob([dataArray], { type: "application/zip" });
+    const zip = new JSZip();
+    const zipData = await zip.loadAsync(blob);
+
+    const imageFile = Object.keys(zipData.files).find(fileName => /\.(jpg|jpeg|png)$/.test(fileName));
+    if (imageFile) {
+        const imageData = await zipData.files[imageFile].async("base64");
+        return `data:image/png;base64,${imageData}`;
+    } else {
+        throw new Error("No image found in ZIP file");
+    }
+}

src/ts/process/stableDiff.ts

@@ -2,11 +2,13 @@ import { get } from "svelte/store"
 import { DataBase, type character } from "../storage/database"
 import { requestChatData } from "./request"
 import { alertError } from "../alert"
-import { globalFetch } from "../storage/globalApi"
+import { globalFetch, readImage } from "../storage/globalApi"
 import { CharEmotion } from "../stores"
 import type { OpenAIChat } from "."
-
+import { processZip } from "./processzip"
-
+import { convertToBase64 } from "./uinttobase64" 
+import type { List } from "lodash"
+import { generateRandomSeed } from "./generateSeed"
 export async function stableDiff(currentChar:character,prompt:string){
     const mainPrompt = "assistant is a chat analyzer.\nuser will input a data of situation with key and values before chat, and a chat of a user and character.\nView the status of the chat and change the data.\nif data's key starts with $, it must change it every time.\nif data value is none, it must change it."
     let db = get(DataBase)
@@ -129,7 +131,7 @@ export async function stableDiff(currentChar:character,prompt:string){
                     "cfg_scale": db.sdCFG,
                     "prompt": prompts.join(','),
                     "negative_prompt": neg,
-                    'sampler_name': db.sdConfig.sampler_name,
+                    "sampler_name": db.sdConfig.sampler_name,
                     "enable_hr": db.sdConfig.enable_hr,
                     "denoising_strength": db.sdConfig.denoising_strength,
                     "hr_scale": db.sdConfig.hr_scale,
@@ -161,6 +163,113 @@ export async function stableDiff(currentChar:character,prompt:string){
             return false   
         }
     }
+    if(db.sdProvider === 'novelai'){
 
+        let prompts:string[] = []
+        let neg = ''
+        for(let i=0;i<currentSd.length;i++){
+            if(currentSd[i][0] !== 'negative'){
+                prompts.push(currentSd[i][1])
+            }
+            else{
+                neg = currentSd[i][1]
+            }
+        }
+
+
+
+        let reqlist= {}
+
+        if(db.NAII2I){
+            let base64img = ''
+            if(db.NAIImgConfig.image === ''){
+                const charimg = currentChar.image;
+                
+                const img = await readImage(charimg)
+                const base64 = await convertToBase64(img);
+                base64img = base64.split('base64,')[1];
+            }else{
+                base64img = db.NAIImgConfig.image.split('base64,')[1];
+            }
+            
+            let randomseed = generateRandomSeed(10);
+            let seed = parseInt(randomseed, 10);
+            reqlist = {
+                body: {
+                    "action": "img2img",
+                    "input": prompts.join(','),
+                    "model": db.NAIImgModel,
+                    "parameters": {
+                        "seed": seed,
+                        "extra_noise_seed": seed,
+                        "add_original_image": false,
+                        "cfg_rescale": 0,
+                        "controlnet_strength": 1,
+                        "dynamic_threshold": false,
+                        "n_samples": 1,
+                        "width": db.NAIImgConfig.width,
+                        "height": db.NAIImgConfig.height,
+                        "sampler": db.NAIImgConfig.sampler,
+                        "steps": db.NAIImgConfig.steps,
+                        "scale": db.NAIImgConfig.scale,
+                        "negative_prompt": neg,
+                        "sm": false,
+                        "sm_dyn": false,
+                        "noise": db.NAIImgConfig.noise,
+                        "noise_schedule": "native",
+                        "strength": db.NAIImgConfig.strength,
+                        "image": base64img,
+                        "ucPreset": 2,
+                        "uncond_scale": 1
+                    }
+                },
+                headers:{
+                    "Authorization": "Bearer " + db.NAIApiKey
+                }
+            }
+        }else{
+            reqlist = {
+                body: {
+                    "input": prompts.join(','),
+                    "model": db.NAIImgModel,
+                    "parameters": {
+                        "width": db.NAIImgConfig.width,
+                        "height": db.NAIImgConfig.height,
+                        "sampler": db.NAIImgConfig.sampler,
+                        "steps": db.NAIImgConfig.steps,
+                        "scale": db.NAIImgConfig.scale,
+                        "negative_prompt": neg,
+                        "sm": db.NAIImgConfig.sm,
+                        "sm_dyn": db.NAIImgConfig.sm_dyn
+                    }
+                },
+                headers:{
+                    "Authorization": "Bearer " + db.NAIApiKey
+                }
+            }
+        }
+        try {
+            const da = await globalFetch(db.NAIImgUrl, reqlist)   
+
+            if(da){
+                let charemotions = get(CharEmotion)
+                const img = await processZip(da.data);
+                const emos:[string, string,number][] = [[img, img, Date.now()]]
+                charemotions[currentChar.chaId] = emos
+                CharEmotion.set(charemotions)
+            }
+            else{
+                alertError(JSON.stringify(da.data))
+                return false   
+            }
+
+            return returnSdData
+
+
+        } catch (error) {
+            alertError(error)
+            return false   
+        }
+    }
     return ''
 }

src/ts/process/tts.ts

@@ -139,8 +139,29 @@ export async function sayTTS(character:character,text:string) {
             }
 
         }
+        case 'novelai': {
+            const audioContext = new AudioContext();
+            const response = await fetch(`https://api.novelai.net/ai/generate-voice?text=${text}&voice=-1&seed=${character.naittsConfig.voice}&opus=false&version=${character.naittsConfig.version}`, {
+                method: 'GET',
+                headers: {
+                    "Authorization": "Bearer " + db.NAIApiKey,
+                }
+            });
+        
+            if (response.status === 200 && response.headers.get('content-type') === 'audio/mpeg') {
+                const audioBuffer = await response.arrayBuffer();
+                audioContext.decodeAudioData(audioBuffer, (decodedData) => {
+                    const sourceNode = audioContext.createBufferSource();
+                    sourceNode.buffer = decodedData;
+                    sourceNode.connect(audioContext.destination);
+                    sourceNode.start();
+                });
+            } else {
+                alertError("Error fetching or decoding audio data");
+            }
+            break;
+        }
     }
-
 }
 
 export const oaiVoices = [
@@ -189,4 +210,29 @@ export async function getVOICEVOXVoices() {
     })
     speakersInfo.unshift({ name: "None", list: null})
     return speakersInfo;
+}
+
+export async function getNovelAIVoices(){
+    return [
+        {
+            gender: "UNISEX",
+            voices: ['Anananan']
+        },
+        {
+            gender: "FEMALE",
+            voices: ['Aini', 'Orea', 'Claea', 'Lim', 'Aurae', 'Naia']
+        },
+        {
+            gender: "MALE",
+            voices: ['Aulon', 'Elei', 'Ogma', 'Raid', 'Pega', 'Lam']
+        }
+    ];
+}
+
+export async function FixNAITTS(data:character){
+    if (data.naittsConfig === undefined){
+        data.naittsConfig.voice = 'Anananan'
+    }
+
+    return data
 }

src/ts/process/uinttobase64.ts

@@ -0,0 +1,17 @@
+export async function convertToBase64(data: Uint8Array): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const blob = new Blob([data]);
+        const reader = new FileReader();
+
+        reader.onloadend = function() {
+            const base64String = reader.result as string;
+            resolve(base64String);
+        };
+
+        reader.onerror = function(error) {
+            reject(error);
+        };
+
+        reader.readAsDataURL(blob);
+    });
+}

src/ts/storage/database.ts

@@ -174,6 +174,18 @@ export function setDatabase(data:Database){
     if(checkNullish(data.sdCFG)){
         data.sdCFG = 7
     }
+    if(checkNullish(data.NAIImgUrl)){
+        data.NAIImgUrl = 'https://api.novelai.net/ai/generate-image'
+    }
+    if(checkNullish(data.NAIApiKey)){
+        data.NAIApiKey = ''
+    }
+    if(checkNullish(data.NAIImgModel)){
+        data.NAIImgModel = 'nai-diffusion-3'
+    }
+    if(checkNullish(data.NAII2I)){
+        data.NAII2I = true
+    }
     if(checkNullish(data.textTheme)){
         data.textTheme = "standard"
     }
@@ -231,6 +243,20 @@ export function setDatabase(data:Database){
             hr_upscaler:"Latent"
         }
     }
+    if(checkNullish(data.NAIImgConfig)){
+        data.NAIImgConfig = {
+            width:512,
+            height:768,
+            sampler:"k_dpmpp_sde",
+            steps:28,
+            scale:5,
+            sm:true,
+            sm_dyn:true,
+            noise:0.0,
+            strength:0.3,
+            image:""
+        }
+    }
     if(checkNullish(data.customTextTheme)){
         data.customTextTheme = {
             FontColorStandard: "#f8f8f2",
@@ -394,6 +420,11 @@ export interface Database{
     sdSteps:number
     sdCFG:number
     sdConfig:sdConfig
+    NAIImgUrl:string
+    NAIApiKey:string
+    NAIImgModel:string
+    NAII2I:boolean
+    NAIImgConfig:NAIImgConfig
     runpodKey:string
     promptPreprocess:boolean
     bias: [string, number][]
@@ -588,6 +619,11 @@ export interface character{
         INTONATION_SCALE?: number
         VOLUME_SCALE?: number
     }
+    naittsConfig?:{
+        customvoice?: boolean
+        voice?: string
+        version?: string
+    }
     supaMemory?:boolean
     additionalAssets?:[string, string, string][]
     ttsReadOnlyQuoted?:boolean
@@ -717,6 +753,18 @@ interface sdConfig{
     hr_upscaler:string
 }
 
+interface NAIImgConfig{
+    width:number,
+    height:number,
+    sampler:string,
+    steps:number,
+    scale:number,
+    sm:boolean,
+    sm_dyn:boolean,
+    noise:number,
+    strength:number,
+    image:string
+}
 export type FormatingOrderItem = 'main'|'jailbreak'|'chats'|'lorebook'|'globalNote'|'authorNote'|'lastChat'|'description'|'postEverything'|'personaPrompt'
 
 export interface Chat{

src/ts/storage/globalApi.ts

@@ -661,7 +661,7 @@ export async function globalFetch(url:string, arg:{
                         method: method,
                         signal: arg.abortSignal
                     })
-    
+
                     addFetchLog("Uint8Array Response", da.ok && da.status >= 200 && da.status < 300)
                     return {
                         ok: da.ok && da.status >= 200 && da.status < 300,
@@ -685,6 +685,16 @@ export async function globalFetch(url:string, arg:{
                         headers: headers,
                         method: method
                     })
+                    if(da.headers.get('content-type')?.includes('application/x-zip-compressed')){
+                        const daText = await da.blob()
+                        
+                        addFetchLog(daText, da.ok && da.status >= 200 && da.status < 300)
+                        return {
+                            ok: da.ok && da.status >= 200 && da.status < 300,
+                            data: daText,
+                            headers: Object.fromEntries(da.headers)
+                        }   
+                    }
                     const daText = await da.text()
                     try {
                         const dat = JSON.parse(daText)