add nai diffusion

Signed-off-by: hashcoko <hashcoko@gmail.com>

functions/proxy2.js

@@ -31,9 +31,6 @@ async function fetchProxy(request) {
         const header = JSON.parse(decodeURIComponent(request.headers.get('risu-header') ?? 'null')) ?? request.headers
         let requestHeaders = new Headers(header);
 
-        console.log(urlParam)
-        console.log(requestHeaders)
-        console.log(request.body)
         let originalResponse = await fetch(urlParam, {
             method: method,
             headers: requestHeaders,

package-lock.json

@@ -30,6 +30,7 @@
         "gpt3-tokenizer": "^1.1.5",
         "html-to-image": "^1.11.11",
         "isomorphic-dompurify": "^1.8.0",
+        "jszip": "^3.10.1",
         "libsodium-wrappers-sumo": "^0.7.11",
         "localforage": "^1.10.0",
         "lodash": "^4.17.21",
@@ -2202,6 +2203,11 @@
         "url": "https://opencollective.com/core-js"
       }
     },
+    "node_modules/core-util-is": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
+      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
+    },
     "node_modules/cors": {
       "version": "2.8.5",
       "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
@@ -3341,6 +3347,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
+    },
     "node_modules/isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -3407,6 +3418,52 @@
         }
       }
     },
+    "node_modules/jszip": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+      "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
+      "dependencies": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "setimmediate": "^1.0.5"
+      }
+    },
+    "node_modules/jszip/node_modules/lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "dependencies": {
+        "immediate": "~3.0.5"
+      }
+    },
+    "node_modules/jszip/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/jszip/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
+    "node_modules/jszip/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
     "node_modules/kleur": {
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz",
@@ -4060,6 +4117,11 @@
         "node": ">=8"
       }
     },
+    "node_modules/pako": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
+    },
     "node_modules/parent-module": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -4442,6 +4504,11 @@
         "node": ">=6"
       }
     },
+    "node_modules/process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+    },
     "node_modules/protobufjs": {
       "version": "6.11.4",
       "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz",
@@ -4866,6 +4933,11 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/setimmediate": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
+      "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="
+    },
     "node_modules/setprototypeof": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",

package.json

@@ -36,6 +36,7 @@
     "gpt3-tokenizer": "^1.1.5",
     "html-to-image": "^1.11.11",
     "isomorphic-dompurify": "^1.8.0",
+    "jszip": "^3.10.1",
     "libsodium-wrappers-sumo": "^0.7.11",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
@@ -91,4 +92,4 @@
     "vite-plugin-top-level-await": "^1.3.1",
     "vite-plugin-wasm": "^3.2.2"
   }
-}
+}

server/node/server.cjs

@@ -61,7 +61,6 @@ const reverseProxyFunc = async (req, res, next) => {
             headers: header,
             body: JSON.stringify(req.body)
         });
-
         // get response body as stream
         const originalBody = originalResponse.body;
         // get response headers

src/lib/Setting/Pages/OtherBotSettings.svelte

@@ -8,7 +8,7 @@
     import TextInput from "src/lib/UI/GUI/TextInput.svelte";
     import SelectInput from "src/lib/UI/GUI/SelectInput.svelte";
     import OptionInput from "src/lib/UI/GUI/OptionInput.svelte";
-    import CheckInput from "src/lib/UI/GUI/CheckInput.svelte";
+    import SliderInput from "src/lib/UI/GUI/SliderInput.svelte";
 
 </script>
 <h2 class="mb-2 text-2xl font-bold mt-2">{language.otherBots}</h2>
@@ -67,6 +67,21 @@
     <span class="text-textcolor">Model</span>
     <TextInput size="sm" marginBottom placeholder="nai-diffusion-3" bind:value={$DataBase.NAIImgModel}/>
 
+    <span class="text-textcolor">Enable I2I</span>
+    <Check bind:check={$DataBase.NAII2I}/>
+
+
+    {#if $DataBase.NAII2I}
+
+        <span class="text-textcolor">strength</span>
+        <SliderInput min={0} max={0.99} step={0.01} bind:value={$DataBase.NAIImgConfig.strength}/>
+        <span class="text-textcolor2 mb-6 text-sm">{$DataBase.NAIImgConfig.strength}</span>
+        <span class="text-textcolor">noise</span>
+        <SliderInput min={0} max={0.99} step={0.01} bind:value={$DataBase.NAIImgConfig.noise}/>
+        <span class="text-textcolor2 mb-6 text-sm">{$DataBase.NAIImgConfig.noise}</span>
+
+    {/if}
+
     <span class="text-textcolor">Width</span>
     <NumberInput size="sm" marginBottom min={0} max={2048} bind:value={$DataBase.NAIImgConfig.width}/>
     <span class="text-textcolor">Height</span>

src/ts/process/generateSeed.ts

@@ -0,0 +1,9 @@
+export function generateRandomSeed(length) {
+    let result = '';
+    const characters = '0123456789';
+    const charactersLength = characters.length;
+    for (let i = 0; i < length; i++) {
+        result += characters.charAt(Math.floor(Math.random() * charactersLength));
+    }
+    return result;
+}

src/ts/process/processzip.ts

@@ -0,0 +1,15 @@
+import JSZip from "jszip";
+
+export async function processZip(dataArray: Uint8Array): Promise<string> {
+    const blob = new Blob([dataArray], { type: "application/zip" });
+    const zip = new JSZip();
+    const zipData = await zip.loadAsync(blob);
+
+    const imageFile = Object.keys(zipData.files).find(fileName => /\.(jpg|jpeg|png)$/.test(fileName));
+    if (imageFile) {
+        const imageData = await zipData.files[imageFile].async("base64");
+        return `data:image/png;base64,${imageData}`;
+    } else {
+        throw new Error("No image found in ZIP file");
+    }
+}

src/ts/process/stableDiff.ts

@@ -2,11 +2,13 @@ import { get } from "svelte/store"
 import { DataBase, type character } from "../storage/database"
 import { requestChatData } from "./request"
 import { alertError } from "../alert"
-import { globalFetch } from "../storage/globalApi"
+import { globalFetch, readImage } from "../storage/globalApi"
 import { CharEmotion } from "../stores"
 import type { OpenAIChat } from "."
-
+import { processZip } from "./processzip"
-
+import { convertToBase64 } from "./uinttobase64" 
+import type { List } from "lodash"
+import { generateRandomSeed } from "./generateSeed"
 export async function stableDiff(currentChar:character,prompt:string){
     const mainPrompt = "assistant is a chat analyzer.\nuser will input a data of situation with key and values before chat, and a chat of a user and character.\nView the status of the chat and change the data.\nif data's key starts with $, it must change it every time.\nif data value is none, it must change it."
     let db = get(DataBase)
@@ -175,39 +177,80 @@ export async function stableDiff(currentChar:character,prompt:string){
         }
 
 
-        const uri = new URL(db.NAIImgUrl)
+        const charimg = currentChar.image; // Uint8Array 형태의 이미지 데이터
-        uri.pathname = '/ai/generate-image'
+        console.log("charimg:" + charimg);
-        const reqlist = {
+        
-            body: {
+        const img = await readImage(charimg)
-                "input": prompts.join(','),
+        console.log("img:" + img);
-                "model": db.NAIImgModel,
+        const base64 = await convertToBase64(img);
-                "parameters": {
+        const base64img = base64.split('base64,')[1];
-                    "width": db.NAIImgConfig.width,
+        
-                    "height": db.NAIImgConfig.height,
+        console.log("base64img:" + base64img);
-                    "sampler": db.NAIImgConfig.sampler,
+        let reqlist= {}
-                    "steps": db.NAIImgConfig.steps,
+
-                    "scale": db.NAIImgConfig.scale,
+        if(db.NAII2I){
-                    "negative_prompt": neg,
+            let randomseed = generateRandomSeed(10);
-                    "sm": db.NAIImgConfig.sm,
+            let seed = parseInt(randomseed, 10);
-                    "sm_dyn": db.NAIImgConfig.sm_dyn
+            reqlist = {
+                body: {
+                    "action": "img2img",
+                    "input": prompts.join(','),
+                    "model": db.NAIImgModel,
+                    "parameters": {
+                        "seed": seed,
+                        "extra_noise_seed": seed,
+                        "add_original_image": false,
+                        "cfg_rescale": 0,
+                        "controlnet_strength": 1,
+                        "dynamic_threshold": false,
+                        "n_samples": 1,
+                        "width": db.NAIImgConfig.width,
+                        "height": db.NAIImgConfig.height,
+                        "sampler": db.NAIImgConfig.sampler,
+                        "steps": db.NAIImgConfig.steps,
+                        "scale": db.NAIImgConfig.scale,
+                        "negative_prompt": neg,
+                        "sm": false,
+                        "sm_dyn": false,
+                        "noise": db.NAIImgConfig.noise,
+                        "noise_schedule": "native",
+                        "strength": db.NAIImgConfig.strength,
+                        "image": base64img,
+                        "ucPreset": 2,
+                        "uncond_scale": 1
+                    }
+                },
+                headers:{
+                    "Authorization": "Bearer " + db.NAIApiKey
+                }
+            }
+        }else{
+            reqlist = {
+                body: {
+                    "input": prompts.join(','),
+                    "model": db.NAIImgModel,
+                    "parameters": {
+                        "width": db.NAIImgConfig.width,
+                        "height": db.NAIImgConfig.height,
+                        "sampler": db.NAIImgConfig.sampler,
+                        "steps": db.NAIImgConfig.steps,
+                        "scale": db.NAIImgConfig.scale,
+                        "negative_prompt": neg,
+                        "sm": db.NAIImgConfig.sm,
+                        "sm_dyn": db.NAIImgConfig.sm_dyn
+                    }
+                },
+                headers:{
+                    "Authorization": "Bearer " + db.NAIApiKey
                 }
-            },
-            headers:{
-                'Authorization:': 'Bearer ' + db.NAIApiKey,
-                'Content-Type': 'application/json'
             }
         }
-
-        console.log(uri)
-        console.log(reqlist)
         try {
-            const da = await globalFetch(uri.toString(), reqlist)   
+            const da = await globalFetch(db.NAIImgUrl, reqlist)   
 
-            console.log(da)
+            if(da){
-            if(da.ok){
                 let charemotions = get(CharEmotion)
-                const img = `data:image/png;base64,${da.data.images[0]}`
+                const img = await processZip(da.data);
-                console.log(img)
                 const emos:[string, string,number][] = [[img, img, Date.now()]]
                 charemotions[currentChar.chaId] = emos
                 CharEmotion.set(charemotions)

src/ts/process/uinttobase64.ts

@@ -0,0 +1,17 @@
+export async function convertToBase64(data: Uint8Array): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const blob = new Blob([data]);
+        const reader = new FileReader();
+
+        reader.onloadend = function() {
+            const base64String = reader.result as string;
+            resolve(base64String);
+        };
+
+        reader.onerror = function(error) {
+            reject(error);
+        };
+
+        reader.readAsDataURL(blob);
+    });
+}

src/ts/storage/database.ts

@@ -175,7 +175,7 @@ export function setDatabase(data:Database){
         data.sdCFG = 7
     }
     if(checkNullish(data.NAIImgUrl)){
-        data.NAIImgUrl = 'https://api.novelai.net'
+        data.NAIImgUrl = 'https://api.novelai.net/ai/generate-image'
     }
     if(checkNullish(data.NAIApiKey)){
         data.NAIApiKey = ''
@@ -183,6 +183,9 @@ export function setDatabase(data:Database){
     if(checkNullish(data.NAIImgModel)){
         data.NAIImgModel = 'nai-diffusion-3'
     }
+    if(checkNullish(data.NAII2I)){
+        data.NAII2I = true
+    }
     if(checkNullish(data.textTheme)){
         data.textTheme = "standard"
     }
@@ -248,7 +251,9 @@ export function setDatabase(data:Database){
             steps:28,
             scale:5,
             sm:true,
-            sm_dyn:true
+            sm_dyn:true,
+            noise:0.0,
+            strength:0.3
         }
     }
     if(checkNullish(data.customTextTheme)){
@@ -417,6 +422,7 @@ export interface Database{
     NAIImgUrl:string
     NAIApiKey:string
     NAIImgModel:string
+    NAII2I:boolean
     NAIImgConfig:NAIImgConfig
     runpodKey:string
     promptPreprocess:boolean
@@ -748,7 +754,9 @@ interface NAIImgConfig{
     steps:number,
     scale:number,
     sm:boolean,
-    sm_dyn:boolean
+    sm_dyn:boolean,
+    noise:number,
+    strength:number
 }
 export type FormatingOrderItem = 'main'|'jailbreak'|'chats'|'lorebook'|'globalNote'|'authorNote'|'lastChat'|'description'|'postEverything'|'personaPrompt'
 

src/ts/storage/globalApi.ts

@@ -661,7 +661,7 @@ export async function globalFetch(url:string, arg:{
                         method: method,
                         signal: arg.abortSignal
                     })
-    
+
                     addFetchLog("Uint8Array Response", da.ok && da.status >= 200 && da.status < 300)
                     return {
                         ok: da.ok && da.status >= 200 && da.status < 300,
@@ -685,6 +685,16 @@ export async function globalFetch(url:string, arg:{
                         headers: headers,
                         method: method
                     })
+                    if(da.headers.get('content-type')?.includes('application/x-zip-compressed')){
+                        const daText = await da.blob()
+                        
+                        addFetchLog(daText, da.ok && da.status >= 200 && da.status < 300)
+                        return {
+                            ok: da.ok && da.status >= 200 && da.status < 300,
+                            data: daText,
+                            headers: Object.fromEntries(da.headers)
+                        }   
+                    }
                     const daText = await da.text()
                     try {
                         const dat = JSON.parse(daText)