add nai diffusion

Signed-off-by: hashcoko <hashcoko@gmail.com>
2023-11-27 07:13:22 +09:00
parent d9e16cc427
commit b69f31caae
12 changed files with 226 additions and 40 deletions
--- a/functions/proxy2.js
+++ b/functions/proxy2.js
@@ -31,9 +31,6 @@ async function fetchProxy(request) {
        const header = JSON.parse(decodeURIComponent(request.headers.get('risu-header') ?? 'null')) ?? request.headers
        let requestHeaders = new Headers(header);

-        console.log(urlParam)
-        console.log(requestHeaders)
-        console.log(request.body)
        let originalResponse = await fetch(urlParam, {
            method: method,
            headers: requestHeaders,
--- a/package-lock.json
+++ b/package-lock.json
@@ -30,6 +30,7 @@
        "gpt3-tokenizer": "^1.1.5",
        "html-to-image": "^1.11.11",
        "isomorphic-dompurify": "^1.8.0",
+        "jszip": "^3.10.1",
        "libsodium-wrappers-sumo": "^0.7.11",
        "localforage": "^1.10.0",
        "lodash": "^4.17.21",
@@ -2202,6 +2203,11 @@
        "url": "https://opencollective.com/core-js"
      }
    },
+    "node_modules/core-util-is": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
+      "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
+    },
    "node_modules/cors": {
      "version": "2.8.5",
      "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
@@ -3341,6 +3347,11 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
+    "node_modules/isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
+    },
    "node_modules/isexe": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -3407,6 +3418,52 @@
        }
      }
    },
+    "node_modules/jszip": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
+      "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
+      "dependencies": {
+        "lie": "~3.3.0",
+        "pako": "~1.0.2",
+        "readable-stream": "~2.3.6",
+        "setimmediate": "^1.0.5"
+      }
+    },
+    "node_modules/jszip/node_modules/lie": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz",
+      "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==",
+      "dependencies": {
+        "immediate": "~3.0.5"
+      }
+    },
+    "node_modules/jszip/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/jszip/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
+    "node_modules/jszip/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
    "node_modules/kleur": {
      "version": "4.1.5",
      "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz",
@@ -4060,6 +4117,11 @@
        "node": ">=8"
      }
    },
+    "node_modules/pako": {
+      "version": "1.0.11",
+      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
+    },
    "node_modules/parent-module": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -4442,6 +4504,11 @@
        "node": ">=6"
      }
    },
+    "node_modules/process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+    },
    "node_modules/protobufjs": {
      "version": "6.11.4",
      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz",
@@ -4866,6 +4933,11 @@
        "node": ">= 0.4"
      }
    },
+    "node_modules/setimmediate": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
+      "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA=="
+    },
    "node_modules/setprototypeof": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
--- a/package.json
+++ b/package.json
@@ -36,6 +36,7 @@
    "gpt3-tokenizer": "^1.1.5",
    "html-to-image": "^1.11.11",
    "isomorphic-dompurify": "^1.8.0",
+    "jszip": "^3.10.1",
    "libsodium-wrappers-sumo": "^0.7.11",
    "localforage": "^1.10.0",
    "lodash": "^4.17.21",
--- a/server.sh
+++ b/server.sh
--- a/server/node/server.cjs
+++ b/server/node/server.cjs
@@ -61,7 +61,6 @@ const reverseProxyFunc = async (req, res, next) => {
            headers: header,
            body: JSON.stringify(req.body)
        });
-
        // get response body as stream
        const originalBody = originalResponse.body;
        // get response headers
--- a/src/lib/Setting/Pages/OtherBotSettings.svelte
+++ b/src/lib/Setting/Pages/OtherBotSettings.svelte
@@ -8,7 +8,7 @@
    import TextInput from "src/lib/UI/GUI/TextInput.svelte";
    import SelectInput from "src/lib/UI/GUI/SelectInput.svelte";
    import OptionInput from "src/lib/UI/GUI/OptionInput.svelte";
-    import CheckInput from "src/lib/UI/GUI/CheckInput.svelte";
+    import SliderInput from "src/lib/UI/GUI/SliderInput.svelte";

 </script>
 <h2 class="mb-2 text-2xl font-bold mt-2">{language.otherBots}</h2>
@@ -67,6 +67,21 @@
    <span class="text-textcolor">Model</span>
    <TextInput size="sm" marginBottom placeholder="nai-diffusion-3" bind:value={$DataBase.NAIImgModel}/>

+    <span class="text-textcolor">Enable I2I</span>
+    <Check bind:check={$DataBase.NAII2I}/>
+
+
+    {#if $DataBase.NAII2I}
+
+        <span class="text-textcolor">strength</span>
+        <SliderInput min={0} max={0.99} step={0.01} bind:value={$DataBase.NAIImgConfig.strength}/>
+        <span class="text-textcolor2 mb-6 text-sm">{$DataBase.NAIImgConfig.strength}</span>
+        <span class="text-textcolor">noise</span>
+        <SliderInput min={0} max={0.99} step={0.01} bind:value={$DataBase.NAIImgConfig.noise}/>
+        <span class="text-textcolor2 mb-6 text-sm">{$DataBase.NAIImgConfig.noise}</span>
+
+    {/if}
+
    <span class="text-textcolor">Width</span>
    <NumberInput size="sm" marginBottom min={0} max={2048} bind:value={$DataBase.NAIImgConfig.width}/>
    <span class="text-textcolor">Height</span>
--- a/src/ts/process/generateSeed.ts
+++ b/src/ts/process/generateSeed.ts
@@ -0,0 +1,9 @@
+export function generateRandomSeed(length) {
+    let result = '';
+    const characters = '0123456789';
+    const charactersLength = characters.length;
+    for (let i = 0; i < length; i++) {
+        result += characters.charAt(Math.floor(Math.random() * charactersLength));
+    }
+    return result;
+}
--- a/src/ts/process/processzip.ts
+++ b/src/ts/process/processzip.ts
@@ -0,0 +1,15 @@
+import JSZip from "jszip";
+
+export async function processZip(dataArray: Uint8Array): Promise<string> {
+    const blob = new Blob([dataArray], { type: "application/zip" });
+    const zip = new JSZip();
+    const zipData = await zip.loadAsync(blob);
+
+    const imageFile = Object.keys(zipData.files).find(fileName => /\.(jpg|jpeg|png)$/.test(fileName));
+    if (imageFile) {
+        const imageData = await zipData.files[imageFile].async("base64");
+        return `data:image/png;base64,${imageData}`;
+    } else {
+        throw new Error("No image found in ZIP file");
+    }
+}
--- a/src/ts/process/stableDiff.ts
+++ b/src/ts/process/stableDiff.ts
@@ -2,11 +2,13 @@ import { get } from "svelte/store"
 import { DataBase, type character } from "../storage/database"
 import { requestChatData } from "./request"
 import { alertError } from "../alert"
-import { globalFetch } from "../storage/globalApi"
+import { globalFetch, readImage } from "../storage/globalApi"
 import { CharEmotion } from "../stores"
 import type { OpenAIChat } from "."
-
-
+import { processZip } from "./processzip"
+import { convertToBase64 } from "./uinttobase64" 
+import type { List } from "lodash"
+import { generateRandomSeed } from "./generateSeed"
 export async function stableDiff(currentChar:character,prompt:string){
    const mainPrompt = "assistant is a chat analyzer.\nuser will input a data of situation with key and values before chat, and a chat of a user and character.\nView the status of the chat and change the data.\nif data's key starts with $, it must change it every time.\nif data value is none, it must change it."
    let db = get(DataBase)
@@ -175,39 +177,80 @@ export async function stableDiff(currentChar:character,prompt:string){
        }


-        const uri = new URL(db.NAIImgUrl)
-        uri.pathname = '/ai/generate-image'
-        const reqlist = {
-            body: {
-                "input": prompts.join(','),
-                "model": db.NAIImgModel,
-                "parameters": {
-                    "width": db.NAIImgConfig.width,
-                    "height": db.NAIImgConfig.height,
-                    "sampler": db.NAIImgConfig.sampler,
-                    "steps": db.NAIImgConfig.steps,
-                    "scale": db.NAIImgConfig.scale,
-                    "negative_prompt": neg,
-                    "sm": db.NAIImgConfig.sm,
-                    "sm_dyn": db.NAIImgConfig.sm_dyn
+        const charimg = currentChar.image; // Uint8Array 형태의 이미지 데이터
+        console.log("charimg:" + charimg);
+        
+        const img = await readImage(charimg)
+        console.log("img:" + img);
+        const base64 = await convertToBase64(img);
+        const base64img = base64.split('base64,')[1];
+        
+        console.log("base64img:" + base64img);
+        let reqlist= {}
+
+        if(db.NAII2I){
+            let randomseed = generateRandomSeed(10);
+            let seed = parseInt(randomseed, 10);
+            reqlist = {
+                body: {
+                    "action": "img2img",
+                    "input": prompts.join(','),
+                    "model": db.NAIImgModel,
+                    "parameters": {
+                        "seed": seed,
+                        "extra_noise_seed": seed,
+                        "add_original_image": false,
+                        "cfg_rescale": 0,
+                        "controlnet_strength": 1,
+                        "dynamic_threshold": false,
+                        "n_samples": 1,
+                        "width": db.NAIImgConfig.width,
+                        "height": db.NAIImgConfig.height,
+                        "sampler": db.NAIImgConfig.sampler,
+                        "steps": db.NAIImgConfig.steps,
+                        "scale": db.NAIImgConfig.scale,
+                        "negative_prompt": neg,
+                        "sm": false,
+                        "sm_dyn": false,
+                        "noise": db.NAIImgConfig.noise,
+                        "noise_schedule": "native",
+                        "strength": db.NAIImgConfig.strength,
+                        "image": base64img,
+                        "ucPreset": 2,
+                        "uncond_scale": 1
+                    }
+                },
+                headers:{
+                    "Authorization": "Bearer " + db.NAIApiKey
+                }
+            }
+        }else{
+            reqlist = {
+                body: {
+                    "input": prompts.join(','),
+                    "model": db.NAIImgModel,
+                    "parameters": {
+                        "width": db.NAIImgConfig.width,
+                        "height": db.NAIImgConfig.height,
+                        "sampler": db.NAIImgConfig.sampler,
+                        "steps": db.NAIImgConfig.steps,
+                        "scale": db.NAIImgConfig.scale,
+                        "negative_prompt": neg,
+                        "sm": db.NAIImgConfig.sm,
+                        "sm_dyn": db.NAIImgConfig.sm_dyn
+                    }
+                },
+                headers:{
+                    "Authorization": "Bearer " + db.NAIApiKey
                }
-            },
-            headers:{
-                'Authorization:': 'Bearer ' + db.NAIApiKey,
-                'Content-Type': 'application/json'
            }
        }
-
-        console.log(uri)
-        console.log(reqlist)
        try {
-            const da = await globalFetch(uri.toString(), reqlist)   
+            const da = await globalFetch(db.NAIImgUrl, reqlist)   

-            console.log(da)
-            if(da.ok){
+            if(da){
                let charemotions = get(CharEmotion)
-                const img = `data:image/png;base64,${da.data.images[0]}`
-                console.log(img)
+                const img = await processZip(da.data);
                const emos:[string, string,number][] = [[img, img, Date.now()]]
                charemotions[currentChar.chaId] = emos
                CharEmotion.set(charemotions)
--- a/src/ts/process/uinttobase64.ts
+++ b/src/ts/process/uinttobase64.ts
@@ -0,0 +1,17 @@
+export async function convertToBase64(data: Uint8Array): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const blob = new Blob([data]);
+        const reader = new FileReader();
+
+        reader.onloadend = function() {
+            const base64String = reader.result as string;
+            resolve(base64String);
+        };
+
+        reader.onerror = function(error) {
+            reject(error);
+        };
+
+        reader.readAsDataURL(blob);
+    });
+}
--- a/src/ts/storage/database.ts
+++ b/src/ts/storage/database.ts
@@ -175,7 +175,7 @@ export function setDatabase(data:Database){
        data.sdCFG = 7
    }
    if(checkNullish(data.NAIImgUrl)){
-        data.NAIImgUrl = 'https://api.novelai.net'
+        data.NAIImgUrl = 'https://api.novelai.net/ai/generate-image'
    }
    if(checkNullish(data.NAIApiKey)){
        data.NAIApiKey = ''
@@ -183,6 +183,9 @@ export function setDatabase(data:Database){
    if(checkNullish(data.NAIImgModel)){
        data.NAIImgModel = 'nai-diffusion-3'
    }
+    if(checkNullish(data.NAII2I)){
+        data.NAII2I = true
+    }
    if(checkNullish(data.textTheme)){
        data.textTheme = "standard"
    }
@@ -248,7 +251,9 @@ export function setDatabase(data:Database){
            steps:28,
            scale:5,
            sm:true,
-            sm_dyn:true
+            sm_dyn:true,
+            noise:0.0,
+            strength:0.3
        }
    }
    if(checkNullish(data.customTextTheme)){
@@ -417,6 +422,7 @@ export interface Database{
    NAIImgUrl:string
    NAIApiKey:string
    NAIImgModel:string
+    NAII2I:boolean
    NAIImgConfig:NAIImgConfig
    runpodKey:string
    promptPreprocess:boolean
@@ -748,7 +754,9 @@ interface NAIImgConfig{
    steps:number,
    scale:number,
    sm:boolean,
-    sm_dyn:boolean
+    sm_dyn:boolean,
+    noise:number,
+    strength:number
 }
 export type FormatingOrderItem = 'main'|'jailbreak'|'chats'|'lorebook'|'globalNote'|'authorNote'|'lastChat'|'description'|'postEverything'|'personaPrompt'

--- a/src/ts/storage/globalApi.ts
+++ b/src/ts/storage/globalApi.ts
@@ -685,6 +685,16 @@ export async function globalFetch(url:string, arg:{
                        headers: headers,
                        method: method
                    })
+                    if(da.headers.get('content-type')?.includes('application/x-zip-compressed')){
+                        const daText = await da.blob()
+                        
+                        addFetchLog(daText, da.ok && da.status >= 200 && da.status < 300)
+                        return {
+                            ok: da.ok && da.status >= 200 && da.status < 300,
+                            data: daText,
+                            headers: Object.fromEntries(da.headers)
+                        }   
+                    }
                    const daText = await da.text()
                    try {
                        const dat = JSON.parse(daText)