From b7b7b54dc35f8a8d7e5246b57022f37ab82666e3 Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Thu, 14 Dec 2023 07:33:54 +0900
Subject: [PATCH] [feat] add gemini

---
 src/lib/Setting/Pages/BotSettings.svelte |   2 +-
 src/lib/UI/ModelList.svelte              |   6 +
 src/ts/image.ts                          |   2 +-
 src/ts/process/request.ts                | 208 ++++++++++++++++++++++-
 4 files changed, 213 insertions(+), 5 deletions(-)

diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte
index 61aec249..7ead5cca 100644
--- a/src/lib/Setting/Pages/BotSettings.svelte
+++ b/src/lib/Setting/Pages/BotSettings.svelte
@@ -115,7 +115,7 @@
         <Button size="sm" className="mb-2" on:click={() => {setRecommended($DataBase.aiModel, 'ask')}}>{language.recommendedPreset}</Button>
     </div>
 {/if}
-{#if $DataBase.aiModel.startsWith('palm2') || $DataBase.subModel.startsWith('palm2')}
+{#if $DataBase.aiModel.startsWith('palm2') || $DataBase.subModel.startsWith('palm2') || $DataBase.aiModel.startsWith('gemini') || $DataBase.subModel.startsWith('gemini')}
     <span class="text-textcolor">Google Bearer Token</span>
     <TextInput marginBottom={true} size={"sm"} placeholder="..." bind:value={$DataBase.google.accessToken}/>
 
diff --git a/src/lib/UI/ModelList.svelte b/src/lib/UI/ModelList.svelte
index d7cbd4dd..6a0ad43c 100644
--- a/src/lib/UI/ModelList.svelte
+++ b/src/lib/UI/ModelList.svelte
@@ -75,6 +75,8 @@
                 return "Mistral Small"
             case 'mistral-medium':
                 return "Mistral Medium"
+            case 'gemini-pro':
+                return "Gemini Pro"
             default:
                 if(name.startsWith("horde:::")){
                     return name.replace(":::", " ")
@@ -160,6 +162,10 @@
                     <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('palm2')}}>Bison</button>
                     <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('palm2_unicorn')}}>Unicorn</button>
                 </Arcodion>
+                <Arcodion name="Google Gemini">
+                    <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('gemini-pro')}}>Gemini Pro</button>
+                    <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('gemini-pro-vision')}}>Gemini Pro Vision</button>
+                </Arcodion>
                 <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('kobold')}}>Kobold</button>
             {/if}
             <Arcodion name="Novellist">
diff --git a/src/ts/image.ts b/src/ts/image.ts
index f944dcb3..0db61712 100644
--- a/src/ts/image.ts
+++ b/src/ts/image.ts
@@ -94,7 +94,7 @@ export async function getInlayImage(id: string){
 
 export function supportsInlayImage(){
     const db = get(DataBase)
-    return db.aiModel.startsWith('gptv') || (db.aiModel === 'reverse_proxy' && db.proxyRequestModel?.startsWith('gptv'))
+    return db.aiModel.startsWith('gptv') || (db.aiModel === 'reverse_proxy' && db.proxyRequestModel?.startsWith('gptv')) || db.aiModel === 'gemini-pro-vision'
 }
 
 export async function reencodeImage(img:Uint8Array){
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index f7ec831a..7ff2602b 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -306,14 +306,14 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                         }
                     }
                     else{
-                        const prevChat = reformatedChat[i-1]
+                        const prevChat = reformatedChat[reformatedChat.length-1]
                         if(prevChat.role === chat.role){
-                            reformatedChat[i-1].content += '\n' + chat.content
+                            reformatedChat[reformatedChat.length-1].content += '\n' + chat.content
                             continue
                         }
                         else if(chat.role === 'system'){
                             if(prevChat.role === 'user'){
-                                reformatedChat[i-1].content += '\nSystem:' + chat.content
+                                reformatedChat[reformatedChat.length-1].content += '\nSystem:' + chat.content
                             }
                             else{
                                 reformatedChat.push({
@@ -322,6 +322,12 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                                 })
                             }
                         }
+                        else if(chat.role === 'function'){
+                            reformatedChat.push({
+                                role: 'user',
+                                content: chat.content
+                            })
+                        }
                         else{
                             reformatedChat.push({
                                 role: chat.role,
@@ -1003,6 +1009,202 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 }
             }
         }
+        case 'gemini-pro':
+        case 'gemini-pro-vision':{
+            interface GeminiPart{
+                text?:string
+                "inlineData"?: {
+                    "mimeType": string,
+                    "data": string
+                },
+            }
+            
+            interface GeminiChat {
+                role: "USER"|"MODEL"
+                parts:|GeminiPart[]
+            }
+
+
+            let reformatedChat:GeminiChat[] = []
+            let pendingImage = ''
+
+            for(let i=0;i<formated.length;i++){
+                const chat = formated[i]
+                if(chat.memo && chat.memo.startsWith('inlayImage')){
+                    pendingImage = chat.content
+                    continue
+                }
+                if(i === 0){
+                    if(chat.role === 'user' || chat.role === 'assistant'){
+                        reformatedChat.push({
+                            role: chat.role === 'user' ? 'USER' : 'MODEL',
+                            parts: [{
+                                text: chat.content
+                            }]
+                        })
+                    }
+                    else{
+                        reformatedChat.push({
+                            role: "USER",
+                            parts: [{
+                                text: chat.role + ':' + chat.content
+                            }]
+                        })
+                    }
+                }
+                else{
+                    const prevChat = reformatedChat[reformatedChat.length-1]
+                    const qRole = 
+                        chat.role === 'user' ? 'USER' :
+                        chat.role === 'assistant' ? 'MODEL' :
+                        chat.role
+
+                    if(prevChat.role === qRole){
+                        reformatedChat[reformatedChat.length-1].parts[0].text += '\n' + chat.content
+                        continue
+                    }
+                    else if(chat.role === 'system'){
+                        if(prevChat.role === 'USER'){
+                            reformatedChat[reformatedChat.length-1].parts[0].text += '\nsystem:' + chat.content
+                        }
+                        else{
+                            reformatedChat.push({
+                                role: "USER",
+                                parts: [{
+                                    text: chat.role + ':' + chat.content
+                                }]
+                            })
+                        }
+                    }
+                    else if(chat.role === 'user' && pendingImage !== ''){
+                        //conver image to jpeg so it can be inlined
+                        const canv = document.createElement('canvas')
+                        const img = new Image()
+                        img.src = pendingImage  
+                        await img.decode()
+                        canv.width = img.width
+                        canv.height = img.height
+                        const ctx = canv.getContext('2d')
+                        ctx.drawImage(img, 0, 0)
+                        const base64 = canv.toDataURL('image/jpeg').replace(/^data:image\/jpeg;base64,/, "")
+                        const mimeType = 'image/jpeg'
+                        pendingImage = ''
+                        canv.remove()
+                        img.remove()
+
+                        reformatedChat.push({
+                            role: "USER",
+                            parts: [
+                            {
+                                text: chat.content,
+                            },
+                            {
+                                inlineData: {
+                                    mimeType: mimeType,
+                                    data: base64
+                                }
+                            }]
+                        })
+                    }
+                    else if(chat.role === 'assistant' || chat.role === 'user'){
+                        reformatedChat.push({
+                            role: chat.role === 'user' ? 'USER' : 'MODEL',
+                            parts: [{
+                                text: chat.content
+                            }]
+                        })
+                    }
+                    else{
+                        reformatedChat.push({
+                            role: "USER",
+                            parts: [{
+                                text: chat.role + ':' + chat.content
+                            }]
+                        })
+                    }
+                }
+            }
+
+            const uncensoredCatagory = [
+                {
+                    "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+                    "threshold": "BLOCK_NONE"
+                },
+                {
+                    "category": "HARM_CATEGORY_HATE_SPEECH",
+                    "threshold": "BLOCK_NONE"
+                },
+                {
+                    "category": "HARM_CATEGORY_HARASSMENT",
+                    "threshold": "BLOCK_NONE"
+                },
+                {
+                    "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+                    "threshold": "BLOCK_NONE"
+                },
+            ]
+
+
+            const body = {
+                contents: reformatedChat,
+                generation_config: {
+                    "maxOutputTokens": maxTokens,
+                    "temperature": temperature,
+                    "topP": db.top_p,
+                },
+                safetySettings: uncensoredCatagory
+            }
+
+
+            const PROJECT_ID=db.google.projectId
+            const MODEL_ID= aiModel === 'palm2' ? 'text-bison' :
+                                        'palm2_unicorn' ? 'text-unicorn' : 
+                                        ''
+            const REGION="us-central1"
+            
+            const url = `https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${aiModel}:streamGenerateContent`
+            const res = await globalFetch(url, {
+                headers: {
+                    "Content-Type": "application/json",
+                    "Authorization": "Bearer " + db.google.accessToken
+                },
+                body: body,
+            })
+
+            if(!res.ok){
+                return {
+                    type: 'fail',
+                    result: `${JSON.stringify(res.data)}`
+                }
+            }
+
+            let fullRes = ''
+
+            for(const data of res.data){
+
+                if(data?.candidates?.[0]?.content?.parts?.[0]?.text){
+                    fullRes += data.candidates[0].content.parts[0].text
+                }
+                else if(data?.errors){
+                    return {
+                        type: 'fail',
+                        result: `${JSON.stringify(data.errors)}`
+                    }
+                }
+                else{
+                    return {
+                        type: 'fail',
+                        result: `${JSON.stringify(data)}`
+                    }
+                }
+            }
+
+            return {
+                type: 'success',
+                result: fullRes
+            }
+
+        }
         case "kobold":{
             const proompt = stringlizeChat(formated, currentChar?.name ?? '', arg.continue)
             const url = new URL(db.koboldURL)