From 94bb79df7eff812454cf0eff6e295d31c8390cd8 Mon Sep 17 00:00:00 2001 From: poroyo <132068975+poroyo@users.noreply.github.com> Date: Sun, 8 Dec 2024 05:04:06 +0900 Subject: [PATCH 1/3] Fix Gemini image input issue --- src/ts/process/files/image.ts | 2 +- src/ts/process/request.ts | 63 ++++++++++++++++------------------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/ts/process/files/image.ts b/src/ts/process/files/image.ts index 0bdac70c..1441c628 100644 --- a/src/ts/process/files/image.ts +++ b/src/ts/process/files/image.ts @@ -84,7 +84,7 @@ export async function getInlayImage(id: string){ export function supportsInlayImage(){ const db = getDatabase() - return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') || + return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('gemini-exp') || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') || (db.aiModel === 'reverse_proxy' && ( db.proxyRequestModel?.startsWith('gptv') || db.proxyRequestModel === 'gemini-pro-vision' || db.proxyRequestModel?.startsWith('claude-3') || db.proxyRequestModel.startsWith('gpt4_turbo') || db.proxyRequestModel?.startsWith('gpt5') || db.proxyRequestModel?.startsWith('gpt4o') || diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 47c6a11e..09ea966a 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1374,10 +1374,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise for(let i=0;i 0 && chat.role === "user") { + let geminiParts: GeminiPart[] = []; + + geminiParts.push({ + text: chat.content, + }); + + for (const modal of chat.multimodals) { + if (modal.type === "image") { + const dataurl = modal.base64; + const base64 = dataurl.split(",")[1]; + const mediaType = dataurl.split(";")[0].split(":")[1]; + + geminiParts.push({ + inlineData: { + mimeType: mediaType, + data: base64, + } + }); + } + } + + reformatedChat.push({ + role: "USER", + parts: geminiParts, + }); + + } else if (prevChat.role === qRole) { reformatedChat[reformatedChat.length-1].parts[0].text += '\n' + chat.content continue } @@ -1420,36 +1444,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise }) } } - else if(chat.role === 'user' && pendingImage !== ''){ - //conver image to jpeg so it can be inlined - const canv = document.createElement('canvas') - const img = new Image() - img.src = pendingImage - await img.decode() - canv.width = img.width - canv.height = img.height - const ctx = canv.getContext('2d') - ctx.drawImage(img, 0, 0) - const base64 = canv.toDataURL('image/jpeg').replace(/^data:image\/jpeg;base64,/, "") - const mimeType = 'image/jpeg' - pendingImage = '' - canv.remove() - img.remove() - reformatedChat.push({ - role: "USER", - parts: [ - { - text: chat.content, - }, - { - inlineData: { - mimeType: mimeType, - data: base64 - } - }] - }) - } else if(chat.role === 'assistant' || chat.role === 'user'){ reformatedChat.push({ role: chat.role === 'user' ? 'USER' : 'MODEL', From 697201bbb67518a39287c816beb57437e1897284 Mon Sep 17 00:00:00 2001 From: poroyo <132068975+poroyo@users.noreply.github.com> Date: Sun, 8 Dec 2024 05:16:13 +0900 Subject: [PATCH 2/3] Remove unused variable 'pendingImage' --- src/ts/process/request.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 09ea966a..0143a6c2 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1364,7 +1364,6 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise let reformatedChat:GeminiChat[] = [] - let pendingImage = '' let systemPrompt = '' if(formated[0].role === 'system'){ From 9a1b22eae599572abb9ed4caf23fe43d50474d96 Mon Sep 17 00:00:00 2001 From: poroyo <132068975+poroyo@users.noreply.github.com> Date: Sun, 8 Dec 2024 05:28:57 +0900 Subject: [PATCH 3/3] Fix code indentation --- src/ts/process/request.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 0143a6c2..64f19a85 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1408,16 +1408,16 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise for (const modal of chat.multimodals) { if (modal.type === "image") { - const dataurl = modal.base64; - const base64 = dataurl.split(",")[1]; - const mediaType = dataurl.split(";")[0].split(":")[1]; - - geminiParts.push({ - inlineData: { - mimeType: mediaType, - data: base64, - } - }); + const dataurl = modal.base64; + const base64 = dataurl.split(",")[1]; + const mediaType = dataurl.split(";")[0].split(":")[1]; + + geminiParts.push({ + inlineData: { + mimeType: mediaType, + data: base64, + } + }); } }