diff --git a/src/ts/process/files/image.ts b/src/ts/process/files/image.ts index 0bdac70c..1441c628 100644 --- a/src/ts/process/files/image.ts +++ b/src/ts/process/files/image.ts @@ -84,7 +84,7 @@ export async function getInlayImage(id: string){ export function supportsInlayImage(){ const db = getDatabase() - return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') || + return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('gemini-exp') || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') || (db.aiModel === 'reverse_proxy' && ( db.proxyRequestModel?.startsWith('gptv') || db.proxyRequestModel === 'gemini-pro-vision' || db.proxyRequestModel?.startsWith('claude-3') || db.proxyRequestModel.startsWith('gpt4_turbo') || db.proxyRequestModel?.startsWith('gpt5') || db.proxyRequestModel?.startsWith('gpt4o') || diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 8d88be23..c5f5f8fd 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1364,7 +1364,6 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise let reformatedChat:GeminiChat[] = [] - let pendingImage = '' let systemPrompt = '' if(formated[0].role === 'system'){ @@ -1374,10 +1373,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise for(let i=0;i 0 && chat.role === "user") { + let geminiParts: GeminiPart[] = []; + + geminiParts.push({ + text: chat.content, + }); + + for (const modal of chat.multimodals) { + if (modal.type === "image") { + const dataurl = modal.base64; + const base64 = dataurl.split(",")[1]; + const mediaType = dataurl.split(";")[0].split(":")[1]; + + geminiParts.push({ + inlineData: { + mimeType: mediaType, + data: base64, + } + }); + } + } + + reformatedChat.push({ + role: "USER", + parts: geminiParts, + }); + + } else if (prevChat.role === qRole) { reformatedChat[reformatedChat.length-1].parts[0].text += '\n' + chat.content continue } @@ -1420,36 +1443,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise }) } } - else if(chat.role === 'user' && pendingImage !== ''){ - //conver image to jpeg so it can be inlined - const canv = document.createElement('canvas') - const img = new Image() - img.src = pendingImage - await img.decode() - canv.width = img.width - canv.height = img.height - const ctx = canv.getContext('2d') - ctx.drawImage(img, 0, 0) - const base64 = canv.toDataURL('image/jpeg').replace(/^data:image\/jpeg;base64,/, "") - const mimeType = 'image/jpeg' - pendingImage = '' - canv.remove() - img.remove() - reformatedChat.push({ - role: "USER", - parts: [ - { - text: chat.content, - }, - { - inlineData: { - mimeType: mimeType, - data: base64 - } - }] - }) - } else if(chat.role === 'assistant' || chat.role === 'user'){ reformatedChat.push({ role: chat.role === 'user' ? 'USER' : 'MODEL',