mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Gemini image input issue

Browse Source
This commit is contained in:
poroyo
2024-12-08 05:04:06 +09:00
parent 8d5fb1a139
commit 94bb79df7e
2 changed files with 30 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/ts/process/files/image.ts
View File

@@ -84,7 +84,7 @@ export async function getInlayImage(id: string){
export function supportsInlayImage(){
const db = getDatabase()
return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') ||
return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('gemini-exp') || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') ||
(db.aiModel === 'reverse_proxy' && (
db.proxyRequestModel?.startsWith('gptv') || db.proxyRequestModel === 'gemini-pro-vision' || db.proxyRequestModel?.startsWith('claude-3') || db.proxyRequestModel.startsWith('gpt4_turbo') ||
db.proxyRequestModel?.startsWith('gpt5') || db.proxyRequestModel?.startsWith('gpt4o') ||

63
src/ts/process/request.ts
View File

@@ -1374,10 +1374,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
for(let i=0;i<formated.length;i++){
const chat = formated[i]
if(chat.memo && chat.memo.startsWith('inlayImage')){
pendingImage = chat.content
continue
}
if(i === 0){
if(chat.role === 'user' || chat.role === 'assistant'){
reformatedChat.push({
@@ -1403,7 +1400,34 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
chat.role === 'assistant' ? 'MODEL' :
chat.role
if(prevChat.role === qRole){
if (chat.multimodals && chat.multimodals.length > 0 && chat.role === "user") {
let geminiParts: GeminiPart[] = [];
geminiParts.push({
text: chat.content,
});
for (const modal of chat.multimodals) {
if (modal.type === "image") {
const dataurl = modal.base64;
const base64 = dataurl.split(",")[1];
const mediaType = dataurl.split(";")[0].split(":")[1];
geminiParts.push({
inlineData: {
mimeType: mediaType,
data: base64,
}
});
}
}
reformatedChat.push({
role: "USER",
parts: geminiParts,
});
} else if (prevChat.role === qRole) {
reformatedChat[reformatedChat.length-1].parts[0].text += '\n' + chat.content
continue
}
@@ -1420,36 +1444,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
})
}
}
else if(chat.role === 'user' && pendingImage !== ''){
//conver image to jpeg so it can be inlined
const canv = document.createElement('canvas')
const img = new Image()
img.src = pendingImage
await img.decode()
canv.width = img.width
canv.height = img.height
const ctx = canv.getContext('2d')
ctx.drawImage(img, 0, 0)
const base64 = canv.toDataURL('image/jpeg').replace(/^data:image\/jpeg;base64,/, "")
const mimeType = 'image/jpeg'
pendingImage = ''
canv.remove()
img.remove()
reformatedChat.push({
role: "USER",
parts: [
{
text: chat.content,
},
{
inlineData: {
mimeType: mimeType,
data: base64
}
}]
})
}
else if(chat.role === 'assistant' || chat.role === 'user'){
reformatedChat.push({
role: chat.role === 'user' ? 'USER' : 'MODEL',