Add Gemini related features

src/ts/process/index.svelte.ts

@@ -18,7 +18,7 @@ import { groupOrder } from "./group";
 import { runTrigger } from "./triggers";
 import { HypaProcesser } from "./memory/hypamemory";
 import { additionalInformations } from "./embedding/addinfo";
-import { getInlayImage, supportsInlayImage } from "./files/image";
+import { getInlayAsset, supportsInlayImage } from "./files/inlays";
 import { getGenerationModelString } from "./models/modelString";
 import { connectionOpen, peerRevertChat, peerSafeCheck, peerSync } from "../sync/multiuser";
 import { runInlayScreen } from "./inlayScreen";
@@ -29,6 +29,7 @@ import { hanuraiMemory } from "./memory/hanuraiMemory";
 import { hypaMemoryV2 } from "./memory/hypav2";
 import { runLuaEditTrigger } from "./lua";
 import { parseChatML } from "../parser.svelte";
+import { getModelInfo, LLMFlags } from "../model/modellist";
 
 export interface OpenAIChat{
     role: 'system'|'user'|'assistant'|'function'
@@ -699,12 +700,13 @@ export async function sendChat(chatProcessIndex = -1,arg:{
         }
 
         let multimodal:MultiModal[] = []
+        const modelinfo = getModelInfo(DBState.db.aiModel)
         if(inlays.length > 0){
             for(const inlay of inlays){
                 const inlayName = inlay.replace('{{inlay::', '').replace('}}', '')
-                const inlayData = await getInlayImage(inlayName)
-                if(inlayData){
-                    if(supportsInlayImage()){
+                const inlayData = await getInlayAsset(inlayName)
+                if(inlayData?.type === 'image'){
+                    if(modelinfo.flags.includes(LLMFlags.hasImageInput)){
                         multimodal.push({
                             type: 'image',
                             base64: inlayData.data,
@@ -717,6 +719,14 @@ export async function sendChat(chatProcessIndex = -1,arg:{
                         formatedChat += `[${captionResult[0].generated_text}]`
                     }
                 }
+                if(inlayData?.type === 'video' || inlayData?.type === 'audio'){
+                    if(multimodal.length === 0){
+                        multimodal.push({
+                            type: inlayData.type,
+                            base64: inlayData.data
+                        })
+                    }
+                }
                 formatedChat = formatedChat.replace(inlay, '')
             }
         }