From cc4bbdc8e24308e8906d44c99028d0b3e7415767 Mon Sep 17 00:00:00 2001 From: Kwaroran Date: Fri, 20 Dec 2024 04:31:23 +0900 Subject: [PATCH] Enhance LLM model flags to include requiresAlternateRole and update request processing to handle multimodal and thoughts data --- src/ts/model/modellist.ts | 34 +++++++++++++++++----------------- src/ts/process/request.ts | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/ts/model/modellist.ts b/src/ts/model/modellist.ts index f5ce78db..30d4425f 100644 --- a/src/ts/model/modellist.ts +++ b/src/ts/model/modellist.ts @@ -778,7 +778,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-1.5-pro-exp-0827', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -787,7 +787,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-exp-1121', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud, }, @@ -796,7 +796,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-exp-1206', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -805,7 +805,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-2.0-flash-exp', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasAudioInput, LLMFlags.hasVideoInput, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasAudioInput, LLMFlags.hasVideoInput, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p', 'presence_penalty', 'frequency_penalty'], tokenizer: LLMTokenizer.GoogleCloud, recommended: true @@ -815,7 +815,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-2.0-flash-thinking-exp-1219', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasAudioInput, LLMFlags.hasVideoInput, LLMFlags.hasStreaming, LLMFlags.geminiThinking], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.poolSupported, LLMFlags.hasAudioInput, LLMFlags.hasVideoInput, LLMFlags.hasStreaming, LLMFlags.geminiThinking, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p', 'presence_penalty', 'frequency_penalty'], tokenizer: LLMTokenizer.GoogleCloud, recommended: true @@ -825,7 +825,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-1.5-pro-latest', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], recommended: true, parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud @@ -835,7 +835,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-1.5-flash', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], recommended: true, parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud @@ -846,7 +846,7 @@ export const LLMModels: LLMModel[] = [ internalID: 'gemini-exp-1121', provider: LLMProvider.VertexAI, format: LLMFormat.VertexAIGemini, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.Gemma }, @@ -856,7 +856,7 @@ export const LLMModels: LLMModel[] = [ internalID: 'gemini-1.5-pro-latest', provider: LLMProvider.VertexAI, format: LLMFormat.VertexAIGemini, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.Gemma }, @@ -866,7 +866,7 @@ export const LLMModels: LLMModel[] = [ internalID: 'gemini-1.5-flash', provider: LLMProvider.VertexAI, format: LLMFormat.VertexAIGemini, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.Gemma }, @@ -875,7 +875,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-exp-1114', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -884,7 +884,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-1.5-pro-002', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -893,7 +893,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-1.5-flash-002', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -902,7 +902,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-pro', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -911,7 +911,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-pro-vision', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -920,7 +920,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-ultra', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, @@ -929,7 +929,7 @@ export const LLMModels: LLMModel[] = [ id: 'gemini-ultra-vision', provider: LLMProvider.GoogleCloud, format: LLMFormat.GoogleCloud, - flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming], + flags: [LLMFlags.hasImageInput, LLMFlags.hasFirstSystemPrompt, LLMFlags.hasStreaming, LLMFlags.requiresAlternateRole], parameters: ['temperature', 'top_k', 'top_p'], tokenizer: LLMTokenizer.GoogleCloud }, diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index d36cc9d9..e618f841 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -282,7 +282,23 @@ function reformater(formated:OpenAIChat[],modelInfo:LLMModel){ } if(newFormated[newFormated.length-1].role === m.role){ + newFormated[newFormated.length-1].content += '\n' + m.content + + if(m.multimodals){ + if(!newFormated[newFormated.length-1].multimodals){ + newFormated[newFormated.length-1].multimodals = [] + } + newFormated[newFormated.length-1].multimodals.push(...m.multimodals) + } + + if(m.thoughts){ + if(!newFormated[newFormated.length-1].thoughts){ + newFormated[newFormated.length-1].thoughts = [] + } + newFormated[newFormated.length-1].thoughts.push(...m.thoughts) + } + continue } else{