diff --git a/src/lib/ChatScreens/Chat.svelte b/src/lib/ChatScreens/Chat.svelte index a8640b31..a29cd519 100644 --- a/src/lib/ChatScreens/Chat.svelte +++ b/src/lib/ChatScreens/Chat.svelte @@ -9,15 +9,15 @@ import { HideIconStore, ReloadGUIPointer, selIdState } from "../../ts/stores.svelte"; import { translateHTML } from "../../ts/translator/translator"; import { risuChatParser } from "src/ts/process/scripts"; - import { get, type Unsubscriber } from "svelte/store"; + import { type Unsubscriber } from "svelte/store"; import { isEqual } from "lodash"; import { sayTTS } from "src/ts/process/tts"; - import { getModelShortName } from "src/ts/model/names"; import { capitalize } from "src/ts/util"; import { longpress } from "src/ts/gui/longtouch"; import { ColorSchemeTypeStore } from "src/ts/gui/colorscheme"; import { ConnectionOpenStore } from "src/ts/sync/multiuser"; import { onDestroy, onMount } from "svelte"; + import { getModelInfo } from "src/ts/model/modellist"; let translating = $state(false) let editMode = $state(false) let statusMessage:string = $state('') @@ -232,7 +232,7 @@ > - {capitalize(getModelShortName(messageGenerationInfo.model))} + {capitalize(getModelInfo(messageGenerationInfo.model).shortName)} diff --git a/src/lib/UI/ModelList.svelte b/src/lib/UI/ModelList.svelte index 7b28068a..bf2da255 100644 --- a/src/lib/UI/ModelList.svelte +++ b/src/lib/UI/ModelList.svelte @@ -4,11 +4,8 @@ import { getHordeModels } from "src/ts/horde/getModels"; import Arcodion from "./Arcodion.svelte"; import { language } from "src/lang"; - import Help from "../Others/Help.svelte"; import CheckInput from "./GUI/CheckInput.svelte"; - import { isTauri } from "src/ts/globalApi.svelte"; - import {open} from '@tauri-apps/plugin-dialog' - import { getModelName } from "src/ts/model/names"; + import { getModelInfo, getModelList } from 'src/ts/model/modellist'; interface Props { value?: string; @@ -26,8 +23,11 @@ openOptions = false onChange(name) } - let showUnrec = $state(false) + let providers = $derived(getModelList({ + recommendedOnly: !showUnrec, + groupedByProvider: true + })) {#if openOptions} @@ -42,123 +42,20 @@

{language.model}

- - - - - - - {#if showUnrec} - - - - - - - - - - - - - - - - - - - - - + + {#each providers as provider} + {#if provider.providerName === '@as-is'} + {#each provider.models as model} + + {/each} + {:else} + + {#each provider.models as model} + + {/each} + {/if} - - - - - {#if showUnrec} - - - - - - - - - - - - - {/if} - - - {#if DBState.db.tpo && isTauri} - - {/if} - - {#if showUnrec} - - {/if} - - - - {#if showUnrec} - - - - {/if} - - - - - - - - - - - - - - - - - {#if showUnrec} - - - - - {/if} - - - - - - - - - - - - - - - - - - - - - + {/each} {#await getHordeModels()} @@ -175,15 +72,6 @@ {/each} {/await} - - {#if showUnrec} - - - - - - - {/if} {#if DBState.db.plugins.length > 0} {/if} @@ -197,6 +85,6 @@ diff --git a/src/ts/model/modellist.ts b/src/ts/model/modellist.ts index e70ecef5..462b3e79 100644 --- a/src/ts/model/modellist.ts +++ b/src/ts/model/modellist.ts @@ -4,31 +4,52 @@ export enum LLMFlags{ hasAudioInput, hasAudioOutput, hasPrefill, - hasCache + hasCache, + hasFullSystemPrompt, + hasFirstSystemPrompt, } export enum LLMProvider{ OpenAI, - Antropic, + Anthropic, GoogleCloud, VertexAI, AsIs, - Mistral + Mistral, + NovelList, + Cohere, + NovelAI, + WebLLM, + Horde, + AWS, } export enum LLMFormat{ OpenAICompatible, OpenAILegacyInstruct, - Antropic, + Anthropic, AnthropicLegacy, - AsIs, - Mistral + Mistral, + GoogleCloud, + VertexAIGemini, + NovelList, + Cohere, + NovelAI, + WebLLM, + OobaLegacy, + Plugin, + Ooba, + Kobold, + Ollama, + Horde, + AWSBedrockClaude } export interface LLMModel{ id: string name: string shortName?: string + fullName?: string internalID?: string provider: LLMProvider flags: LLMFlags[] @@ -36,6 +57,21 @@ export interface LLMModel{ recommended?: boolean } +const ProviderNames = new Map([ + [LLMProvider.OpenAI, 'OpenAI'], + [LLMProvider.Anthropic, 'Anthropic'], + [LLMProvider.GoogleCloud, 'Google Cloud'], + [LLMProvider.VertexAI, 'Vertex AI'], + [LLMProvider.AsIs, 'As Is'], + [LLMProvider.Mistral, 'MistralAI'], + [LLMProvider.NovelList, 'NovelList'], + [LLMProvider.Cohere, 'Cohere'], + [LLMProvider.NovelAI, 'NovelAI'], + [LLMProvider.WebLLM, 'WebLLM'], + [LLMProvider.Horde, 'Horde'], + [LLMProvider.AWS, 'AWS'], +]) + export const LLMModels: LLMModel[] = [ { id: 'gpt35', @@ -69,7 +105,8 @@ export const LLMModels: LLMModel[] = [ format: LLMFormat.OpenAICompatible, flags: [ LLMFlags.hasImageInput - ] + ], + recommended: true }, { id: 'gpt4om', @@ -79,7 +116,8 @@ export const LLMModels: LLMModel[] = [ format: LLMFormat.OpenAICompatible, flags: [ LLMFlags.hasImageInput - ] + ], + recommended: true }, { id: 'gpt4', @@ -257,116 +295,169 @@ export const LLMModels: LLMModel[] = [ format: LLMFormat.OpenAICompatible, flags: [], }, + { + name: "Claude 3.5 Sonnet", + id: 'claude-3-5-sonnet-latest', + shortName: "3.5 Sonnet", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, + flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], + recommended: true + }, + { + name: "Claude 3.5 Haiku", + id: 'claude-3-5-haiku-latest', + shortName: "3.5 Haiku", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, + flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], + recommended: true + }, { name: 'Claude 3.5 Sonnet (20241022)', id: 'claude-3-5-sonnet-20241022', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + shortName: "3.5 Sonnet 1022", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, + flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], + }, + { + name: "Claude 3.5 Haiku (20241022)", + id: 'claude-3-5-haiku-20241022', + shortName: "3.5 Haiku 1022", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], }, { name: 'Claude 3 Haiku (20240307)', id: 'claude-3-haiku-20240307', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + shortName: "3 Haiku 0307", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], }, { name: 'Claude 3.5 Sonnet (20240620)', id: 'claude-3-5-sonnet-20240620', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + shortName: "3.5 Sonnet 0620", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], }, { name: 'Claude 3 Opus (20240229)', id: 'claude-3-opus-20240229', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + shortName: "3 Opus 0229", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], }, { name: 'Claude 3 Sonnet (20240229)', id: 'claude-3-sonnet-20240229', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + shortName: "3 Sonnet 0229", + provider: LLMProvider.Anthropic, + format: LLMFormat.Anthropic, flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], }, { name: 'Claude 2.1', id: 'claude-2.1', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude 2', id: 'claude-2', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude 2 100k', id: 'claude-2-100k', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude v1', id: 'claude-v1', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude v1 100k', id: 'claude-v1-100k', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude Instant v1', id: 'claude-instant-v1', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude Instant v1 100k', id: 'claude-instant-v1-100k', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude v1.2', id: 'claude-1.2', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, { name: 'Claude v1.0', id: 'claude-1.0', - provider: LLMProvider.Antropic, - format: LLMFormat.Antropic, + provider: LLMProvider.Anthropic, + format: LLMFormat.AnthropicLegacy, flags: [LLMFlags.hasPrefill], }, + { + name: 'Claude 3.5 Sonnet (20241022) v2', + id: 'anthropic.claude-3-5-sonnet-20241022-v2:0', + provider: LLMProvider.AWS, + format: LLMFormat.AWSBedrockClaude, + flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], + }, + { + name: 'Claude 3.5 Sonnet (20240620) v1', + id: 'anthropic.claude-3-5-sonnet-20240620-v1:0', + provider: LLMProvider.AWS, + format: LLMFormat.AWSBedrockClaude, + flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], + }, + { + name: 'Claude 3 Opus (20240229) v1', + id: 'anthropic.claude-3-opus-20240229-v1:0', + provider: LLMProvider.AWS, + format: LLMFormat.AWSBedrockClaude, + flags: [LLMFlags.hasPrefill, LLMFlags.hasImageInput], + }, { name: 'Ooba', id: 'ooba', provider: LLMProvider.AsIs, - format: LLMFormat.AsIs, + format: LLMFormat.Ooba, flags: [], + recommended: true }, { name: 'Mancer', id: 'mancer', provider: LLMProvider.AsIs, - format: LLMFormat.AsIs, + format: LLMFormat.OobaLegacy, flags: [], }, { @@ -375,12 +466,382 @@ export const LLMModels: LLMModel[] = [ provider: LLMProvider.AsIs, format: LLMFormat.OpenAICompatible, flags: [], + recommended: true }, { - name: 'mistral-small-latest', + name: 'Mistral Small Latest', id: 'mistral-small-latest', + shortName: 'Mistral S', provider: LLMProvider.Mistral, format: LLMFormat.Mistral, flags: [], + recommended: true + }, + { + name: 'Mistral Medium Latest', + id: 'mistral-medium-latest', + shortName: 'Mistral M', + provider: LLMProvider.Mistral, + format: LLMFormat.Mistral, + flags: [], + recommended: true + }, + { + name: 'Mistral Large 2411', + id: 'mistral-large-2411', + shortName: 'Mistral L 2411', + provider: LLMProvider.Mistral, + format: LLMFormat.Mistral, + flags: [], + }, + { + name: 'Mistral Nemo', + id: 'open-mistral-nemo', + shortName: 'Mistral Nemo', + provider: LLMProvider.Mistral, + format: LLMFormat.Mistral, + flags: [], + }, + { + name: 'Mistral Large Latest', + id: 'mistral-large-latest', + shortName: 'Mistral L', + provider: LLMProvider.Mistral, + format: LLMFormat.Mistral, + flags: [], + recommended: true + }, + { + name: "Gemini Pro 1.5 0827", + id: 'gemini-1.5-pro-exp-0827', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Exp 1121", + id: 'gemini-exp-1121', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + recommended: true + }, + { + name: "Gemini Pro 1.5", + id: 'gemini-1.5-pro-latest', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + recommended: true + }, + { + name: "Gemini Flash 1.5", + id: 'gemini-1.5-flash', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + recommended: true + }, + { + name: "Gemini Exp 1121", + id: 'gemini-exp-1121-vertex', + internalID: 'gemini-exp-1121', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.VertexAIGemini, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Pro 1.5", + id: 'gemini-1.5-pro-latest-vertex', + internalID: 'gemini-1.5-pro-latest', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.VertexAIGemini, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Flash 1.5", + id: 'gemini-1.5-flash-vertex', + internalID: 'gemini-1.5-flash', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.VertexAIGemini, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Exp 1114", + id: 'gemini-exp-1114', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Pro 1.5 002", + id: 'gemini-1.5-pro-002', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Flash 1.5 002", + id: 'gemini-1.5-flash-002', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Pro", + id: 'gemini-pro', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Pro Vision", + id: 'gemini-pro-vision', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Ultra", + id: 'gemini-ultra', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: "Gemini Ultra Vision", + id: 'gemini-ultra-vision', + provider: LLMProvider.GoogleCloud, + format: LLMFormat.GoogleCloud, + flags: [LLMFlags.hasImageInput], + }, + { + name: 'Kobold', + id: 'kobold', + provider: LLMProvider.AsIs, + format: LLMFormat.Kobold, + flags: [], + recommended: true + }, + { + name: "SuperTrin", + id: 'novellist', + provider: LLMProvider.NovelList, + format: LLMFormat.NovelList, + flags: [], + }, + { + name: "Damsel", + id: 'novellist_damsel', + provider: LLMProvider.NovelList, + format: LLMFormat.NovelList, + flags: [], + }, + { + name: "Command R", + id: 'cohere-command-r', + internalID: 'command-r', + provider: LLMProvider.Cohere, + format: LLMFormat.Cohere, + flags: [], + recommended: true + }, + { + name: "Command R Plus", + id: 'cohere-command-r-plus', + internalID: 'command-r-plus', + provider: LLMProvider.Cohere, + format: LLMFormat.Cohere, + flags: [], + recommended: true + }, + { + name: "Command R 08-2024", + id: 'cohere-command-r-08-2024', + internalID: 'command-r-08-2024', + provider: LLMProvider.Cohere, + format: LLMFormat.Cohere, + flags: [], + }, + { + name: "Command R 03-2024", + id: 'cohere-command-r-03-2024', + internalID: 'command-r-03-2024', + provider: LLMProvider.Cohere, + format: LLMFormat.Cohere, + flags: [], + }, + { + name: "Command R Plus 08-2024", + id: 'cohere-command-r-plus-08-2024', + internalID: 'command-r-plus-08-2024', + provider: LLMProvider.Cohere, + format: LLMFormat.Cohere, + flags: [], + }, + { + name: "Command R Plus 04-2024", + id: 'cohere-command-r-plus-04-2024', + internalID: 'command-r-plus-04-2024', + provider: LLMProvider.Cohere, + format: LLMFormat.Cohere, + flags: [], + }, + { + name: "Clio", + id: 'novelai', + provider: LLMProvider.NovelAI, + format: LLMFormat.NovelAI, + flags: [], + recommended: true + }, + { + name: "Kayra", + id: 'novelai_kayra', + provider: LLMProvider.NovelAI, + format: LLMFormat.NovelAI, + flags: [], + recommended: true + }, + { + id: 'ollama-hosted', + name: 'Ollama', + provider: LLMProvider.AsIs, + format: LLMFormat.Ollama, + flags: [], + }, + { + id: 'hf:::Xenova/opt-350m', + name: 'opt-350m', + provider: LLMProvider.WebLLM, + format: LLMFormat.WebLLM, + flags: [], + }, + { + id: 'hf:::Xenova/tiny-random-mistral', + name: 'tiny-random-mistral', + provider: LLMProvider.WebLLM, + format: LLMFormat.WebLLM, + flags: [], + }, + { + id: 'hf:::Xenova/gpt2-large-conversational', + name: 'gpt2-large-conversational', + provider: LLMProvider.WebLLM, + format: LLMFormat.WebLLM, + flags: [], + }, + { + id: 'custom', + name: "Plugin", + provider: LLMProvider.AsIs, + format: LLMFormat.Plugin, + flags: [], + recommended: true + }, + { + id: 'reverse_proxy', + name: "Custom API", + provider: LLMProvider.AsIs, + format: LLMFormat.OpenAICompatible, + flags: [], + recommended: true } -] \ No newline at end of file +] + +for(let model of LLMModels){ + model.shortName ??= model.name + model.internalID ??= model.id + model.fullName ??= model.provider !== LLMProvider.AsIs ? `${ProviderNames.get(model.provider) ?? ''} ${model.name}`.trim() : model.name +} + +export function getModelInfo(id: string): LLMModel{ + + const found = LLMModels.find(model => model.id === id) ?? { + id, + name: id, + provider: LLMProvider.AsIs, + format: LLMFormat.OpenAICompatible, + flags: [], + } + + if(found) return found + + if(id.startsWith('hf:::')){ + const withoutPrefix = id.replace('hf:::', '') + return { + id, + name: withoutPrefix, + shortName: withoutPrefix, + fullName: withoutPrefix, + internalID: withoutPrefix, + provider: LLMProvider.WebLLM, + format: LLMFormat.WebLLM, + flags: [], + } + } + if(id.startsWith('horde:::')){ + const withoutPrefix = id.replace('horde:::', '') + return { + id, + name: withoutPrefix, + shortName: withoutPrefix, + fullName: withoutPrefix, + internalID: withoutPrefix, + provider: LLMProvider.Horde, + format: LLMFormat.Horde, + flags: [], + } + } + + return { + id, + name: id, + shortName: id, + fullName: id, + internalID: id, + provider: LLMProvider.AsIs, + format: LLMFormat.OpenAICompatible, + flags: [], + } +} + +interface GetModelListGroup { + providerName: string + models: LLMModel[] +} + +export function getModelList(arg:{ + recommendedOnly?:boolean, + groupedByProvider?:T +} = {}): T extends true ? GetModelListGroup[] : LLMModel[]{ + let models = LLMModels + if(arg.recommendedOnly){ + models = models.filter(model => model.recommended) + } + if(arg.groupedByProvider){ + let group: GetModelListGroup[] = [] + for(let model of models){ + if(model.provider === LLMProvider.AsIs){ + group.push({ + providerName: '@as-is', + models: [model] + }) + continue + } + + let providerName = ProviderNames.get(model.provider) || 'Unknown' + let groupIndex = group.findIndex(g => g.providerName === providerName) + if(groupIndex === -1){ + group.push({ + providerName, + models: [model] + }) + }else{ + group[groupIndex].models.push(model) + } + } + return group as any + } + return models as any +} \ No newline at end of file diff --git a/src/ts/model/names.ts b/src/ts/model/names.ts deleted file mode 100644 index ea34842e..00000000 --- a/src/ts/model/names.ts +++ /dev/null @@ -1,204 +0,0 @@ - -export function getModelName(name:string){ - switch(name){ - case "gpt35": - return "GPT-3.5 Turbo" - case "gpt35_0613": - return "GPT-3.5 Turbo 0613" - case "gpt35_0301": - return "GPT-3.5 Turbo 0301" - case "gpt35_16k": - return "GPT-3.5 Turbo 16k" - case "gpt35_16k_0613": - return "GPT-3.5 Turbo 16k 0613" - case 'instructgpt35': - return 'GPT-3.5 Turbo Instruct' - case "gpt4": - return "GPT-4" - case "gpt4_0301": - return "GPT-4 0301" - case "gpt4_32k": - return "GPT-4 32k" - case "gpt4_0613": - return "GPT-4 0613" - case "gpt4_32k_0613": - return "GPT-4 32k 0613" - case "gpt4_1106": - return "GPT-4 Turbo 1106" - case 'gpt45': - return 'GPT-4.5' - case "gpt35_1106": - return "GPT-3.5 Turbo 1106" - case 'local_gptq': - return 'Local Model GPTQ' - case "palm2": - return "PaLM2 Bison" - case "textgen_webui": - return "Oobabooga Legacy" - case 'ooba': - return 'Oobabooga' - case "mancer": - return "Mancer" - case "kobold": - return "Kobold" - case "custom": - return "Plugin" - case "novelai": - return "NovelAI Clio" - case "novelai_kayra": - return "NovelAI Kayra" - case "novellist": - return "NovelList SuperTrin" - case "novellist damsel": - return "NovelList Damsel" - case 'reverse_proxy': - return "Custom (OpenAI-compatible)" - case 'openrouter': - return "OpenRouter" - case 'gptvi4_1106': - return "GPT-4 Turbo 1106 Vision" - case 'palm2_unicorn': - return "PaLM2 Unicorn" - case 'mistral-tiny': - return "Mistral Tiny" - case 'mistral-small': - return "Mistral Small" - case 'mistral-medium': - return "Mistral Medium" - case 'gemini-pro': - return "Gemini Pro" - case 'horde:::auto': - return 'Horde Auto Model' - case 'gpt4_0125': - return 'GPT-4 Turbo 0125' - case 'gpt35_0125': - return 'GPT-3.5 Turbo 0125' - case 'gemini-ultra': - return 'Gemini Ultra' - case 'gemini-ultra-vision': - return 'Gemini Ultra Vision' - case 'claude-3-opus-20240229': - return 'Claude 3 Opus (20240229)' - case 'claude-3-5-sonnet-20240620': - return 'Claude 3.5 Sonnet (20240620)' - case 'claude-3-5-sonnet-20241022': - return 'Claude 3.5 Sonnet (20241022)' - case 'claude-3-sonnet-20240229': - return 'Claude 3 Sonnet (20240229)' - case 'mistral-large-latest': - return 'Mistral Large' - case 'mistral-small-latest': - return 'Mistral Small' - case 'mistral-medium-latest': - return 'Mistral Medium' - case 'claude-3-haiku-20240307': - return 'Claude 3 Haiku (20240307)' - case 'gpt4_turbo': - return 'GPT-4 Turbo' - case 'gpt4_turbo_20240409': - return 'GPT-4 Turbo (20240409)' - case 'gpt4o': - return 'GPT-4o' - case 'gpt4o-2024-05-13': - return 'GPT-4o (2024-05-13)' - case 'gpt4o-2024-08-06': - return 'GPT-4o (2024-08-06)' - case 'gpt4o-2024-11-20': - return 'GPT-4o (2024-11-20)' - case 'gpt4o-chatgpt': - return 'GPT-4o ChatGPT' - case 'gpt4om': - return 'GPT-4o Mini' - case 'gpt4o1-preview': - return 'o1 Preview' - case 'gpt4o1-mini': - return 'o1 Mini' - case 'gpt4om-2024-07-18': - return 'GPT-4o Mini (2024-07-18)' - case 'gemini-1.5-pro-latest': - return 'Gemini 1.5 Pro' - case 'gemini-1.5-pro-exp-0801': - return 'Gemini 1.5 Pro Exp (0801)' - case 'gemini-1.5-pro-exp-0827': - return 'Gemini 1.5 Pro Exp (0827)' - case 'gemini-1.5-flash': - return 'Gemini 1.5 Flash' - case 'ollama-hosted': - return 'Ollama' - case 'cohere-command-r': - return 'Cohere Command-R' - case 'cohere-command-r-plus': - return 'Cohere Command-R Plus' - default: - if(name.startsWith("horde:::")){ - const split = name.split(":::") - return `Horde ${split[1]}` - } - if(name.startsWith('tf:::')){ - const split = name.split(":::") - return `${split[1]}` - } - if(name.startsWith('local_')){ - const realName = name.replace('local_', '').split(/(\\|\/)/g).at(-1) - return `GGUF ${realName}` - } - return name - } -} - -export function getModelShortName(model:string){ - if(model.startsWith("gpt35")){ - return "GPT-3.5" - } - if(model.startsWith("cohere-")){ - return model.replace("cohere-", "") - } - if(model.startsWith('gpt4om')){ - return "GPT-4o Mini" - } - if(model.startsWith("gpt4o")){ - return "GPT-4o" - } - if(model.startsWith("gpt4")){ - return "GPT-4" - } - if(model.startsWith("gptvi4")){ - return "GPT-4V" - } - if(model.startsWith("mistral")){ - return getModelName(model).split(" ").at(-1) - } - if(model.startsWith("mancer")){ - return "Mancer" - } - if(model.startsWith('tf:::')){ - const split = model.split(":::") - return split[1] - } - if(model.startsWith('local_')){ - const realName = model.replace('local_', '').split(/(\\|\/)/g).at(-1) - return realName - } - if(model.startsWith('horde:::')){ - const split = model.split(":::") - return split[1] - } - - if(model.startsWith('claude-3')){ - const split = model.split("-") - if(!isNaN(parseInt(split[split.length-1]))){ - return split[split.length-2] - } - else{ - return split[split.length-1] - } - } - if(model.startsWith('reverse_proxy')){ - return 'Custom' - } - if(model.startsWith('oaicomp')){ - return 'Custom' - } - return getModelName(model) - -} \ No newline at end of file diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index e22c24d2..0932b9c5 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1,6 +1,6 @@ import { get } from "svelte/store"; import type { MultiModal, OpenAIChat, OpenAIChatFull } from "./index.svelte"; -import { getDatabase, type character } from "../storage/database.svelte"; +import { getCurrentCharacter, getDatabase, type character } from "../storage/database.svelte"; import { pluginProcess } from "../plugins/plugins"; import { language } from "../../lang"; import { stringlizeAINChat, stringlizeChat, getStopStrings, unstringlizeAIN, unstringlizeChat } from "./stringlize"; @@ -23,6 +23,7 @@ import {Ollama} from 'ollama/dist/browser.mjs' import { applyChatTemplate } from "./templates/chatTemplate"; import { OobaParams } from "./prompt"; import { extractJSON, getOpenAIJSONSchema } from "./templates/jsonSchema"; +import { getModelInfo, LLMFormat, type LLMModel } from "../model/modellist"; @@ -43,6 +44,14 @@ interface requestDataArgument{ noMultiGen?:boolean } +interface RequestDataArgumentExtended extends requestDataArgument{ + aiModel?:string + multiGen?:boolean + realAIModel?:string + abortSignal?:AbortSignal + modelInfo?:LLMModel +} + type requestDataResponse = { type: 'success'|'fail' result: string @@ -179,2531 +188,2260 @@ export interface OpenAIChatExtra { export async function requestChatDataMain(arg:requestDataArgument, model:'model'|'submodel', abortSignal:AbortSignal=null):Promise { const db = getDatabase() - let formated = safeStructuredClone(arg.formated) - let maxTokens = arg.maxTokens ??db.maxResponse - let temperature = arg.temperature ?? (db.temperature / 100) - let bias = arg.bias - let currentChar = arg.currentChar - let useStreaming = db.useStreaming && arg.useStreaming - arg.continue = arg.continue ?? false - let biasString = arg.biasString ?? [] - const aiModel = model === 'model' ? db.aiModel : db.subModel - const multiGen = (db.genTime > 1 && aiModel.startsWith('gpt') && (!arg.continue)) && (!arg.noMultiGen) - - let raiModel = aiModel - if(aiModel === 'reverse_proxy'){ + const targ:RequestDataArgumentExtended = arg + targ.formated = safeStructuredClone(arg.formated) + targ.maxTokens = arg.maxTokens ??db.maxResponse + targ.temperature = arg.temperature ?? (db.temperature / 100) + targ.bias = arg.bias + targ.currentChar = arg.currentChar + targ.useStreaming = db.useStreaming && arg.useStreaming + targ.continue = arg.continue ?? false + targ.biasString = arg.biasString ?? [] + targ.aiModel = (model === 'model' ? db.aiModel : db.subModel) + targ.multiGen = ((db.genTime > 1 && targ.aiModel.startsWith('gpt') && (!arg.continue)) && (!arg.noMultiGen)) + targ.realAIModel = targ.aiModel + targ.abortSignal = abortSignal + targ.modelInfo = getModelInfo(targ.aiModel) + if(targ.aiModel === 'reverse_proxy'){ if(db.proxyRequestModel === 'custom' && db.customProxyRequestModel.startsWith('claude')){ - raiModel = db.customProxyRequestModel + targ.realAIModel = db.customProxyRequestModel } if(db.proxyRequestModel.startsWith('claude')){ - raiModel = db.proxyRequestModel + targ.realAIModel = db.proxyRequestModel } if(db.forceProxyAsOpenAI){ - raiModel = 'reverse_proxy' + targ.realAIModel = 'reverse_proxy' } } - console.log(formated) - switch(raiModel){ - case 'gpt35': - case 'gpt35_0613': - case 'gpt35_16k': - case 'gpt35_16k_0613': - case 'gpt4': - case 'gpt45': - case 'gpt4_32k': - case 'gpt4_0613': - case 'gpt4_32k_0613': - case 'gpt4_1106': - case 'gpt4_0125': - case 'gpt35_0125': - case 'gpt35_1106': - case 'gpt35_0301': - case 'gpt4_0314': - case 'gptvi4_1106': - case 'openrouter': - case 'mistral-tiny': - case 'mistral-small': - case 'mistral-medium': - case 'mistral-small-latest': - case 'mistral-medium-latest': - case 'mistral-large-latest': - case 'mistral-large-2411': - case 'open-mistral-nemo': - case 'gpt4_turbo_20240409': - case 'gpt4_turbo': - case 'gpt4o': - case 'gpt4o-2024-05-13': - case 'gpt4om': - case 'gpt4om-2024-07-18': - case 'gpt4o-2024-08-06': - case 'gpt4o-2024-11-20': - case 'gpt4o-chatgpt': - case 'gpt4o1-preview': - case 'gpt4o1-mini': - case 'jamba-1.5-large': - case 'jamba-1.5-medium': - case 'reverse_proxy':{ - let formatedChat:OpenAIChatExtra[] = [] - for(let i=0;i 0 && m.role === 'user'){ - let v:OpenAIChatExtra = safeStructuredClone(m) - let contents:OpenAIContents[] = [] - for(let j=0;j{ + let formatedChat:OpenAIChatExtra[] = [] + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + for(let i=0;i 0 && m.role === 'user'){ + let v:OpenAIChatExtra = safeStructuredClone(m) + let contents:OpenAIContents[] = [] + for(let j=0;j 0){ + formatedChat.push({ + role: 'system', + content: oobaSystemPrompts.join('\n') + }) + } + + + if(db.newOAIHandle){ + formatedChat = formatedChat.filter(m => { + return m.content !== '' + }) + } + + if(aiModel.startsWith('gpt4o1')){ + for(let i=0;i${formatedChat[i].content}` + formatedChat[i].role = 'user' + } + } + } + + for(let i=0;i 0){ - formatedChat.push({ - role: 'system', - content: oobaSystemPrompts.join('\n') - }) - } - - - if(db.newOAIHandle){ - formatedChat = formatedChat.filter(m => { - return m.content !== '' - }) - } - - if(aiModel.startsWith('gpt4o1')){ - for(let i=0;i${formatedChat[i].content}` - formatedChat[i].role = 'user' - } - } - } - - for(let i=0;i 0){ - body.seed = db.generationSeed - } - - if(db.putUserOpen){ - body.user = getOpenUserString() - } - - if(db.jsonSchemaEnabled){ - body.response_format = { - "type": "json_schema", - "json_schema": getOpenAIJSONSchema() - } - } - - if(db.OAIPrediction){ - body.prediction = { - type: "content", - content: db.OAIPrediction - } - } - - if(aiModel === 'openrouter'){ - if(db.openrouterFallback){ - body.route = "fallback" - } - body.transforms = db.openrouterMiddleOut ? ['middle-out'] : [] - - if(db.openrouterProvider){ - body.provider = { - order: [db.openrouterProvider] - } - } - - if(db.useInstructPrompt){ - delete body.messages - const prompt = applyChatTemplate(formated) - body.prompt = prompt - } - } - - body = applyParameters(body, - aiModel === 'openrouter' ? ['temperature', 'top_p', 'frequency_penalty', 'presence_penalty', 'repetition_penalty', 'min_p', 'top_a', 'top_k'] : ['temperature', 'top_p', 'frequency_penalty', 'presence_penalty'] - ) - - if(aiModel === 'reverse_proxy' && db.reverseProxyOobaMode){ - const OobaBodyTemplate = db.reverseProxyOobaArgs - - const keys = Object.keys(OobaBodyTemplate) - for(const key of keys){ - if(OobaBodyTemplate[key] !== undefined && OobaBodyTemplate[key] !== null){ - // @ts-ignore - body[key] = OobaBodyTemplate[key] - } - } - - } - - if(supportsInlayImage()){ - // inlay models doesn't support logit_bias - // OpenAI's gpt based llm model supports both logit_bias and inlay image - if(!( - aiModel.startsWith('gpt') || - (aiModel == 'reverse_proxy' && ( - db.proxyRequestModel?.startsWith('gpt') || - (db.proxyRequestModel === 'custom' && db.customProxyRequestModel.startsWith('gpt')) - )))){ - // @ts-ignore - delete body.logit_bias - } - } - - let replacerURL = aiModel === 'openrouter' ? "https://openrouter.ai/api/v1/chat/completions" : - (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.openai.com/v1/chat/completions') - - let risuIdentify = false - if(replacerURL.startsWith("risu::")){ - risuIdentify = true - replacerURL = replacerURL.replace("risu::", '') - } - - if(aiModel === 'reverse_proxy' && db.autofillRequestUrl){ - if(replacerURL.endsWith('v1')){ - replacerURL += '/chat/completions' - } - else if(replacerURL.endsWith('v1/')){ - replacerURL += 'chat/completions' - } - else if(!(replacerURL.endsWith('completions') || replacerURL.endsWith('completions/'))){ - if(replacerURL.endsWith('/')){ - replacerURL += 'v1/chat/completions' - } - else{ - replacerURL += '/v1/chat/completions' - } - } - } - - let headers = { - "Authorization": "Bearer " + (aiModel === 'reverse_proxy' ? db.proxyKey : (aiModel === 'openrouter' ? db.openrouterKey : db.openAIKey)), - "Content-Type": "application/json" - } - - if(aiModel === 'openrouter'){ - headers["X-Title"] = 'RisuAI' - headers["HTTP-Referer"] = 'https://risuai.xyz' - } - if(risuIdentify){ - headers["X-Proxy-Risu"] = 'RisuAI' - } - if(aiModel.startsWith('jamba')){ - headers['Authorization'] = 'Bearer ' + db.ai21Key - replacerURL = 'https://api.ai21.com/studio/v1/chat/completions' - } - if(multiGen){ - // @ts-ignore - body.n = db.genTime - } - let throughProxi = (!isTauri) && (!isNodeServer) && (!db.usePlainFetch) && (!Capacitor.isNativePlatform()) - if(useStreaming){ - body.stream = true - let urlHost = new URL(replacerURL).host - if(urlHost.includes("localhost") || urlHost.includes("172.0.0.1") || urlHost.includes("0.0.0.0")){ - if(!isTauri){ - return { - type: 'fail', - result: 'You are trying local request on streaming. this is not allowed dude to browser/os security policy. turn off streaming.', - } - } - } - const da = await fetchNative(replacerURL, { - body: JSON.stringify(body), - method: "POST", - headers: headers, - signal: abortSignal, - chatId: arg.chatId - }) - - if(da.status !== 200){ - return { - type: "fail", - result: await textifyReadableStream(da.body) - } - } - - if (!da.headers.get('Content-Type').includes('text/event-stream')){ - return { - type: "fail", - result: await textifyReadableStream(da.body) - } - } - - addFetchLog({ - body: body, - response: "Streaming", - success: true, - url: replacerURL, - }) - - let dataUint:Uint8Array|Buffer = new Uint8Array([]) - - const transtream = new TransformStream( { - async transform(chunk, control) { - dataUint = Buffer.from(new Uint8Array([...dataUint, ...chunk])) - let JSONreaded:{[key:string]:string} = {} - try { - const datas = dataUint.toString().split('\n') - let readed:{[key:string]:string} = {} - for(const data of datas){ - if(data.startsWith("data: ")){ - try { - const rawChunk = data.replace("data: ", "") - if(rawChunk === "[DONE]"){ - if(db.extractJson && db.jsonSchemaEnabled){ - for(const key in readed){ - const extracted = extractJSON(readed[key], db.extractJson) - JSONreaded[key] = extracted - } - console.log(JSONreaded) - control.enqueue(JSONreaded) - } - else{ - control.enqueue(readed) - } - return - } - const choices = JSON.parse(rawChunk).choices - for(const choice of choices){ - const chunk = choice.delta.content ?? choices.text - if(chunk){ - if(multiGen){ - const ind = choice.index.toString() - if(!readed[ind]){ - readed[ind] = "" - } - readed[ind] += chunk - } - else{ - if(!readed["0"]){ - readed["0"] = "" - } - readed["0"] += chunk - } - } - } - } catch (error) {} - } - } - if(db.extractJson && db.jsonSchemaEnabled){ - for(const key in readed){ - const extracted = extractJSON(readed[key], db.extractJson) - JSONreaded[key] = extracted - } - console.log(JSONreaded) - control.enqueue(JSONreaded) - } - else{ - control.enqueue(readed) - } - } catch (error) { - - } - } - },) - - da.body.pipeTo(transtream.writable) - - return { - type: 'streaming', - result: transtream.readable - } - } - - if(raiModel === 'reverse_proxy'){ - const additionalParams = db.additionalParams - for(let i=0;i { - const extracted = extractJSON(v.message.content, db.extractJson) - return ["char",extracted] - }) - - return { - type: 'multiline', - result: c - } - - } - return { - type: 'multiline', - result: dat.choices.map((v) => { - return ["char",v.message.content] - }) - } - - } - - if(dat?.choices[0]?.text){ - if(db.extractJson && db.jsonSchemaEnabled){ - try { - const parsed = JSON.parse(dat.choices[0].text) - const extracted = extractJSON(parsed, db.extractJson) - return { - type: 'success', - result: extracted - } - } catch (error) { - console.log(error) - return { - type: 'success', - result: dat.choices[0].text - } - } - } - return { - type: 'success', - result: dat.choices[0].text - } - } - if(db.extractJson && db.jsonSchemaEnabled){ - return { - type: 'success', - result: extractJSON(dat.choices[0].message.content, db.extractJson) - } - } - const msg:OpenAIChatFull = (dat.choices[0].message) - return { - type: 'success', - result: msg.content - } - } catch (error) { - return { - type: 'fail', - result: (language.errors.httpError + `${JSON.stringify(dat)}`) - } - } - } - else{ - if(dat.error && dat.error.message){ - return { - type: 'fail', - result: (language.errors.httpError + `${dat.error.message}`) - } - } - else{ - return { - type: 'fail', - result: (language.errors.httpError + `${JSON.stringify(res.data)}`) - } - } - } - - break - } - case 'novelai': - case 'novelai_kayra':{ - console.log(arg.continue) - const prompt = stringlizeNAIChat(formated, currentChar?.name ?? '', arg.continue) - let logit_bias_exp:{ - sequence: number[], bias: number, ensure_sequence_finish: false, generate_once: true - }[] = [] - - for(let i=0;i m.content?.trim()).map(m => { - let author = ''; - - if(m.role == 'system'){ - m.content = m.content.trim(); - } - - console.log(m.role +":"+m.content); - switch (m.role) { - case 'user': author = 'User'; break; - case 'assistant': author = 'Assistant'; break; - case 'system': author = 'Instruction'; break; - default: author = m.role; break; - } - - return `\n## ${author}\n${m.content.trim()}`; - //return `\n\n${author}: ${m.content.trim()}`; - }).join("") + `\n## Response\n`; - - const response = await globalFetch( "https://api.openai.com/v1/completions", { - body: { - model: "gpt-3.5-turbo-instruct", - prompt: prompt, - max_tokens: maxTokens, - temperature: temperature, - top_p: 1, - stop:["User:"," User:", "user:", " user:"], - presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), - frequency_penalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), - }, - headers: { - "Content-Type": "application/json", - "Authorization": "Bearer " + db.openAIKey, - }, - chatId: arg.chatId - }); - - if(!response.ok){ - return { - type: 'fail', - result: (language.errors.httpError + `${JSON.stringify(response.data)}`) - } - } - const text:string = response.data.choices[0].text - return { - type: 'success', - result: text.replace(/##\n/g, '') - } - } - case "textgen_webui": - case 'mancer':{ - let streamUrl = db.textgenWebUIStreamURL.replace(/\/api.*/, "/api/v1/stream") - let blockingUrl = db.textgenWebUIBlockingURL.replace(/\/api.*/, "/api/v1/generate") - let bodyTemplate:{[key:string]:any} = {} - const suggesting = model === "submodel" - const prompt = applyChatTemplate(formated) - let stopStrings = getStopStrings(suggesting) - if(db.localStopStrings){ - stopStrings = db.localStopStrings.map((v) => { - return risuChatParser(v.replace(/\\n/g, "\n")) - }) - } - bodyTemplate = { - 'max_new_tokens': db.maxResponse, - 'do_sample': db.ooba.do_sample, - 'temperature': (db.temperature / 100), - 'top_p': db.ooba.top_p, - 'typical_p': db.ooba.typical_p, - 'repetition_penalty': db.ooba.repetition_penalty, - 'encoder_repetition_penalty': db.ooba.encoder_repetition_penalty, - 'top_k': db.ooba.top_k, - 'min_length': db.ooba.min_length, - 'no_repeat_ngram_size': db.ooba.no_repeat_ngram_size, - 'num_beams': db.ooba.num_beams, - 'penalty_alpha': db.ooba.penalty_alpha, - 'length_penalty': db.ooba.length_penalty, - 'early_stopping': false, - 'truncation_length': maxTokens, - 'ban_eos_token': db.ooba.ban_eos_token, - 'stopping_strings': stopStrings, - 'seed': -1, - add_bos_token: db.ooba.add_bos_token, - topP: db.top_p, - prompt: prompt - } - - const headers = (aiModel === 'textgen_webui') ? {} : { - 'X-API-KEY': db.mancerHeader - } - - if(useStreaming){ - const oobaboogaSocket = new WebSocket(streamUrl); - const statusCode = await new Promise((resolve) => { - oobaboogaSocket.onopen = () => resolve(0) - oobaboogaSocket.onerror = () => resolve(1001) - oobaboogaSocket.onclose = ({ code }) => resolve(code) - }) - if(abortSignal.aborted || statusCode !== 0) { - oobaboogaSocket.close() - return ({ - type: "fail", - result: abortSignal.reason || `WebSocket connection failed to '${streamUrl}' failed!`, + else if(chat.role === 'function'){ + reformatedChat.push({ + role: 'user', + content: chat.content }) } - - const close = () => { - oobaboogaSocket.close() + else{ + reformatedChat.push({ + role: chat.role, + content: chat.content + }) } - const stream = new ReadableStream({ - start(controller){ - let readed = ""; - oobaboogaSocket.onmessage = async (event) => { - const json = JSON.parse(event.data); - if (json.event === "stream_end") { - close() - controller.close() - return - } - if (json.event !== "text_stream") return - readed += json.text - controller.enqueue(readed) - }; - oobaboogaSocket.send(JSON.stringify(bodyTemplate)); - }, - cancel(){ - close() - } - }) - oobaboogaSocket.onerror = close - oobaboogaSocket.onclose = close - abortSignal.addEventListener("abort", close) + } + } + + const res = await globalFetch("https://api.mistral.ai/v1/chat/completions", { + body: { + model: requestModel, + messages: reformatedChat, + temperature: arg.temperature, + max_tokens: arg.maxTokens, + top_p: db.top_p, + safe_prompt: false + }, + headers: { + "Authorization": "Bearer " + db.mistralKey, + }, + abortSignal: arg.abortSignal, + chatId: arg.chatId + }) + const dat = res.data as any + if(res.ok){ + try { + const msg:OpenAIChatFull = (dat.choices[0].message) return { - type: 'streaming', - result: stream + type: 'success', + result: msg.content + } + } catch (error) { + return { + type: 'fail', + result: (language.errors.httpError + `${JSON.stringify(dat)}`) } } - - const res = await globalFetch(blockingUrl, { - body: bodyTemplate, - headers: headers, - abortSignal, - chatId: arg.chatId - }) - - const dat = res.data as any - if(res.ok){ - try { - let result:string = dat.results[0].text - if(suggesting){ - result = "\n" + db.autoSuggestPrefix + result - } - - return { - type: 'success', - result: unstringlizeChat(result, formated, currentChar?.name ?? '') - } - } catch (error) { - return { - type: 'fail', - result: (language.errors.httpError + `${error}`) - } + } + else{ + if(dat.error && dat.error.message){ + return { + type: 'fail', + result: (language.errors.httpError + `${dat.error.message}`) } } - else{ + else{ return { type: 'fail', result: (language.errors.httpError + `${JSON.stringify(res.data)}`) } } } - - case 'ooba': { - const suggesting = model === "submodel" + } + + db.cipherChat = false + let body:{ + [key:string]:any + } = ({ + model: aiModel === 'openrouter' ? openrouterRequestModel : + requestModel === 'gpt35' ? 'gpt-3.5-turbo' + : requestModel === 'gpt35_0613' ? 'gpt-3.5-turbo-0613' + : requestModel === 'gpt35_16k' ? 'gpt-3.5-turbo-16k' + : requestModel === 'gpt35_16k_0613' ? 'gpt-3.5-turbo-16k-0613' + : requestModel === 'gpt4' ? 'gpt-4' + : requestModel === 'gpt45' ? 'gpt-4.5-preview' + : requestModel === 'gpt4_32k' ? 'gpt-4-32k' + : requestModel === "gpt4_0613" ? 'gpt-4-0613' + : requestModel === "gpt4_32k_0613" ? 'gpt-4-32k-0613' + : requestModel === "gpt4_1106" ? 'gpt-4-1106-preview' + : requestModel === 'gpt4_0125' ? 'gpt-4-0125-preview' + : requestModel === "gptvi4_1106" ? 'gpt-4-vision-preview' + : requestModel === "gpt35_0125" ? 'gpt-3.5-turbo-0125' + : requestModel === "gpt35_1106" ? 'gpt-3.5-turbo-1106' + : requestModel === 'gpt35_0301' ? 'gpt-3.5-turbo-0301' + : requestModel === 'gpt4_0314' ? 'gpt-4-0314' + : requestModel === 'gpt4_turbo_20240409' ? 'gpt-4-turbo-2024-04-09' + : requestModel === 'gpt4_turbo' ? 'gpt-4-turbo' + : requestModel === 'gpt4o' ? 'gpt-4o' + : requestModel === 'gpt4o-2024-05-13' ? 'gpt-4o-2024-05-13' + : requestModel === 'gpt4om' ? 'gpt-4o-mini' + : requestModel === 'gpt4om-2024-07-18' ? 'gpt-4o-mini-2024-07-18' + : requestModel === 'gpt4o-2024-08-06' ? 'gpt-4o-2024-08-06' + : requestModel === 'gpt4o-2024-11-20' ? 'gpt-4o-2024-11-20' + : requestModel === 'gpt4o-chatgpt' ? 'chatgpt-4o-latest' + : requestModel === 'gpt4o1-preview' ? 'o1-preview' + : requestModel === 'gpt4o1-mini' ? 'o1-mini' + : (!requestModel) ? 'gpt-3.5-turbo' + : requestModel, + messages: formatedChat, + max_tokens: arg.maxTokens, + logit_bias: arg.bias, + stream: false, + + }) + + if(aiModel.startsWith('gpt4o1')){ + body.max_completion_tokens = body.max_tokens + delete body.max_tokens + } + + if(db.generationSeed > 0){ + body.seed = db.generationSeed + } + + if(db.jsonSchemaEnabled){ + body.response_format = { + "type": "json_schema", + "json_schema": getOpenAIJSONSchema() + } + } + + if(db.OAIPrediction){ + body.prediction = { + type: "content", + content: db.OAIPrediction + } + } + + if(aiModel === 'openrouter'){ + if(db.openrouterFallback){ + body.route = "fallback" + } + body.transforms = db.openrouterMiddleOut ? ['middle-out'] : [] + + if(db.openrouterProvider){ + body.provider = { + order: [db.openrouterProvider] + } + } + + if(db.useInstructPrompt){ + delete body.messages const prompt = applyChatTemplate(formated) - let stopStrings = getStopStrings(suggesting) - if(db.localStopStrings){ - stopStrings = db.localStopStrings.map((v) => { - return risuChatParser(v.replace(/\\n/g, "\n")) - }) + body.prompt = prompt + } + } + + body = applyParameters(body, + aiModel === 'openrouter' ? ['temperature', 'top_p', 'frequency_penalty', 'presence_penalty', 'repetition_penalty', 'min_p', 'top_a', 'top_k'] : ['temperature', 'top_p', 'frequency_penalty', 'presence_penalty'] + ) + + if(aiModel === 'reverse_proxy' && db.reverseProxyOobaMode){ + const OobaBodyTemplate = db.reverseProxyOobaArgs + + const keys = Object.keys(OobaBodyTemplate) + for(const key of keys){ + if(OobaBodyTemplate[key] !== undefined && OobaBodyTemplate[key] !== null){ + // @ts-ignore + body[key] = OobaBodyTemplate[key] } - let bodyTemplate:Record = { - 'prompt': prompt, - presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), - frequency_penalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), - logit_bias: {}, - max_tokens: maxTokens, - stop: stopStrings, - temperature: temperature, - top_p: db.top_p, + } + + } + + if(supportsInlayImage()){ + // inlay models doesn't support logit_bias + // OpenAI's gpt based llm model supports both logit_bias and inlay image + if(!( + aiModel.startsWith('gpt') || + (aiModel == 'reverse_proxy' && ( + db.proxyRequestModel?.startsWith('gpt') || + (db.proxyRequestModel === 'custom' && db.customProxyRequestModel.startsWith('gpt')) + )))){ + // @ts-ignore + delete body.logit_bias + } + } + + let replacerURL = aiModel === 'openrouter' ? "https://openrouter.ai/api/v1/chat/completions" : + (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.openai.com/v1/chat/completions') + + let risuIdentify = false + if(replacerURL.startsWith("risu::")){ + risuIdentify = true + replacerURL = replacerURL.replace("risu::", '') + } + + if(aiModel === 'reverse_proxy' && db.autofillRequestUrl){ + if(replacerURL.endsWith('v1')){ + replacerURL += '/chat/completions' + } + else if(replacerURL.endsWith('v1/')){ + replacerURL += 'chat/completions' + } + else if(!(replacerURL.endsWith('completions') || replacerURL.endsWith('completions/'))){ + if(replacerURL.endsWith('/')){ + replacerURL += 'v1/chat/completions' } - - const url = new URL(db.textgenWebUIBlockingURL) - url.pathname = "/v1/completions" - const urlStr = url.toString() - - const OobaBodyTemplate = db.reverseProxyOobaArgs - const keys = Object.keys(OobaBodyTemplate) - for(const key of keys){ - if(OobaBodyTemplate[key] !== undefined && OobaBodyTemplate[key] !== null && OobaParams.includes(key)){ - bodyTemplate[key] = OobaBodyTemplate[key] - } - else if(bodyTemplate[key]){ - delete bodyTemplate[key] - } + else{ + replacerURL += '/v1/chat/completions' } + } + } - const response = await globalFetch(urlStr, { - body: bodyTemplate, - chatId: arg.chatId - }) + let headers = { + "Authorization": "Bearer " + (aiModel === 'reverse_proxy' ? db.proxyKey : (aiModel === 'openrouter' ? db.openrouterKey : db.openAIKey)), + "Content-Type": "application/json" + } - if(!response.ok){ + if(aiModel === 'openrouter'){ + headers["X-Title"] = 'RisuAI' + headers["HTTP-Referer"] = 'https://risuai.xyz' + } + if(risuIdentify){ + headers["X-Proxy-Risu"] = 'RisuAI' + } + if(aiModel.startsWith('jamba')){ + headers['Authorization'] = 'Bearer ' + db.ai21Key + replacerURL = 'https://api.ai21.com/studio/v1/chat/completions' + } + if(arg.multiGen){ + // @ts-ignore + body.n = db.genTime + } + let throughProxi = (!isTauri) && (!isNodeServer) && (!db.usePlainFetch) && (!Capacitor.isNativePlatform()) + if(arg.useStreaming){ + body.stream = true + let urlHost = new URL(replacerURL).host + if(urlHost.includes("localhost") || urlHost.includes("172.0.0.1") || urlHost.includes("0.0.0.0")){ + if(!isTauri){ return { type: 'fail', - result: (language.errors.httpError + `${JSON.stringify(response.data)}`) + result: 'You are trying local request on streaming. this is not allowed dude to browser/os security policy. turn off streaming.', } } - const text:string = response.data.choices[0].text + } + const da = await fetchNative(replacerURL, { + body: JSON.stringify(body), + method: "POST", + headers: headers, + signal: arg.abortSignal, + chatId: arg.chatId + }) + + if(da.status !== 200){ return { - type: 'success', - result: text.replace(/##\n/g, '') - } - - } - - case 'custom':{ - const d = await pluginProcess({ - bias: bias, - prompt_chat: formated, - temperature: (db.temperature / 100), - max_tokens: maxTokens, - presence_penalty: (db.PresensePenalty / 100), - frequency_penalty: (db.frequencyPenalty / 100) - }) - if(!d){ - return { - type: 'fail', - result: (language.errors.unknownModel) - } - } - else if(!d.success){ - return { - type: 'fail', - result: d.content - } - } - else{ - return { - type: 'success', - result: d.content - } - } - break - } - case 'palm2': - case 'palm2_unicorn':{ - const bodyData = { - "instances": [ - { - "content": stringlizeChat(formated, currentChar?.name ?? '', arg.continue) - } - ], - "parameters": { - "candidateCount": 1, - "maxOutputTokens": maxTokens, - "stopSequences": [ - "system:", "user:", "assistant:" - ], - "temperature": temperature, - } - }; - - const API_ENDPOINT="us-central1-aiplatform.googleapis.com" - const PROJECT_ID=db.google.projectId - const MODEL_ID= aiModel === 'palm2' ? 'text-bison' : - aiModel ==='palm2_unicorn' ? 'text-unicorn' : - '' - const LOCATION_ID="us-central1" - - const url = `https://${API_ENDPOINT}/v1/projects/${PROJECT_ID}/locations/${LOCATION_ID}/publishers/google/models/${MODEL_ID}:predict`; - const res = await globalFetch(url, { - body: bodyData, - headers: { - "Content-Type": "application/json", - "Authorization": "Bearer " + db.google.accessToken - }, - abortSignal, - chatId: arg.chatId - }) - if(res.ok){ - console.log(res.data) - if(res.data.predictions){ - let output:string = res.data.predictions[0].content - const ind = output.search(/(system note)|(user)|(assistant):/gi) - if(ind >= 0){ - output = output.substring(0, ind) - } - return { - type: 'success', - result: output - } - } - else{ - return { - type: 'fail', - result: `${JSON.stringify(res.data)}` - } - } - } - else{ - return { - type: 'fail', - result: `${JSON.stringify(res.data)}` - } + type: "fail", + result: await textifyReadableStream(da.body) } } - case 'gemini-pro': - case 'gemini-pro-vision': - case 'gemini-1.5-pro-latest': - case 'gemini-1.5-pro-exp-0801': - case 'gemini-1.5-pro-exp-0827': - case 'gemini-exp-1114': - case 'gemini-exp-1121': - case 'gemini-1.5-flash': - case 'gemini-1.5-pro-002': - case 'gemini-1.5-flash-002': - case 'gemini-ultra': - case 'gemini-ultra-vision':{ - interface GeminiPart{ - text?:string - "inlineData"?: { - "mimeType": string, - "data": string - }, - } - - interface GeminiChat { - role: "USER"|"MODEL" - parts:|GeminiPart[] + + if (!da.headers.get('Content-Type').includes('text/event-stream')){ + return { + type: "fail", + result: await textifyReadableStream(da.body) } + } + addFetchLog({ + body: body, + response: "Streaming", + success: true, + url: replacerURL, + }) - let reformatedChat:GeminiChat[] = [] - let pendingImage = '' + let dataUint:Uint8Array|Buffer = new Uint8Array([]) - for(let i=0;i( { + async transform(chunk, control) { + dataUint = Buffer.from(new Uint8Array([...dataUint, ...chunk])) + let JSONreaded:{[key:string]:string} = {} + try { + const datas = dataUint.toString().split('\n') + let readed:{[key:string]:string} = {} + for(const data of datas){ + if(data.startsWith("data: ")){ + try { + const rawChunk = data.replace("data: ", "") + if(rawChunk === "[DONE]"){ + if(db.extractJson && db.jsonSchemaEnabled){ + for(const key in readed){ + const extracted = extractJSON(readed[key], db.extractJson) + JSONreaded[key] = extracted + } + console.log(JSONreaded) + control.enqueue(JSONreaded) + } + else{ + control.enqueue(readed) + } + return + } + const choices = JSON.parse(rawChunk).choices + for(const choice of choices){ + const chunk = choice.delta.content ?? choices.text + if(chunk){ + if(arg.multiGen){ + const ind = choice.index.toString() + if(!readed[ind]){ + readed[ind] = "" + } + readed[ind] += chunk + } + else{ + if(!readed["0"]){ + readed["0"] = "" + } + readed["0"] += chunk + } + } + } + } catch (error) {} + } + } + if(db.extractJson && db.jsonSchemaEnabled){ + for(const key in readed){ + const extracted = extractJSON(readed[key], db.extractJson) + JSONreaded[key] = extracted + } + console.log(JSONreaded) + control.enqueue(JSONreaded) } else{ - reformatedChat.push({ - role: "USER", - parts: [{ - text: chat.role + ':' + chat.content - }] - }) + control.enqueue(readed) } + } catch (error) { + + } + } + },) + + da.body.pipeTo(transtream.writable) + + return { + type: 'streaming', + result: transtream.readable + } + } + + if(arg.realAIModel === 'reverse_proxy'){ + const additionalParams = db.additionalParams + for(let i=0;i { + const extracted = extractJSON(v.message.content, db.extractJson) + return ["char",extracted] + }) + + return { + type: 'multiline', + result: c } - else if(chat.role === 'system'){ - if(prevChat.role === 'USER'){ - reformatedChat[reformatedChat.length-1].parts[0].text += '\nsystem:' + chat.content + + } + return { + type: 'multiline', + result: dat.choices.map((v) => { + return ["char",v.message.content] + }) + } + + } + + if(dat?.choices[0]?.text){ + if(db.extractJson && db.jsonSchemaEnabled){ + try { + const parsed = JSON.parse(dat.choices[0].text) + const extracted = extractJSON(parsed, db.extractJson) + return { + type: 'success', + result: extracted } - else{ - reformatedChat.push({ - role: "USER", - parts: [{ - text: chat.role + ':' + chat.content - }] - }) + } catch (error) { + console.log(error) + return { + type: 'success', + result: dat.choices[0].text } } - else if(chat.role === 'user' && pendingImage !== ''){ - //conver image to jpeg so it can be inlined - const canv = document.createElement('canvas') - const img = new Image() - img.src = pendingImage - await img.decode() - canv.width = img.width - canv.height = img.height - const ctx = canv.getContext('2d') - ctx.drawImage(img, 0, 0) - const base64 = canv.toDataURL('image/jpeg').replace(/^data:image\/jpeg;base64,/, "") - const mimeType = 'image/jpeg' - pendingImage = '' - canv.remove() - img.remove() + } + return { + type: 'success', + result: dat.choices[0].text + } + } + if(db.extractJson && db.jsonSchemaEnabled){ + return { + type: 'success', + result: extractJSON(dat.choices[0].message.content, db.extractJson) + } + } + const msg:OpenAIChatFull = (dat.choices[0].message) + return { + type: 'success', + result: msg.content + } + } catch (error) { + return { + type: 'fail', + result: (language.errors.httpError + `${JSON.stringify(dat)}`) + } + } + } + else{ + if(dat.error && dat.error.message){ + return { + type: 'fail', + result: (language.errors.httpError + `${dat.error.message}`) + } + } + else{ + return { + type: 'fail', + result: (language.errors.httpError + `${JSON.stringify(res.data)}`) + } + } + } +} - reformatedChat.push({ - role: "USER", - parts: [ - { - text: chat.content, - }, - { - inlineData: { - mimeType: mimeType, +async function requestOpenAILegacyInstruct(arg:RequestDataArgumentExtended):Promise{ + const formated = arg.formated + const db = getDatabase() + const maxTokens = arg.maxTokens + const temperature = arg.temperature + const prompt = formated.filter(m => m.content?.trim()).map(m => { + let author = ''; + + if(m.role == 'system'){ + m.content = m.content.trim(); + } + + console.log(m.role +":"+m.content); + switch (m.role) { + case 'user': author = 'User'; break; + case 'assistant': author = 'Assistant'; break; + case 'system': author = 'Instruction'; break; + default: author = m.role; break; + } + + return `\n## ${author}\n${m.content.trim()}`; + //return `\n\n${author}: ${m.content.trim()}`; + }).join("") + `\n## Response\n`; + + const response = await globalFetch( "https://api.openai.com/v1/completions", { + body: { + model: "gpt-3.5-turbo-instruct", + prompt: prompt, + max_tokens: maxTokens, + temperature: temperature, + top_p: 1, + stop:["User:"," User:", "user:", " user:"], + presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), + frequency_penalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), + }, + headers: { + "Content-Type": "application/json", + "Authorization": "Bearer " + db.openAIKey, + }, + chatId: arg.chatId + }); + + if(!response.ok){ + return { + type: 'fail', + result: (language.errors.httpError + `${JSON.stringify(response.data)}`) + } + } + const text:string = response.data.choices[0].text + return { + type: 'success', + result: text.replace(/##\n/g, '') + } + +} + +async function requestNovelAI(arg:RequestDataArgumentExtended):Promise{ + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + const temperature = arg.temperature + const maxTokens = arg.maxTokens + const biasString = arg.biasString + const currentChar = getCurrentCharacter() + const prompt = stringlizeNAIChat(formated, currentChar?.name ?? '', arg.continue) + const abortSignal = arg.abortSignal + let logit_bias_exp:{ + sequence: number[], bias: number, ensure_sequence_finish: false, generate_once: true + }[] = [] + + for(let i=0;i { + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + const maxTokens = arg.maxTokens + const currentChar = getCurrentCharacter() + const useStreaming = arg.useStreaming + const abortSignal = arg.abortSignal + let streamUrl = db.textgenWebUIStreamURL.replace(/\/api.*/, "/api/v1/stream") + let blockingUrl = db.textgenWebUIBlockingURL.replace(/\/api.*/, "/api/v1/generate") + let bodyTemplate:{[key:string]:any} = {} + const prompt = applyChatTemplate(formated) + let stopStrings = getStopStrings(false) + if(db.localStopStrings){ + stopStrings = db.localStopStrings.map((v) => { + return risuChatParser(v.replace(/\\n/g, "\n")) + }) + } + bodyTemplate = { + 'max_new_tokens': db.maxResponse, + 'do_sample': db.ooba.do_sample, + 'temperature': (db.temperature / 100), + 'top_p': db.ooba.top_p, + 'typical_p': db.ooba.typical_p, + 'repetition_penalty': db.ooba.repetition_penalty, + 'encoder_repetition_penalty': db.ooba.encoder_repetition_penalty, + 'top_k': db.ooba.top_k, + 'min_length': db.ooba.min_length, + 'no_repeat_ngram_size': db.ooba.no_repeat_ngram_size, + 'num_beams': db.ooba.num_beams, + 'penalty_alpha': db.ooba.penalty_alpha, + 'length_penalty': db.ooba.length_penalty, + 'early_stopping': false, + 'truncation_length': maxTokens, + 'ban_eos_token': db.ooba.ban_eos_token, + 'stopping_strings': stopStrings, + 'seed': -1, + add_bos_token: db.ooba.add_bos_token, + topP: db.top_p, + prompt: prompt + } + + const headers = (aiModel === 'textgen_webui') ? {} : { + 'X-API-KEY': db.mancerHeader + } + + if(useStreaming){ + const oobaboogaSocket = new WebSocket(streamUrl); + const statusCode = await new Promise((resolve) => { + oobaboogaSocket.onopen = () => resolve(0) + oobaboogaSocket.onerror = () => resolve(1001) + oobaboogaSocket.onclose = ({ code }) => resolve(code) + }) + if(abortSignal.aborted || statusCode !== 0) { + oobaboogaSocket.close() + return ({ + type: "fail", + result: abortSignal.reason || `WebSocket connection failed to '${streamUrl}' failed!`, + }) + } + + const close = () => { + oobaboogaSocket.close() + } + const stream = new ReadableStream({ + start(controller){ + let readed = ""; + oobaboogaSocket.onmessage = async (event) => { + const json = JSON.parse(event.data); + if (json.event === "stream_end") { + close() + controller.close() + return + } + if (json.event !== "text_stream") return + readed += json.text + controller.enqueue(readed) + }; + oobaboogaSocket.send(JSON.stringify(bodyTemplate)); + }, + cancel(){ + close() + } + }) + oobaboogaSocket.onerror = close + oobaboogaSocket.onclose = close + abortSignal.addEventListener("abort", close) + + return { + type: 'streaming', + result: stream + } + } + + const res = await globalFetch(blockingUrl, { + body: bodyTemplate, + headers: headers, + abortSignal, + chatId: arg.chatId + }) + + const dat = res.data as any + if(res.ok){ + try { + let result:string = dat.results[0].text + + return { + type: 'success', + result: unstringlizeChat(result, formated, currentChar?.name ?? '') + } + } catch (error) { + return { + type: 'fail', + result: (language.errors.httpError + `${error}`) + } + } + } + else{ + return { + type: 'fail', + result: (language.errors.httpError + `${JSON.stringify(res.data)}`) + } + } +} + +async function requestOoba(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + const maxTokens = arg.maxTokens + const temperature = arg.temperature + const prompt = applyChatTemplate(formated) + let stopStrings = getStopStrings(false) + if(db.localStopStrings){ + stopStrings = db.localStopStrings.map((v) => { + return risuChatParser(v.replace(/\\n/g, "\n")) + }) + } + let bodyTemplate:Record = { + 'prompt': prompt, + presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), + frequency_penalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), + logit_bias: {}, + max_tokens: maxTokens, + stop: stopStrings, + temperature: temperature, + top_p: db.top_p, + } + + const url = new URL(db.textgenWebUIBlockingURL) + url.pathname = "/v1/completions" + const urlStr = url.toString() + + const OobaBodyTemplate = db.reverseProxyOobaArgs + const keys = Object.keys(OobaBodyTemplate) + for(const key of keys){ + if(OobaBodyTemplate[key] !== undefined && OobaBodyTemplate[key] !== null && OobaParams.includes(key)){ + bodyTemplate[key] = OobaBodyTemplate[key] + } + else if(bodyTemplate[key]){ + delete bodyTemplate[key] + } + } + + const response = await globalFetch(urlStr, { + body: bodyTemplate, + chatId: arg.chatId + }) + + if(!response.ok){ + return { + type: 'fail', + result: (language.errors.httpError + `${JSON.stringify(response.data)}`) + } + } + const text:string = response.data.choices[0].text + return { + type: 'success', + result: text.replace(/##\n/g, '') + } + +} + +async function requestPlugin(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const maxTokens = arg.maxTokens + const bias = arg.biasString + const d = await pluginProcess({ + bias: bias, + prompt_chat: formated, + temperature: (db.temperature / 100), + max_tokens: maxTokens, + presence_penalty: (db.PresensePenalty / 100), + frequency_penalty: (db.frequencyPenalty / 100) + }) + if(!d){ + return { + type: 'fail', + result: (language.errors.unknownModel) + } + } + else if(!d.success){ + return { + type: 'fail', + result: d.content + } + } + else{ + return { + type: 'success', + result: d.content + } + } +} + +async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise { + + const formated = arg.formated + const db = getDatabase() + const maxTokens = arg.maxTokens + + interface GeminiPart{ + text?:string + "inlineData"?: { + "mimeType": string, + "data": string + }, + } + + interface GeminiChat { + role: "USER"|"MODEL" + parts:|GeminiPart[] + } + + + let reformatedChat:GeminiChat[] = [] + let pendingImage = '' + + for(let i=0;i { + if(data?.candidates?.[0]?.content?.parts?.[0]?.text){ + fullRes += data.candidates[0].content.parts[0].text + } + else if(data?.errors){ + return { + type: 'fail', + result: `${JSON.stringify(data.errors)}` + } + } + else{ + return { + type: 'fail', + result: `${JSON.stringify(data)}` + } + } + } + + // traverse responded data if it contains multipart contents + if (typeof (res.data)[Symbol.iterator] === 'function') { + for(const data of res.data){ + processDataItem(data) + } + } else { + processDataItem(res.data) + } + + return { + type: 'success', + result: fullRes + } +} + +async function requestKobold(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const maxTokens = arg.maxTokens + const abortSignal = arg.abortSignal + + const prompt = applyChatTemplate(formated) + const url = new URL(db.koboldURL) + if(url.pathname.length < 3){ + url.pathname = 'api/v1/generate' + } + + const body = applyParameters({ + "prompt": prompt, + max_length: maxTokens, + max_context_length: db.maxContext, + n: 1 + }, [ + 'temperature', + 'top_p', + 'repetition_penalty', + 'top_k', + 'top_a' + ], { + 'repetition_penalty': 'rep_pen' + }) as KoboldGenerationInputSchema + + const da = await globalFetch(url.toString(), { + method: "POST", + body: body, + headers: { + "content-type": "application/json", + }, + abortSignal, + chatId: arg.chatId + }) + + if(!da.ok){ + return { + type: "fail", + result: da.data, + noRetry: true + } + } + + const data = da.data + return { + type: 'success', + result: data.results[0].text + } +} + +async function requestNovelList(arg:RequestDataArgumentExtended):Promise { + + const formated = arg.formated + const db = getDatabase() + const maxTokens = arg.maxTokens + const temperature = arg.temperature + const biasString = arg.biasString + const currentChar = getCurrentCharacter() + const aiModel = arg.aiModel + const auth_key = db.novellistAPI; + const api_server_url = 'https://api.tringpt.com/'; + const logit_bias:string[] = [] + const logit_bias_values:string[] = [] + for(let i=0;i>") + db.ainconfig.stoptokens, + logit_bias: (logit_bias.length > 0) ? logit_bias.join("<<|>>") : undefined, + logit_bias_values: (logit_bias_values.length > 0) ? logit_bias_values.join("|") : undefined, + }; + const response = await globalFetch(api_server_url + '/api', { + method: 'POST', + headers: headers, + body: send_body, + chatId: arg.chatId + }); + + if(!response.ok){ + return { + type: 'fail', + result: response.data + } + } + + if(response.data.error){ + return { + 'type': 'fail', + 'result': `${response.data.error.replace("token", "api key")}` + } + } + + const result = response.data.data[0]; + const unstr = unstringlizeAIN(result, formated, currentChar?.name ?? '') + return { + 'type': 'multiline', + 'result': unstr + } +} + +async function requestOllama(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + + const ollama = new Ollama({host: db.ollamaURL}) + + const response = await ollama.chat({ + model: db.ollamaModel, + messages: formated.map((v) => { + return { + role: v.role, + content: v.content + } + }).filter((v) => { + return v.role === 'assistant' || v.role === 'user' || v.role === 'system' + }), + stream: true + }) + + const readableStream = new ReadableStream({ + async start(controller){ + for await(const chunk of response){ + controller.enqueue({ + "0": chunk.message.content + }) + } + controller.close() + } + }) + + return { + type: 'streaming', + result: readableStream + } +} + +async function requestCohere(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + + let lastChatPrompt = '' + let preamble = '' + + let lastChat = formated[formated.length-1] + if(lastChat.role === 'user'){ + lastChatPrompt = lastChat.content + formated.pop() + } + else{ + while(lastChat.role !== 'user'){ + lastChat = formated.pop() + if(!lastChat){ + return { + type: 'fail', + result: 'Cohere requires a user message to generate a response' + } + } + lastChatPrompt = (lastChat.role === 'user' ? '' : `${lastChat.role}: `) + '\n' + lastChat.content + lastChatPrompt + } + } + + const firstChat = formated[0] + if(firstChat.role === 'system'){ + preamble = firstChat.content + formated.shift() + } + + //reformat chat + + let body = applyParameters({ + message: lastChatPrompt, + chat_history: formated.map((v) => { + if(v.role === 'assistant'){ + return { + role: 'CHATBOT', + message: v.content + } + } + if(v.role === 'system'){ + return { + role: 'SYSTEM', + message: v.content + } + } + if(v.role === 'user'){ + return { + role: 'USER', + message: v.content + } + } + return null + }).filter((v) => v !== null).filter((v) => { + return v.message + }), + }, [ + 'temperature', 'top_k', 'top_p', 'presence_penalty', 'frequency_penalty' + ], { + 'top_k': 'k', + 'top_p': 'p', + }) + + if(aiModel !== 'cohere-command-r-03-2024' && aiModel !== 'cohere-command-r-plus-04-2024'){ + body.safety_mode = "NONE" + } + + if(preamble){ + if(body.chat_history.length > 0){ + // @ts-ignore + body.preamble = preamble + } + else{ + body.message = `system: ${preamble}` + } + } + + console.log(body) + + const res = await globalFetch('https://api.cohere.com/v1/chat', { + method: "POST", + headers: { + "Authorization": "Bearer " + db.cohereAPIKey, + "Content-Type": "application/json" + }, + body: body + }) + + if(!res.ok){ + return { + type: 'fail', + result: JSON.stringify(res.data) + } + } + + const result = res.data.text + if(!result){ + return { + type: 'fail', + result: JSON.stringify(res.data) + } + } + + return { + type: 'success', + result: result + } + +} + +async function requestClaude(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + const useStreaming = arg.useStreaming + let replacerURL = (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.anthropic.com/v1/messages') + let apiKey = (aiModel === 'reverse_proxy') ? db.proxyKey : db.claudeAPIKey + const maxTokens = arg.maxTokens + if(aiModel === 'reverse_proxy' && db.autofillRequestUrl){ + if(replacerURL.endsWith('v1')){ + replacerURL += '/messages' + } + else if(replacerURL.endsWith('v1/')){ + replacerURL += 'messages' + } + else if(!(replacerURL.endsWith('messages') || replacerURL.endsWith('messages/'))){ + if(replacerURL.endsWith('/')){ + replacerURL += 'v1/messages' + } + else{ + replacerURL += '/v1/messages' + } + } + } + + interface Claude3TextBlock { + type: 'text', + text: string, + cache_control?: {"type": "ephemeral"} + } + + interface Claude3ImageBlock { + type: 'image', + source: { + type: 'base64' + media_type: string, + data: string + } + cache_control?: {"type": "ephemeral"} + } + + type Claude3ContentBlock = Claude3TextBlock|Claude3ImageBlock + + interface Claude3Chat { + role: 'user'|'assistant' + content: Claude3ContentBlock[] + } + + interface Claude3ExtendedChat { + role: 'user'|'assistant' + content: Claude3ContentBlock[]|string + } + + let claudeChat: Claude3Chat[] = [] + let systemPrompt:string = '' + + const addClaudeChat = (chat:{ + role: 'user'|'assistant' + content: string + }, multimodals?:MultiModal[]) => { + if(claudeChat.length > 0 && claudeChat[claudeChat.length-1].role === chat.role){ + let content = claudeChat[claudeChat.length-1].content + if(multimodals && multimodals.length > 0 && !Array.isArray(content)){ + content = [{ + type: 'text', + text: content + }] + } + + if(Array.isArray(content)){ + let lastContent = content[content.length-1] + if( lastContent?.type === 'text'){ + lastContent.text += "\n\n" + chat.content + content[content.length-1] = lastContent + } + else{ + content.push({ + type: 'text', + text: chat.content + }) + } + + if(multimodals && multimodals.length > 0){ + for(const modal of multimodals){ + if(modal.type === 'image'){ + const dataurl = modal.base64 + const base64 = dataurl.split(',')[1] + const mediaType = dataurl.split(';')[0].split(':')[1] + + content.unshift({ + type: 'image', + source: { + type: 'base64', + media_type: mediaType, data: base64 } - }] - }) - } - else if(chat.role === 'assistant' || chat.role === 'user'){ - reformatedChat.push({ - role: chat.role === 'user' ? 'USER' : 'MODEL', - parts: [{ - text: chat.content - }] - }) - } - else{ - reformatedChat.push({ - role: "USER", - parts: [{ - text: chat.role + ':' + chat.content - }] - }) - } - } - } - - const uncensoredCatagory = [ - { - "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_HATE_SPEECH", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_HARASSMENT", - "threshold": "BLOCK_NONE" - }, - { - "category": "HARM_CATEGORY_DANGEROUS_CONTENT", - "threshold": "BLOCK_NONE" - }, - ] - - - const body = { - contents: reformatedChat, - generation_config: applyParameters({ - "maxOutputTokens": maxTokens, - }, ['temperature', 'top_p'], { - 'top_p': "topP" - }), - safetySettings: uncensoredCatagory - } - - let headers:{[key:string]:string} = {} - - const PROJECT_ID=db.google.projectId - const REGION="us-central1" - if(PROJECT_ID !== 'aigoogle'){ - headers['Authorization'] = "Bearer " + db.google.accessToken - } - - const url = PROJECT_ID !== 'aigoogle' ? - `https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${aiModel}:streamGenerateContent` - : `https://generativelanguage.googleapis.com/v1beta/models/${aiModel}:generateContent?key=${db.google.accessToken}` - const res = await globalFetch(url, { - headers: headers, - body: body, - chatId: arg.chatId - }) - - if(!res.ok){ - return { - type: 'fail', - result: `${JSON.stringify(res.data)}` - } - } - - let fullRes = '' - - const processDataItem = (data:any) => { - if(data?.candidates?.[0]?.content?.parts?.[0]?.text){ - fullRes += data.candidates[0].content.parts[0].text - } - else if(data?.errors){ - return { - type: 'fail', - result: `${JSON.stringify(data.errors)}` - } - } - else{ - return { - type: 'fail', - result: `${JSON.stringify(data)}` - } - } - } - - // traverse responded data if it contains multipart contents - if (typeof (res.data)[Symbol.iterator] === 'function') { - for(const data of res.data){ - processDataItem(data) - } - } else { - processDataItem(res.data) - } - - return { - type: 'success', - result: fullRes - } - - } - case "kobold":{ - const prompt = applyChatTemplate(formated) - const url = new URL(db.koboldURL) - if(url.pathname.length < 3){ - url.pathname = 'api/v1/generate' - } - - const body = applyParameters({ - "prompt": prompt, - max_length: maxTokens, - max_context_length: db.maxContext, - n: 1 - }, [ - 'temperature', - 'top_p', - 'repetition_penalty', - 'top_k', - 'top_a' - ], { - 'repetition_penalty': 'rep_pen' - }) as KoboldGenerationInputSchema - - const da = await globalFetch(url.toString(), { - method: "POST", - body: body, - headers: { - "content-type": "application/json", - }, - abortSignal, - chatId: arg.chatId - }) - - if(!da.ok){ - return { - type: "fail", - result: da.data, - noRetry: true - } - } - - const data = da.data - return { - type: 'success', - result: data.results[0].text - } - } - case "novellist": - case "novellist_damsel":{ - const auth_key = db.novellistAPI; - const api_server_url = 'https://api.tringpt.com/'; - const logit_bias:string[] = [] - const logit_bias_values:string[] = [] - for(let i=0;i>") + db.ainconfig.stoptokens, - logit_bias: (logit_bias.length > 0) ? logit_bias.join("<<|>>") : undefined, - logit_bias_values: (logit_bias_values.length > 0) ? logit_bias_values.join("|") : undefined, - }; - const response = await globalFetch(api_server_url + '/api', { - method: 'POST', - headers: headers, - body: send_body, - chatId: arg.chatId - }); - - if(!response.ok){ - return { - type: 'fail', - result: response.data - } - } - - if(response.data.error){ - return { - 'type': 'fail', - 'result': `${response.data.error.replace("token", "api key")}` - } - } - - const result = response.data.data[0]; - const unstr = unstringlizeAIN(result, formated, currentChar?.name ?? '') - return { - 'type': 'multiline', - 'result': unstr - } - } - case 'risullm-proto':{ - const res = await globalFetch('https://sv.risuai.xyz/risullm', { - body: { - messages: formated.map((v) => { - if(v.role === 'system'){ - return { - role: "user", - content: "System: " + v.content - } - } - if(v.role === 'function'){ - return { - role: "user", - content: "Function: " + v.content - - } - } - return { - role: v.role, - content: v.content - } - }) - }, - headers: { - "X-Api-Key": db.proxyKey - } - }) - - const resp:string = res?.data?.response - - if(!resp){ - return { - type: 'fail', - result: JSON.stringify(res.data) - } - } - - return { - type: 'success', - result: resp.replace(/\\n/g, '\n') - } - } - case 'ollama-hosted':{ - const ollama = new Ollama({host: db.ollamaURL}) - - const response = await ollama.chat({ - model: db.ollamaModel, - messages: formated.map((v) => { - return { - role: v.role, - content: v.content - } - }).filter((v) => { - return v.role === 'assistant' || v.role === 'user' || v.role === 'system' - }), - stream: true - }) - - const readableStream = new ReadableStream({ - async start(controller){ - for await(const chunk of response){ - controller.enqueue({ - "0": chunk.message.content - }) - } - controller.close() - } - }) - - return { - type: 'streaming', - result: readableStream - } - } - case 'cohere-command-r': - case 'cohere-command-r-plus': - case 'cohere-command-r-08-2024': - case 'cohere-command-r-03-2024': - case 'cohere-command-r-plus-04-2024': - case 'cohere-command-r-plus-08-2024':{ - const modelName = aiModel.replace('cohere-', '') - let lastChatPrompt = '' - let preamble = '' - - let lastChat = formated[formated.length-1] - if(lastChat.role === 'user'){ - lastChatPrompt = lastChat.content - formated.pop() - } - else{ - while(lastChat.role !== 'user'){ - lastChat = formated.pop() - if(!lastChat){ - return { - type: 'fail', - result: 'Cohere requires a user message to generate a response' - } - } - lastChatPrompt = (lastChat.role === 'user' ? '' : `${lastChat.role}: `) + '\n' + lastChat.content + lastChatPrompt - } - } - - const firstChat = formated[0] - if(firstChat.role === 'system'){ - preamble = firstChat.content - formated.shift() - } - - //reformat chat - - let body = applyParameters({ - message: lastChatPrompt, - chat_history: formated.map((v) => { - if(v.role === 'assistant'){ - return { - role: 'CHATBOT', - message: v.content - } - } - if(v.role === 'system'){ - return { - role: 'SYSTEM', - message: v.content - } - } - if(v.role === 'user'){ - return { - role: 'USER', - message: v.content - } - } - return null - }).filter((v) => v !== null).filter((v) => { - return v.message - }), - }, [ - 'temperature', 'top_k', 'top_p', 'presence_penalty', 'frequency_penalty' - ], { - 'top_k': 'k', - 'top_p': 'p', - }) - - if(aiModel !== 'cohere-command-r-03-2024' && aiModel !== 'cohere-command-r-plus-04-2024'){ - body.safety_mode = "NONE" - } - - if(preamble){ - if(body.chat_history.length > 0){ - // @ts-ignore - body.preamble = preamble - } - else{ - body.message = `system: ${preamble}` - } - } - - console.log(body) - - const res = await globalFetch('https://api.cohere.com/v1/chat', { - method: "POST", - headers: { - "Authorization": "Bearer " + db.cohereAPIKey, - "Content-Type": "application/json" - }, - body: body - }) - - if(!res.ok){ - return { - type: 'fail', - result: JSON.stringify(res.data) - } - } - - const result = res.data.text - if(!result){ - return { - type: 'fail', - result: JSON.stringify(res.data) - } - } - - return { - type: 'success', - result: result - } - } - - default:{ - if(raiModel.startsWith('claude-3')){ - let replacerURL = (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.anthropic.com/v1/messages') - let apiKey = (aiModel === 'reverse_proxy') ? db.proxyKey : db.claudeAPIKey - if(aiModel === 'reverse_proxy' && db.autofillRequestUrl){ - if(replacerURL.endsWith('v1')){ - replacerURL += '/messages' - } - else if(replacerURL.endsWith('v1/')){ - replacerURL += 'messages' - } - else if(!(replacerURL.endsWith('messages') || replacerURL.endsWith('messages/'))){ - if(replacerURL.endsWith('/')){ - replacerURL += 'v1/messages' - } - else{ - replacerURL += '/v1/messages' - } - } - } - - interface Claude3TextBlock { - type: 'text', - text: string, - cache_control?: {"type": "ephemeral"} - } - - interface Claude3ImageBlock { - type: 'image', - source: { - type: 'base64' - media_type: string, - data: string - } - cache_control?: {"type": "ephemeral"} - } - - type Claude3ContentBlock = Claude3TextBlock|Claude3ImageBlock - - interface Claude3Chat { - role: 'user'|'assistant' - content: Claude3ContentBlock[] - } - - interface Claude3ExtendedChat { - role: 'user'|'assistant' - content: Claude3ContentBlock[]|string - } - - let claudeChat: Claude3Chat[] = [] - let systemPrompt:string = '' - - const addClaudeChat = (chat:{ - role: 'user'|'assistant' - content: string - }, multimodals?:MultiModal[]) => { - if(claudeChat.length > 0 && claudeChat[claudeChat.length-1].role === chat.role){ - let content = claudeChat[claudeChat.length-1].content - if(multimodals && multimodals.length > 0 && !Array.isArray(content)){ - content = [{ - type: 'text', - text: content - }] - } - - if(Array.isArray(content)){ - let lastContent = content[content.length-1] - if( lastContent?.type === 'text'){ - lastContent.text += "\n\n" + chat.content - content[content.length-1] = lastContent - } - else{ - content.push({ - type: 'text', - text: chat.content - }) - } - - if(multimodals && multimodals.length > 0){ - for(const modal of multimodals){ - if(modal.type === 'image'){ - const dataurl = modal.base64 - const base64 = dataurl.split(',')[1] - const mediaType = dataurl.split(';')[0].split(':')[1] - - content.unshift({ - type: 'image', - source: { - type: 'base64', - media_type: mediaType, - data: base64 - } - }) - } - } - } - } - claudeChat[claudeChat.length-1].content = content - } - else{ - let formatedChat:Claude3Chat = { - role: chat.role, - content: [{ - type: 'text', - text: chat.content - }] - } - if(multimodals && multimodals.length > 0){ - formatedChat.content = [{ - type: 'text', - text: chat.content - }] - for(const modal of multimodals){ - if(modal.type === 'image'){ - const dataurl = modal.base64 - const base64 = dataurl.split(',')[1] - const mediaType = dataurl.split(';')[0].split(':')[1] - - formatedChat.content.unshift({ - type: 'image', - source: { - type: 'base64', - media_type: mediaType, - data: base64 - } - }) - } - } - - } - claudeChat.push(formatedChat) - } - } - for(const chat of formated){ - switch(chat.role){ - case 'user':{ - addClaudeChat({ - role: 'user', - content: chat.content - }, chat.multimodals) - break - } - case 'assistant':{ - addClaudeChat({ - role: 'assistant', - content: chat.content - }, chat.multimodals) - break - } - case 'system':{ - if(claudeChat.length === 0){ - systemPrompt += '\n\n' + chat.content - } - else{ - addClaudeChat({ - role: 'user', - content: "System: " + chat.content - }) - } - break - } - case 'function':{ - //ignore function for now - break - } - } - } - if(claudeChat.length === 0 && systemPrompt === ''){ - return { - type: 'fail', - result: 'No input' - } - } - if(claudeChat.length === 0 && systemPrompt !== ''){ - claudeChat.push({ - role: 'user', - content: [{ - type: 'text', - text: 'Start' - }] - }) - systemPrompt = '' - } - if(claudeChat[0].role !== 'user'){ - claudeChat.unshift({ - role: 'user', - content: [{ - type: 'text', - text: 'Start' - }] - }) - } - if(db.claudeCachingExperimental){ - for(let i = 0;i<4;i++){ - const ind = claudeChat.findLastIndex((v) => { - if(v.role !== 'user'){ - return false - } - if(v.content.length === 0){ - return false - } - if(v.content[0].cache_control){ // if it already has cache control, skip - return false - } - return true - }) - console.log(ind) - if(ind === -1){ - break - } - claudeChat[ind].content[0].cache_control = { - type: 'ephemeral' - } - } - } - - let finalChat:Claude3ExtendedChat[] = claudeChat - - if(aiModel === 'reverse_proxy'){ - finalChat = claudeChat.map((v) => { - if(v.content.length > 0 && v.content[0].type === 'text'){ - return { - role: v.role, - content: v.content[0].text - } - } - }) - } - - - let body = applyParameters({ - model: raiModel, - messages: finalChat, - system: systemPrompt.trim(), - max_tokens: maxTokens, - stream: useStreaming ?? false - }, ['temperature', 'top_k', 'top_p']) - - if(systemPrompt === ''){ - delete body.system - } - - const bedrock = db.claudeAws - - if(bedrock && aiModel !== 'reverse_proxy'){ - function getCredentialParts(key:string) { - const [accessKeyId, secretAccessKey, region] = key.split(":"); - - if (!accessKeyId || !secretAccessKey || !region) { - throw new Error("The key assigned to this request is invalid."); - } - - return { accessKeyId, secretAccessKey, region }; - } - const { accessKeyId, secretAccessKey, region } = getCredentialParts(apiKey); - - const AMZ_HOST = "bedrock-runtime.%REGION%.amazonaws.com"; - const host = AMZ_HOST.replace("%REGION%", region); - const stream = false; // todo? - - // https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html - const modelIDs = [ - "anthropic.claude-v2", - "anthropic.claude-v2:1", - "anthropic.claude-3-haiku-20240307-v1:0", - "anthropic.claude-3-sonnet-20240229-v1:0", - "anthropic.claude-3-opus-20240229-v1:0", - "anthropic.claude-3-5-sonnet-20240620-v1:0", - "anthropic.claude-3-5-sonnet-20241022-v2:0" - ]; - - const awsModel = "us." + ( - raiModel.includes("3-5-sonnet-20241022") ? modelIDs[6] : - raiModel.includes("3-5-sonnet-20240620") ? modelIDs[5] : - raiModel.includes("3-opus") ? modelIDs[4] : - raiModel.includes("3-sonnet") ? modelIDs[3] : - modelIDs[2]); - const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}` - - const params = { - messages : claudeChat, - system: systemPrompt.trim(), - max_tokens: maxTokens, - // stop_sequences: null, - temperature: temperature, - top_p: db.top_p, - top_k: db.top_k, - anthropic_version: "bedrock-2023-05-31", - } - - const rq = new HttpRequest({ - method: "POST", - protocol: "https:", - hostname: host, - path: `/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`, - headers: { - ["Host"]: host, - ["Content-Type"]: "application/json", - ["accept"]: "application/json", - }, - body: JSON.stringify(params), - }); - - const signer = new SignatureV4({ - sha256: Sha256, - credentials: { accessKeyId, secretAccessKey }, - region, - service: "bedrock", - }); - - const signed = await signer.sign(rq); - - const res = await globalFetch(url, { - method: "POST", - body: params, - headers: signed.headers, - plainFetchForce: true, - chatId: arg.chatId - }) - - if(!res.ok){ - return { - type: 'fail', - result: JSON.stringify(res.data) - } - } - if(res.data.error){ - return { - type: 'fail', - result: JSON.stringify(res.data.error) - } - } - return { - type: 'success', - result: res.data.content[0].text - - } - } - - - let headers:{ - [key:string]:string - } = { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - "accept": "application/json", - } - - if(db.claudeCachingExperimental){ - headers['anthropic-beta'] = 'prompt-caching-2024-07-31' - } - - if(db.usePlainFetch){ - headers['anthropic-dangerous-direct-browser-access'] = 'true' - } - - if(useStreaming){ - - const res = await fetchNative(replacerURL, { - body: JSON.stringify(body), - headers: headers, - method: "POST", - chatId: arg.chatId - }) - - if(res.status !== 200){ - return { - type: 'fail', - result: await textifyReadableStream(res.body) - } - } - let rerequesting = false - let breakError = '' - - - const stream = new ReadableStream({ - async start(controller){ - let text = '' - let reader = res.body.getReader() - const decoder = new TextDecoder() - const parser = createParser(async (e) => { - try { - if(e.type === 'event'){ - switch(e.event){ - case 'content_block_delta': { - if(e.data){ - text += JSON.parse(e.data).delta?.text - controller.enqueue({ - "0": text - }) - } - break - } - case 'error': { - if(e.data){ - const errormsg:string = JSON.parse(e.data).error?.message - if(errormsg && errormsg.toLocaleLowerCase().includes('overload') && db.antiClaudeOverload){ - console.log('Overload detected, retrying...') - reader.cancel() - rerequesting = true - await sleep(2000) - body.max_tokens -= await tokenize(text) - if(body.max_tokens < 0){ - body.max_tokens = 0 - } - if(body.messages.at(-1)?.role !== 'assistant'){ - body.messages.push({ - role: 'assistant', - content: [{ - type: 'text', - text: '' - }] - }) - } - let block = body.messages[body.messages.length-1].content - if(typeof block === 'string'){ - body.messages[body.messages.length-1].content += text - } - else if(block[0].type === 'text'){ - block[0].text += text - } - const res = await fetchNative(replacerURL, { - body: JSON.stringify(body), - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - "accept": "application/json", - }, - method: "POST", - chatId: arg.chatId - }) - if(res.status !== 200){ - breakError = 'Error: ' + await textifyReadableStream(res.body) - break - } - reader = res.body.getReader() - rerequesting = false - break - } - text += "Error:" + JSON.parse(e.data).error?.message - if(db.extractJson && db.jsonSchemaEnabled){ - controller.enqueue({ - "0": extractJSON(text, db.jsonSchema) - }) - } - else{ - controller.enqueue({ - "0": text - }) - } - } - break - } - } - } - } catch (error) {} }) - while(true){ - if(rerequesting){ - if(breakError){ - controller.enqueue({ - "0": breakError - }) - break - } - await sleep(1000) - continue - } - try { - const {done, value} = await reader.read() - if(done){ - if(rerequesting){ - continue - } - break - } - parser.feed(decoder.decode(value)) - } catch (error) { - await sleep(1) - } - } - controller.close() - }, - cancel(){ } - }) - - return { - type: 'streaming', - result: stream } - - } - const res = await globalFetch(replacerURL, { - body: body, - headers: headers, - method: "POST", - chatId: arg.chatId - }) - - if(!res.ok){ - return { - type: 'fail', - result: JSON.stringify(res.data) - } - } - if(res.data.error){ - return { - type: 'fail', - result: JSON.stringify(res.data.error) - } - } - const resText = res?.data?.content?.[0]?.text - if(!resText){ - return { - type: 'fail', - result: JSON.stringify(res.data) - } - } - if(db.extractJson && db.jsonSchemaEnabled){ - return { - type: 'success', - result: extractJSON(resText, db.jsonSchema) - } - } - return { - type: 'success', - result: resText } } - else if(raiModel.startsWith('claude')){ + claudeChat[claudeChat.length-1].content = content + } + else{ + let formatedChat:Claude3Chat = { + role: chat.role, + content: [{ + type: 'text', + text: chat.content + }] + } + if(multimodals && multimodals.length > 0){ + formatedChat.content = [{ + type: 'text', + text: chat.content + }] + for(const modal of multimodals){ + if(modal.type === 'image'){ + const dataurl = modal.base64 + const base64 = dataurl.split(',')[1] + const mediaType = dataurl.split(';')[0].split(':')[1] - let replacerURL = (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.anthropic.com/v1/complete') - let apiKey = (aiModel === 'reverse_proxy') ? db.proxyKey : db.claudeAPIKey - if(aiModel === 'reverse_proxy'){ - if(replacerURL.endsWith('v1')){ - replacerURL += '/complete' - } - else if(replacerURL.endsWith('v1/')){ - replacerURL += 'complete' - } - else if(!(replacerURL.endsWith('complete') || replacerURL.endsWith('complete/'))){ - if(replacerURL.endsWith('/')){ - replacerURL += 'v1/complete' - } - else{ - replacerURL += '/v1/complete' - } + formatedChat.content.unshift({ + type: 'image', + source: { + type: 'base64', + media_type: mediaType, + data: base64 + } + }) } } - for(let i=0;i { - let prefix = '' - switch (v.role){ - case "assistant": - prefix = "\n\nAssistant: " - break - case "user": - prefix = "\n\nHuman: " - break - case "system": - prefix = "\n\nSystem: " - break - } - latestRole = v.role - if(raiModel.startsWith('claude-2') && (!raiModel.startsWith('claude-2.0'))){ - if(v.role === 'system' && i === 0){ - prefix = '' - } - } - return prefix + v.content - }).join('') - - if(latestRole !== 'assistant'){ - requestPrompt += '\n\nAssistant: ' - } - - - const bedrock = db.claudeAws - - if(bedrock && aiModel !== 'reverse_proxy'){ - function getCredentialParts(key:string) { - const [accessKeyId, secretAccessKey, region] = key.split(":"); - - if (!accessKeyId || !secretAccessKey || !region) { - throw new Error("The key assigned to this request is invalid."); - } - - return { accessKeyId, secretAccessKey, region }; - } - const { accessKeyId, secretAccessKey, region } = getCredentialParts(apiKey); - - const AMZ_HOST = "bedrock-runtime.%REGION%.amazonaws.com"; - const host = AMZ_HOST.replace("%REGION%", region); - - const stream = false - - const LATEST_AWS_V2_MINOR_VERSION = 1; - let awsModel = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`; - - const pattern = /^(claude-)?(instant-)?(v)?(\d+)(\.(\d+))?(-\d+k)?$/i; - const match = raiModel.match(pattern); - - if (match) { - const [, , instant, v, major, dot, minor] = match; - - if (instant) { - awsModel = "anthropic.claude-instant-v1"; - } - - // There's only one v1 model - else if (major === "1") { - awsModel = "anthropic.claude-v1"; - } - - // Try to map Anthropic API v2 models to AWS v2 models - else if (major === "2") { - if (minor === "0") { - awsModel = "anthropic.claude-v2"; - } else if (!v && !dot && !minor) { - awsModel = "anthropic.claude-v2"; - } - } - } - - const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}` - const params = { - prompt : requestPrompt.startsWith("\n\nHuman: ") ? requestPrompt : "\n\nHuman: " + requestPrompt, - max_tokens_to_sample: maxTokens, - stop_sequences: ["\n\nHuman:", "\n\nSystem:", "\n\nAssistant:"], - temperature: temperature, - top_p: db.top_p, - //top_k: db.top_k, - } - const rq = new HttpRequest({ - method: "POST", - protocol: "https:", - hostname: host, - path: `/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`, - headers: { - ["Host"]: host, - ["Content-Type"]: "application/json", - ["accept"]: "application/json", - //"anthropic-version": "2023-06-01", - }, - body: JSON.stringify(params), - }); - - - const signer = new SignatureV4({ - sha256: Sha256, - credentials: { accessKeyId, secretAccessKey }, - region, - service: "bedrock", - }); - - const signed = await signer.sign(rq); - - const da = await globalFetch(url, { - method: "POST", - body: params, - headers: signed.headers, - plainFetchForce: true, - chatId: arg.chatId + else{ + addClaudeChat({ + role: 'user', + content: "System: " + chat.content }) + } + break + } + case 'function':{ + //ignore function for now + break + } + } + } + if(claudeChat.length === 0 && systemPrompt === ''){ + return { + type: 'fail', + result: 'No input' + } + } + if(claudeChat.length === 0 && systemPrompt !== ''){ + claudeChat.push({ + role: 'user', + content: [{ + type: 'text', + text: 'Start' + }] + }) + systemPrompt = '' + } + if(claudeChat[0].role !== 'user'){ + claudeChat.unshift({ + role: 'user', + content: [{ + type: 'text', + text: 'Start' + }] + }) + } + if(db.claudeCachingExperimental){ + for(let i = 0;i<4;i++){ + const ind = claudeChat.findLastIndex((v) => { + if(v.role !== 'user'){ + return false + } + if(v.content.length === 0){ + return false + } + if(v.content[0].cache_control){ // if it already has cache control, skip + return false + } + return true + }) + console.log(ind) + if(ind === -1){ + break + } + claudeChat[ind].content[0].cache_control = { + type: 'ephemeral' + } + } + } - - if((!da.ok) || (da.data.error)){ - return { - type: 'fail', - result: `${JSON.stringify(da.data)}` + let finalChat:Claude3ExtendedChat[] = claudeChat + + if(aiModel === 'reverse_proxy'){ + finalChat = claudeChat.map((v) => { + if(v.content.length > 0 && v.content[0].type === 'text'){ + return { + role: v.role, + content: v.content[0].text + } + } + }) + } + + + let body = applyParameters({ + model: arg.modelInfo.internalID, + messages: finalChat, + system: systemPrompt.trim(), + max_tokens: maxTokens, + stream: useStreaming ?? false + }, ['temperature', 'top_k', 'top_p']) + + if(systemPrompt === ''){ + delete body.system + } + + const bedrock = arg.modelInfo.format === LLMFormat.AWSBedrockClaude + + if(bedrock && aiModel !== 'reverse_proxy'){ + function getCredentialParts(key:string) { + const [accessKeyId, secretAccessKey, region] = key.split(":"); + + if (!accessKeyId || !secretAccessKey || !region) { + throw new Error("The key assigned to this request is invalid."); + } + + return { accessKeyId, secretAccessKey, region }; + } + const { accessKeyId, secretAccessKey, region } = getCredentialParts(apiKey); + + const AMZ_HOST = "bedrock-runtime.%REGION%.amazonaws.com"; + const host = AMZ_HOST.replace("%REGION%", region); + const stream = false; // todo? + + const awsModel = "us." + arg.modelInfo.internalID; + const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}` + + const params = { + messages : claudeChat, + system: systemPrompt.trim(), + max_tokens: maxTokens, + // stop_sequences: null, + temperature: arg.temperature, + top_p: db.top_p, + top_k: db.top_k, + anthropic_version: "bedrock-2023-05-31", + } + + const rq = new HttpRequest({ + method: "POST", + protocol: "https:", + hostname: host, + path: `/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`, + headers: { + ["Host"]: host, + ["Content-Type"]: "application/json", + ["accept"]: "application/json", + }, + body: JSON.stringify(params), + }); + + const signer = new SignatureV4({ + sha256: Sha256, + credentials: { accessKeyId, secretAccessKey }, + region, + service: "bedrock", + }); + + const signed = await signer.sign(rq); + + const res = await globalFetch(url, { + method: "POST", + body: params, + headers: signed.headers, + plainFetchForce: true, + chatId: arg.chatId + }) + + if(!res.ok){ + return { + type: 'fail', + result: JSON.stringify(res.data) + } + } + if(res.data.error){ + return { + type: 'fail', + result: JSON.stringify(res.data.error) + } + } + return { + type: 'success', + result: res.data.content[0].text + + } + } + + + let headers:{ + [key:string]:string + } = { + "Content-Type": "application/json", + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "accept": "application/json", + } + + if(db.claudeCachingExperimental){ + headers['anthropic-beta'] = 'prompt-caching-2024-07-31' + } + + if(db.usePlainFetch){ + headers['anthropic-dangerous-direct-browser-access'] = 'true' + } + + if(useStreaming){ + + const res = await fetchNative(replacerURL, { + body: JSON.stringify(body), + headers: headers, + method: "POST", + chatId: arg.chatId + }) + + if(res.status !== 200){ + return { + type: 'fail', + result: await textifyReadableStream(res.body) + } + } + let rerequesting = false + let breakError = '' + + + const stream = new ReadableStream({ + async start(controller){ + let text = '' + let reader = res.body.getReader() + const decoder = new TextDecoder() + const parser = createParser(async (e) => { + try { + if(e.type === 'event'){ + switch(e.event){ + case 'content_block_delta': { + if(e.data){ + text += JSON.parse(e.data).delta?.text + controller.enqueue({ + "0": text + }) + } + break + } + case 'error': { + if(e.data){ + const errormsg:string = JSON.parse(e.data).error?.message + if(errormsg && errormsg.toLocaleLowerCase().includes('overload') && db.antiClaudeOverload){ + console.log('Overload detected, retrying...') + reader.cancel() + rerequesting = true + await sleep(2000) + body.max_tokens -= await tokenize(text) + if(body.max_tokens < 0){ + body.max_tokens = 0 + } + if(body.messages.at(-1)?.role !== 'assistant'){ + body.messages.push({ + role: 'assistant', + content: [{ + type: 'text', + text: '' + }] + }) + } + let block = body.messages[body.messages.length-1].content + if(typeof block === 'string'){ + body.messages[body.messages.length-1].content += text + } + else if(block[0].type === 'text'){ + block[0].text += text + } + const res = await fetchNative(replacerURL, { + body: JSON.stringify(body), + headers: { + "Content-Type": "application/json", + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "accept": "application/json", + }, + method: "POST", + chatId: arg.chatId + }) + if(res.status !== 200){ + breakError = 'Error: ' + await textifyReadableStream(res.body) + break + } + reader = res.body.getReader() + rerequesting = false + break + } + text += "Error:" + JSON.parse(e.data).error?.message + if(db.extractJson && db.jsonSchemaEnabled){ + controller.enqueue({ + "0": extractJSON(text, db.jsonSchema) + }) + } + else{ + controller.enqueue({ + "0": text + }) + } + } + break + } + } } - } - - const res = da.data - - return { - type: "success", - result: res.completion, - } - } - - const da = await globalFetch(replacerURL, { - method: "POST", - body: { - prompt : "\n\nHuman: " + requestPrompt, - model: raiModel, - max_tokens_to_sample: maxTokens, - stop_sequences: ["\n\nHuman:", "\n\nSystem:", "\n\nAssistant:"], - temperature: temperature, - }, - headers: { - "Content-Type": "application/json", - "x-api-key": apiKey, - "anthropic-version": "2023-06-01", - "accept": "application/json" - }, - useRisuToken: aiModel === 'reverse_proxy', - chatId: arg.chatId + } catch (error) {} }) - - if((!da.ok) || (da.data.error)){ - return { - type: 'fail', - result: `${JSON.stringify(da.data)}` + while(true){ + if(rerequesting){ + if(breakError){ + controller.enqueue({ + "0": breakError + }) + break + } + await sleep(1000) + continue + } + try { + const {done, value} = await reader.read() + if(done){ + if(rerequesting){ + continue + } + break + } + parser.feed(decoder.decode(value)) + } catch (error) { + await sleep(1) } } + controller.close() + }, + cancel(){ + } + }) - const res = da.data + return { + type: 'streaming', + result: stream + } + } + const res = await globalFetch(replacerURL, { + body: body, + headers: headers, + method: "POST", + chatId: arg.chatId + }) + + if(!res.ok){ + return { + type: 'fail', + result: JSON.stringify(res.data) + } + } + if(res.data.error){ + return { + type: 'fail', + result: JSON.stringify(res.data.error) + } + } + const resText = res?.data?.content?.[0]?.text + if(!resText){ + return { + type: 'fail', + result: JSON.stringify(res.data) + } + } + if(db.extractJson && db.jsonSchemaEnabled){ + return { + type: 'success', + result: extractJSON(resText, db.jsonSchema) + } + } + return { + type: 'success', + result: resText + } +} + +async function requestHorde(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + const currentChar = getCurrentCharacter() + const abortSignal = arg.abortSignal + + const prompt = applyChatTemplate(formated) + + const realModel = aiModel.split(":::")[1] + + const argument = { + "prompt": prompt, + "params": { + "n": 1, + "max_context_length": db.maxContext + 100, + "max_length": db.maxResponse, + "singleline": false, + "temperature": db.temperature / 100, + "top_k": db.top_k, + "top_p": db.top_p, + }, + "trusted_workers": false, + "workerslow_workers": true, + "_blacklist": false, + "dry_run": false, + "models": [realModel, realModel.trim(), ' ' + realModel, realModel + ' '] + } + + if(realModel === 'auto'){ + delete argument.models + } + + let apiKey = '0000000000' + if(db.hordeConfig.apiKey.length > 2){ + apiKey = db.hordeConfig.apiKey + } + + const da = await fetch("https://stablehorde.net/api/v2/generate/text/async", { + body: JSON.stringify(argument), + method: "POST", + headers: { + "content-type": "application/json", + "apikey": apiKey + }, + signal: abortSignal + }) + + if(da.status !== 202){ + return { + type: "fail", + result: await da.text() + } + } + + const json:{ + id:string, + kudos:number, + message:string + } = await da.json() + + let warnMessage = "" + if(json.message){ + warnMessage = "with " + json.message + } + + while(true){ + await sleep(2000) + const data = await (await fetch("https://stablehorde.net/api/v2/generate/text/status/" + json.id)).json() + if(!data.is_possible){ + fetch("https://stablehorde.net/api/v2/generate/text/status/" + json.id, { + method: "DELETE" + }) + return { + type: 'fail', + result: "Response not possible" + warnMessage, + noRetry: true + } + } + if(data.done && Array.isArray(data.generations) && data.generations.length > 0){ + const generations:{text:string}[] = data.generations + if(generations && generations.length > 0){ return { type: "success", - result: res.completion, - } - - } - if(aiModel.startsWith("horde:::")){ - const prompt = applyChatTemplate(formated) - - const realModel = aiModel.split(":::")[1] - - const argument = { - "prompt": prompt, - "params": { - "n": 1, - "max_context_length": db.maxContext + 100, - "max_length": db.maxResponse, - "singleline": false, - "temperature": db.temperature / 100, - "top_k": db.top_k, - "top_p": db.top_p, - }, - "trusted_workers": false, - "workerslow_workers": true, - "_blacklist": false, - "dry_run": false, - "models": [realModel, realModel.trim(), ' ' + realModel, realModel + ' '] - } - - if(realModel === 'auto'){ - delete argument.models - } - - let apiKey = '0000000000' - if(db.hordeConfig.apiKey.length > 2){ - apiKey = db.hordeConfig.apiKey - } - - const da = await fetch("https://stablehorde.net/api/v2/generate/text/async", { - body: JSON.stringify(argument), - method: "POST", - headers: { - "content-type": "application/json", - "apikey": apiKey - }, - signal: abortSignal - }) - - if(da.status !== 202){ - return { - type: "fail", - result: await da.text() - } - } - - const json:{ - id:string, - kudos:number, - message:string - } = await da.json() - - let warnMessage = "" - if(json.message){ - warnMessage = "with " + json.message - } - - while(true){ - await sleep(2000) - const data = await (await fetch("https://stablehorde.net/api/v2/generate/text/status/" + json.id)).json() - if(!data.is_possible){ - fetch("https://stablehorde.net/api/v2/generate/text/status/" + json.id, { - method: "DELETE" - }) - return { - type: 'fail', - result: "Response not possible" + warnMessage, - noRetry: true - } - } - if(data.done && Array.isArray(data.generations) && data.generations.length > 0){ - const generations:{text:string}[] = data.generations - if(generations && generations.length > 0){ - return { - type: "success", - result: unstringlizeChat(generations[0].text, formated, currentChar?.name ?? '') - } - } - return { - type: 'fail', - result: "No Generations when done", - noRetry: true - } - } - } - - - } - if(aiModel.startsWith('hf:::')){ - const realModel = aiModel.split(":::")[1] - const suggesting = model === "submodel" - const prompt = applyChatTemplate(formated) - const v = await runTransformers(prompt, realModel, { - temperature: temperature, - max_new_tokens: maxTokens, - top_k: db.ooba.top_k, - top_p: db.ooba.top_p, - repetition_penalty: db.ooba.repetition_penalty, - typical_p: db.ooba.typical_p, - }) - return { - type: 'success', - result: unstringlizeChat(v.generated_text as string, formated, currentChar?.name ?? '') - } - } - if(aiModel.startsWith('local_')){ - console.log('running local model') - const suggesting = model === "submodel" - const prompt = applyChatTemplate(formated) - const stopStrings = getStopStrings(suggesting) - console.log(stopStrings) - const modelPath = aiModel.replace('local_', '') - const res = await runGGUFModel({ - prompt: prompt, - modelPath: modelPath, - temperature: temperature, - top_p: db.top_p, - top_k: db.top_k, - maxTokens: maxTokens, - presencePenalty: arg.PresensePenalty || (db.PresensePenalty / 100), - frequencyPenalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), - repeatPenalty: 0, - maxContext: db.maxContext, - stop: stopStrings, - }) - let decoded = '' - const transtream = new TransformStream({ - async transform(chunk, control) { - const decodedChunk = new TextDecoder().decode(chunk) - decoded += decodedChunk - control.enqueue({ - "0": decoded - }) - } - }) - res.pipeTo(transtream.writable) - - return { - type: 'streaming', - result: transtream.readable + result: unstringlizeChat(generations[0].text, formated, currentChar?.name ?? '') } } return { type: 'fail', - result: (language.errors.unknownModel) + result: "No Generations when done", + noRetry: true } } } } - -let userString = '' -let requestedTimes = 999 -let refreshTime = 0 -function getOpenUserString(){ - if(refreshTime < Date.now() && requestedTimes > 2 ){ - refreshTime = Date.now() + (300000 * Math.random()) + 60000 - userString = v4() - requestedTimes = 0 +async function requestWebLLM(arg:RequestDataArgumentExtended):Promise { + const formated = arg.formated + const db = getDatabase() + const aiModel = arg.aiModel + const currentChar = getCurrentCharacter() + const maxTokens = arg.maxTokens + const temperature = arg.temperature + const realModel = aiModel.split(":::")[1] + const prompt = applyChatTemplate(formated) + const v = await runTransformers(prompt, realModel, { + temperature: temperature, + max_new_tokens: maxTokens, + top_k: db.ooba.top_k, + top_p: db.ooba.top_p, + repetition_penalty: db.ooba.repetition_penalty, + typical_p: db.ooba.typical_p, + }) + return { + type: 'success', + result: unstringlizeChat(v.generated_text as string, formated, currentChar?.name ?? '') } - requestedTimes += 1 - console.log(userString) - return userString } - - export interface KoboldSamplerSettingsSchema { rep_pen?: number; rep_pen_range?: number; diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts index 782ecfab..33c152da 100644 --- a/src/ts/storage/database.svelte.ts +++ b/src/ts/storage/database.svelte.ts @@ -721,6 +721,8 @@ export interface Database{ google: { accessToken: string projectId: string + privateKey: string + clientEmail: string } mistralKey?:string chainOfThought?:boolean