diff --git a/src/lang/en.ts b/src/lang/en.ts index 832d0e6a..1beef228 100644 --- a/src/lang/en.ts +++ b/src/lang/en.ts @@ -97,6 +97,8 @@ export const languageEnglish = { additionalText: "The text that would be added to Character Description only when ai thinks its needed, so you can put long texts here. seperate with double newlines.", charjs: "A javascript code that would run with character. for example, you can check `https://github.com/kwaroran/RisuAI/blob/main/src/etc/example-char.js`", romanizer: "Romanizer is a plugin that converts non-roman characters to roman characters to reduce tokens when using non-roman characters while requesting data. this can result diffrent output from the original model. it is not recommended to use this plugin when using roman characters on chat.", + oaiRandomUser: "If enabled, random uuid would be put on user parameter on request, and would be changed on refresh. this can be used to prevent AI from identifying user.", + inlayImages: "If enabled, images could be inlayed to the chat and AIs can see it if they support it.", }, setup: { chooseProvider: "Choose AI Provider", @@ -442,5 +444,6 @@ export const languageEnglish = { seed: "Seed", charjs: "CharacterJS", depthPrompt: "Depth Prompt", - largePortrait: "Portrait" + largePortrait: "Portrait", + postImage: "Post Image", } \ No newline at end of file diff --git a/src/lib/ChatScreens/DefaultChatScreen.svelte b/src/lib/ChatScreens/DefaultChatScreen.svelte index bb8a086e..3a2a058c 100644 --- a/src/lib/ChatScreens/DefaultChatScreen.svelte +++ b/src/lib/ChatScreens/DefaultChatScreen.svelte @@ -1,6 +1,6 @@

{language.advancedSettings}

@@ -35,6 +36,14 @@ Tauri +GPT Vision Quality +{#if $DataBase.inlayImage} + + Low + High + +{/if} +
@@ -56,6 +65,19 @@
+
+ +
+
+ + + +
+
+ + + +
{#if openAdv} + + diff --git a/src/ts/image.ts b/src/ts/image.ts new file mode 100644 index 00000000..52483bda --- /dev/null +++ b/src/ts/image.ts @@ -0,0 +1,89 @@ +import localforage from "localforage"; +import { selectSingleFile } from "./util"; +import { v4 } from "uuid"; +import { DataBase } from "./storage/database"; +import { get } from "svelte/store"; + +const inlayStorage = localforage.createInstance({ + name: 'inlay', + storeName: 'inlay' +}) + +export async function postInlayImage(){ + const img = await selectSingleFile([ + //image format + 'jpg', + 'jpeg', + 'png', + 'webp' + ]) + + if(!img){ + return null + } + + const extention = img.name.split('.').at(-1) + + //darw in canvas to convert to png + const canvas = document.createElement('canvas') + const ctx = canvas.getContext('2d') + const imgObj = new Image() + let drawHeight, drawWidth = 0 + imgObj.src = URL.createObjectURL(new Blob([img.data], {type: `image/${extention}`})) + await new Promise((resolve) => { + imgObj.onload = () => { + drawHeight = imgObj.height + drawWidth = imgObj.width + + //resize image to fit inlay, if it's too big (max 1024px) + if(drawHeight > 1024){ + drawWidth = drawWidth * (1024 / drawHeight) + drawHeight = 1024 + } + if(drawWidth > 1024){ + drawHeight = drawHeight * (1024 / drawWidth) + drawWidth = 1024 + } + drawHeight = Math.floor(drawHeight) + drawWidth = Math.floor(drawWidth) + + canvas.width = drawWidth + canvas.height = drawHeight + ctx.drawImage(imgObj, 0, 0, drawWidth, drawHeight) + resolve(null) + } + }) + const dataURI = canvas.toDataURL('image/png') + + + const imgid = v4() + + await inlayStorage.setItem(imgid, { + name: img.name, + data: dataURI, + ext: extention, + height: drawHeight, + width: drawWidth + }) + + return `{{inlay::${imgid}}}` +} + +export async function getInlayImage(id: string){ + const img:{ + name: string, + data: string + ext: string + height: number + width: number + } = await inlayStorage.getItem(id) + if(img === null){ + return null + } + return img +} + +export function supportsInlayImage(){ + const db = get(DataBase) + return db.aiModel.startsWith('gptv') +} \ No newline at end of file diff --git a/src/ts/parser.ts b/src/ts/parser.ts index 747b5630..9d05fc03 100644 --- a/src/ts/parser.ts +++ b/src/ts/parser.ts @@ -10,6 +10,7 @@ import css from '@adobe/css-tools' import { selectedCharID } from './stores'; import { calcString } from './process/infunctions'; import { findCharacterbyId } from './util'; +import { getInlayImage } from './image'; const convertora = new showdown.Converter({ simpleLineBreaks: true, @@ -93,11 +94,25 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c if(mode === 'back'){ return `
` } + break } return '' }) } + if(db.inlayImage){ + const inlayMatch = data.match(/{{inlay::(.+?)}}/g) + if(inlayMatch){ + for(const inlay of inlayMatch){ + const id = inlay.substring(9, inlay.length - 2) + const img = await getInlayImage(id) + if(img){ + data = data.replace(inlay, ``) + } + } + } + } + return data } diff --git a/src/ts/process/index.ts b/src/ts/process/index.ts index 06ca78dc..5cb9a547 100644 --- a/src/ts/process/index.ts +++ b/src/ts/process/index.ts @@ -19,6 +19,7 @@ import { runTrigger, type additonalSysPrompt } from "./triggers"; import { HypaProcesser } from "./memory/hypamemory"; import { additionalInformations } from "./embedding/addinfo"; import { cipherChat, decipherChat } from "./cipherChat"; +import { getInlayImage, supportsInlayImage } from "../image"; export interface OpenAIChat{ role: 'system'|'user'|'assistant'|'function' @@ -33,7 +34,6 @@ export interface OpenAIChatFull extends OpenAIChat{ name: string arguments:string } - } export const doingChat = writable(false) @@ -464,6 +464,35 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n if(!msg.chatId){ msg.chatId = v4() } + let inlays:string[] = [] + if(db.inlayImage){ + const inlayMatch = formedChat.match(/{{inlay::(.+?)}}/g) + if(inlayMatch){ + for(const inlay of inlayMatch){ + inlays.push(inlay) + } + } + } + + if(inlays.length > 0){ + for(const inlay of inlays){ + const inlayName = inlay.replace('{{inlay::', '').replace('}}', '') + const inlayData = await getInlayImage(inlayName) + if(inlayData){ + if(supportsInlayImage()){ + const imgchat = { + role: msg.role === 'user' ? 'user' : 'assistant', + content: inlayData.data, + memo: `inlayImage-${inlayData.height}-${inlayData.width}`, + } as const + chats.push(imgchat) + currentTokens += await tokenizer.tokenizeChat(imgchat) + } + } + formedChat = formedChat.replace(inlay, '') + } + } + const chat:OpenAIChat = { role: msg.role === 'user' ? 'user' : 'assistant', content: formedChat, @@ -786,7 +815,6 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n } } - const req = await requestChatData({ formated: formated, biasString: biases, diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 2af3bd5d..e9ee01fc 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -16,6 +16,8 @@ import { SignatureV4 } from "@smithy/signature-v4"; import { HttpRequest } from "@smithy/protocol-http"; import { Sha256 } from "@aws-crypto/sha256-js"; import { v4 } from "uuid"; +import { cloneDeep } from "lodash"; +import { supportsInlayImage } from "../image"; @@ -88,13 +90,34 @@ export async function requestChatData(arg:requestDataArgument, model:'model'|'su } } +interface OpenAITextContents { + type: 'text' + text: string +} +interface OpenAIImageContents { + type: 'image' + image_url: { + url: string + detail: string + } +} + +type OpenAIContents = OpenAITextContents|OpenAIImageContents + +export interface OpenAIChatExtra { + role: 'system'|'user'|'assistant'|'function' + content: string|OpenAIContents[] + memo?:string + name?:string + removable?:boolean +} export async function requestChatDataMain(arg:requestDataArgument, model:'model'|'submodel', abortSignal:AbortSignal=null):Promise { const db = get(DataBase) let result = '' - let formated = arg.formated + let formated = cloneDeep(arg.formated) let maxTokens = arg.maxTokens ??db.maxResponse let temperature = arg.temperature ?? (db.temperature / 100) let bias = arg.bias @@ -125,27 +148,66 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' case 'gpt35_1106': case 'gpt35_0301': case 'gpt4_0301': + case 'gptvi4_1106': case 'openrouter': case 'reverse_proxy':{ - for(let i=0;i 0 && m.role === 'user'){ + let v:OpenAIChatExtra = cloneDeep(m) + let contents:OpenAIContents[] = pendingImages + contents.push({ + "type": "text", + "text": m.content + }) + v.content = contents + formatedChat.push(v) + pendingImages = [] + } + else{ + formatedChat.push(m) + } } - if(db.newOAIHandle && formated[i].memo && formated[i].memo.startsWith('NewChat')){ - formated[i].content === '' + } + } + else{ + formatedChat = formated + } + + for(let i=0;i { + formatedChat = formatedChat.filter(m => { return m.content !== '' }) } @@ -195,6 +257,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' } + console.log(bias) db.cipherChat = false let body = ({ model: aiModel === 'openrouter' ? db.openrouterRequestModel : @@ -207,12 +270,13 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' : requestModel === "gpt4_0613" ? 'gpt-4-0613' : requestModel === "gpt4_32k_0613" ? 'gpt-4-32k-0613' : requestModel === "gpt4_1106" ? 'gpt-4-1106-preview' + : requestModel === "gptvi4_1106" ? 'gpt-4-vision-preview' : requestModel === "gpt35_1106" ? 'gpt-3.5-turbo-1106' : requestModel === 'gpt35_0301' ? 'gpt-3.5-turbo-0301' : requestModel === 'gpt4_0301' ? 'gpt-4-0301' : (!requestModel) ? 'gpt-3.5-turbo' : requestModel, - messages: formated, + messages: formatedChat, temperature: temperature, max_tokens: maxTokens, presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), @@ -226,11 +290,17 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' body.seed = db.generationSeed } - if(db.newOAIHandle){ + if(db.putUserOpen){ // @ts-ignore body.user = getOpenUserString() } + if(supportsInlayImage()){ + // inlay models doesn't support logit_bias + // @ts-ignore + delete body.logit_bias + } + let replacerURL = aiModel === 'openrouter' ? "https://openrouter.ai/api/v1/chat/completions" : (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.openai.com/v1/chat/completions') diff --git a/src/ts/storage/database.ts b/src/ts/storage/database.ts index 9843f083..c7ce28ce 100644 --- a/src/ts/storage/database.ts +++ b/src/ts/storage/database.ts @@ -319,6 +319,7 @@ export function setDatabase(data:Database){ data.customProxyRequestModel ??= '' data.generationSeed ??= -1 data.newOAIHandle ??= true + data.gptVisionQuality ??= 'low' changeLanguage(data.language) DataBase.set(data) } @@ -494,6 +495,9 @@ export interface Database{ customProxyRequestModel:string generationSeed:number newOAIHandle:boolean + putUserOpen: boolean + inlayImage:boolean + gptVisionQuality:string } export interface customscript{ diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index 77e76b9d..3a54c064 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -3,6 +3,7 @@ import type { Tokenizer } from "@mlc-ai/web-tokenizers"; import { DataBase, type character } from "./storage/database"; import { get } from "svelte/store"; import type { OpenAIChat } from "./process"; +import { supportsInlayImage } from "./image"; async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{ let db = get(DataBase) @@ -94,6 +95,46 @@ export class ChatTokenizer { this.useName = useName } async tokenizeChat(data:OpenAIChat) { + if(data.memo && data.memo.startsWith('inlayImage')){ + const db = get(DataBase) + if(!supportsInlayImage()){ + return this.chatAdditonalTokens + } + if(db.gptVisionQuality === 'low'){ + return 87 + } + + let encoded = this.chatAdditonalTokens + const memo = data.memo.split('-') + let height = parseInt(memo[1]) + let width = parseInt(memo[2]) + + if(height === width){ + if(height > 768){ + height = 768 + width = 768 + } + } + else if(height > width){ + if(width > 768){ + width = 768 + height = height * (768 / width) + } + } + else{ + if(height > 768){ + height = 768 + width = width * (768 / height) + } + } + + const chunkSize = Math.ceil(width / 512) * Math.ceil(height / 512) + encoded += chunkSize * 2 + encoded += 85 + + return encoded + } + let encoded = (await encode(data.content)).length + this.chatAdditonalTokens if(data.name && this.useName ==='name'){ encoded += (await encode(data.name)).length + 1