Add postfile function
This commit is contained in:
kwaroran
109
src/ts/process/files/image.ts
Normal file
109
src/ts/process/files/image.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import localforage from "localforage";
|
||||
import { selectSingleFile } from "../../util";
|
||||
import { v4 } from "uuid";
|
||||
import { DataBase } from "../../storage/database";
|
||||
import { get } from "svelte/store";
|
||||
import { checkImageType } from "../../parser";
|
||||
|
||||
const inlayStorage = localforage.createInstance({
|
||||
name: 'inlay',
|
||||
storeName: 'inlay'
|
||||
})
|
||||
|
||||
export async function postInlayImage(img:{
|
||||
name:string,
|
||||
data:Uint8Array
|
||||
}){
|
||||
|
||||
const extention = img.name.split('.').at(-1)
|
||||
const imgObj = new Image()
|
||||
imgObj.src = URL.createObjectURL(new Blob([img.data], {type: `image/${extention}`}))
|
||||
|
||||
return await writeInlayImage(imgObj, {
|
||||
name: img.name,
|
||||
ext: extention
|
||||
})
|
||||
}
|
||||
|
||||
export async function writeInlayImage(imgObj:HTMLImageElement, arg:{name?:string, ext?:string} = {}) {
|
||||
|
||||
let drawHeight = 0
|
||||
let drawWidth = 0
|
||||
const canvas = document.createElement('canvas')
|
||||
const ctx = canvas.getContext('2d')
|
||||
await new Promise((resolve) => {
|
||||
imgObj.onload = () => {
|
||||
drawHeight = imgObj.height
|
||||
drawWidth = imgObj.width
|
||||
|
||||
//resize image to fit inlay, if it's too big (max 1024px)
|
||||
if(drawHeight > 1024){
|
||||
drawWidth = drawWidth * (1024 / drawHeight)
|
||||
drawHeight = 1024
|
||||
}
|
||||
if(drawWidth > 1024){
|
||||
drawHeight = drawHeight * (1024 / drawWidth)
|
||||
drawWidth = 1024
|
||||
}
|
||||
drawHeight = Math.floor(drawHeight)
|
||||
drawWidth = Math.floor(drawWidth)
|
||||
|
||||
canvas.width = drawWidth
|
||||
canvas.height = drawHeight
|
||||
ctx.drawImage(imgObj, 0, 0, drawWidth, drawHeight)
|
||||
resolve(null)
|
||||
}
|
||||
})
|
||||
const dataURI = canvas.toDataURL('image/png')
|
||||
|
||||
|
||||
const imgid = v4()
|
||||
|
||||
await inlayStorage.setItem(imgid, {
|
||||
name: arg.name ?? imgid,
|
||||
data: dataURI,
|
||||
ext: arg.ext ?? 'png',
|
||||
height: drawHeight,
|
||||
width: drawWidth
|
||||
})
|
||||
|
||||
return `{{inlay::${imgid}}}`
|
||||
}
|
||||
|
||||
export async function getInlayImage(id: string){
|
||||
const img:{
|
||||
name: string,
|
||||
data: string
|
||||
ext: string
|
||||
height: number
|
||||
width: number
|
||||
} = await inlayStorage.getItem(id)
|
||||
if(img === null){
|
||||
return null
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
export function supportsInlayImage(){
|
||||
const db = get(DataBase)
|
||||
return db.aiModel.startsWith('gptv') || (db.aiModel === 'reverse_proxy' && db.proxyRequestModel?.startsWith('gptv')) || db.aiModel === 'gemini-pro-vision'
|
||||
}
|
||||
|
||||
export async function reencodeImage(img:Uint8Array){
|
||||
if(checkImageType(img) === 'PNG'){
|
||||
return img
|
||||
}
|
||||
const canvas = document.createElement('canvas')
|
||||
const imgObj = new Image()
|
||||
imgObj.src = URL.createObjectURL(new Blob([img], {type: `image/png`}))
|
||||
await imgObj.decode()
|
||||
let drawHeight = imgObj.height
|
||||
let drawWidth = imgObj.width
|
||||
canvas.width = drawWidth
|
||||
canvas.height = drawHeight
|
||||
const ctx = canvas.getContext('2d')
|
||||
ctx.drawImage(imgObj, 0, 0, drawWidth, drawHeight)
|
||||
const b64 = canvas.toDataURL('image/png').split(',')[1]
|
||||
const b = Buffer.from(b64, 'base64')
|
||||
return b
|
||||
}
|
||||
192
src/ts/process/files/multisend.ts
Normal file
192
src/ts/process/files/multisend.ts
Normal file
@@ -0,0 +1,192 @@
|
||||
import { DataBase, setDatabase } from 'src/ts/storage/database';
|
||||
import { selectedCharID } from 'src/ts/stores';
|
||||
import { get } from 'svelte/store';
|
||||
import { doingChat, sendChat } from '..';
|
||||
import { downloadFile, isTauri } from 'src/ts/storage/globalApi';
|
||||
import { HypaProcesser } from '../memory/hypamemory';
|
||||
import { BufferToText as BufferToText, selectSingleFile, sleep } from 'src/ts/util';
|
||||
import { postInlayImage } from './image';
|
||||
|
||||
type sendFileArg = {
|
||||
file:string
|
||||
query:string
|
||||
}
|
||||
|
||||
async function sendPofile(arg:sendFileArg){
|
||||
|
||||
let result = ''
|
||||
let msgId = ''
|
||||
let parseMode = 0
|
||||
const db = get(DataBase)
|
||||
let currentChar = db.characters[get(selectedCharID)]
|
||||
let currentChat = currentChar.chats[currentChar.chatPage]
|
||||
const lines = arg.file.split('\n')
|
||||
for(let i=0;i<lines.length;i++){
|
||||
console.log(i)
|
||||
const line = lines[i]
|
||||
if(line === ''){
|
||||
if(msgId === ''){
|
||||
result += '\n'
|
||||
continue
|
||||
}
|
||||
const id = msgId
|
||||
currentChat.message.push({
|
||||
role: 'user',
|
||||
data: id
|
||||
})
|
||||
currentChar.chats[currentChar.chatPage] = currentChat
|
||||
db.characters[get(selectedCharID)] = currentChar
|
||||
setDatabase(db)
|
||||
doingChat.set(false)
|
||||
await sendChat(-1);
|
||||
currentChar = db.characters[get(selectedCharID)]
|
||||
currentChat = currentChar.chats[currentChar.chatPage]
|
||||
const res = currentChat.message[currentChat.message.length-1]
|
||||
const msgStr = res.data.split('\n').filter((a) => {
|
||||
return a !== ''
|
||||
}).map((str) => {
|
||||
return `"${str.replaceAll('"', '\\"')}"`
|
||||
}).join('\n')
|
||||
result += `msgstr ""\n${msgStr}\n\n`
|
||||
|
||||
msgId = ''
|
||||
if(isTauri){
|
||||
await downloadFile('translated.po', result)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if(line.startsWith('msgid')){
|
||||
parseMode = 0
|
||||
msgId = line.replace('msgid ', '').trim().replaceAll('\\"', '♠#').replaceAll('"', '').replaceAll('♠#', '\\"')
|
||||
if(msgId === ''){
|
||||
parseMode = 1
|
||||
}
|
||||
result += line + '\n'
|
||||
continue
|
||||
}
|
||||
if(parseMode === 1 && line.startsWith('"') && line.endsWith('"')){
|
||||
msgId += line.substring(1, line.length-1).replaceAll('\\"', '"')
|
||||
result += line + '\n'
|
||||
continue
|
||||
}
|
||||
if(line.startsWith('msgstr')){
|
||||
if(msgId === ''){
|
||||
result += line + '\n'
|
||||
parseMode = 0
|
||||
}
|
||||
else{
|
||||
parseMode = 2
|
||||
}
|
||||
continue
|
||||
}
|
||||
if(parseMode === 2 && line.startsWith('"') && line.endsWith('"')){
|
||||
continue
|
||||
}
|
||||
result += line + '\n'
|
||||
|
||||
if(i > 100){
|
||||
break //prevent too long message in testing
|
||||
}
|
||||
|
||||
}
|
||||
await downloadFile('translated.po', result)
|
||||
}
|
||||
|
||||
async function sendPDFFile(arg:sendFileArg) {
|
||||
const pdfjsLib = (await import('pdfjs-dist')).default;
|
||||
const pdf = await pdfjsLib.getDocument({data: arg.file}).promise;
|
||||
const db = get(DataBase)
|
||||
const texts:string[] = []
|
||||
for(let i = 1; i<=pdf.numPages; i++){
|
||||
const page = await pdf.getPage(i);
|
||||
const content = await page.getTextContent();
|
||||
const items = content.items as {str:string}[];
|
||||
for(const item of items){
|
||||
texts.push(item.str)
|
||||
}
|
||||
}
|
||||
const hypa = new HypaProcesser('MiniLM')
|
||||
hypa.addText(texts)
|
||||
let currentChar = db.characters[get(selectedCharID)]
|
||||
let currentChat = currentChar.chats[currentChar.chatPage]
|
||||
const result = await hypa.similaritySearch(arg.query)
|
||||
let message = arg.query
|
||||
for(let i = 0; i<result.length; i++){
|
||||
message += "\n" + texts[result[i]]
|
||||
if(i>5){
|
||||
break
|
||||
}
|
||||
}
|
||||
currentChat.message.push({
|
||||
role: 'user',
|
||||
data: message
|
||||
})
|
||||
|
||||
currentChar.chats[currentChar.chatPage] = currentChat
|
||||
db.characters[get(selectedCharID)] = currentChar
|
||||
setDatabase(db)
|
||||
await sendChat(-1)
|
||||
}
|
||||
|
||||
type postFileResult = postFileResultImage | postFileResultVoid
|
||||
|
||||
type postFileResultImage = {
|
||||
data: string,
|
||||
type: 'image',
|
||||
}
|
||||
|
||||
type postFileResultVoid = {
|
||||
type: 'void',
|
||||
}
|
||||
|
||||
export async function postChatFile(query:string):Promise<postFileResult>{
|
||||
const file = await selectSingleFile([
|
||||
//image format
|
||||
'jpg',
|
||||
'jpeg',
|
||||
'png',
|
||||
'webp',
|
||||
'po',
|
||||
'pdf'
|
||||
])
|
||||
|
||||
if(!file){
|
||||
return null
|
||||
}
|
||||
|
||||
const extention = file.name.split('.').at(-1)
|
||||
console.log(extention)
|
||||
|
||||
switch(extention){
|
||||
case 'po':{
|
||||
await sendPofile({
|
||||
file: BufferToText(file.data),
|
||||
query: query
|
||||
})
|
||||
return {
|
||||
type: 'void'
|
||||
}
|
||||
}
|
||||
case 'pdf':{
|
||||
await sendPDFFile({
|
||||
file: BufferToText(file.data),
|
||||
query: query
|
||||
})
|
||||
return {
|
||||
type: 'void'
|
||||
}
|
||||
}
|
||||
case 'jpg':
|
||||
case 'jpeg':
|
||||
case 'png':
|
||||
case 'webp':{
|
||||
const postData = await postInlayImage(file)
|
||||
return {
|
||||
data: postData,
|
||||
type: 'image'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -19,7 +19,7 @@ import { runTrigger } from "./triggers";
|
||||
import { HypaProcesser } from "./memory/hypamemory";
|
||||
import { additionalInformations } from "./embedding/addinfo";
|
||||
import { cipherChat, decipherChat } from "./cipherChat";
|
||||
import { getInlayImage, supportsInlayImage } from "../image";
|
||||
import { getInlayImage, supportsInlayImage } from "./files/image";
|
||||
import { getGenerationModelString } from "./models/modelString";
|
||||
import { sendPeerChar } from "../sync/multiuser";
|
||||
import { runInlayScreen } from "./inlayScreen";
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { writeInlayImage } from "../image";
|
||||
import { writeInlayImage } from "./files/image";
|
||||
import type { character } from "../storage/database";
|
||||
import { generateAIImage } from "./stableDiff";
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ import { HttpRequest } from "@smithy/protocol-http";
|
||||
import { Sha256 } from "@aws-crypto/sha256-js";
|
||||
import { v4 } from "uuid";
|
||||
import { cloneDeep } from "lodash";
|
||||
import { supportsInlayImage } from "../image";
|
||||
import { supportsInlayImage } from "./files/image";
|
||||
import { OaifixBias } from "../plugins/fixer";
|
||||
import { Capacitor } from "@capacitor/core";
|
||||
import { getFreeOpenRouterModel } from "../model/openrouter";
|
||||
|
||||
Reference in New Issue
Block a user