mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main'

Browse Source
This commit is contained in:
LightningHyperBlaze45654
2024-12-08 20:08:25 -08:00
parent 0c51e898f9 71ef6099a8
commit 69f44c03c6
30 changed files with 887 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/ts/process/embedding/addinfo.ts
View File

@@ -3,7 +3,7 @@ import { HypaProcesser } from '../memory/hypamemory'
import { getUserName } from "src/ts/util";
export async function additionalInformations(char: character,chats:Chat,){
const processer = new HypaProcesser('MiniLM')
const processer = new HypaProcesser()
const db = getDatabase()
const info = char.additionalText

2
src/ts/process/files/image.ts
View File

@@ -84,7 +84,7 @@ export async function getInlayImage(id: string){
export function supportsInlayImage(){
const db = getDatabase()
return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') ||
return db.aiModel.startsWith('gptv') || db.aiModel === 'gemini-pro-vision' || db.aiModel.startsWith('gemini-exp') || db.aiModel.startsWith('claude-3') || db.aiModel.startsWith('gpt4_turbo') || db.aiModel.startsWith('gpt5') || db.aiModel.startsWith('gpt4o') ||
(db.aiModel === 'reverse_proxy' && (
db.proxyRequestModel?.startsWith('gptv') || db.proxyRequestModel === 'gemini-pro-vision' || db.proxyRequestModel?.startsWith('claude-3') || db.proxyRequestModel.startsWith('gpt4_turbo') ||
db.proxyRequestModel?.startsWith('gpt5') || db.proxyRequestModel?.startsWith('gpt4o') ||

6
src/ts/process/files/multisend.ts
View File

@@ -124,7 +124,7 @@ async function sendPDFFile(arg:sendFileArg) {
}
}
console.log(texts)
const hypa = new HypaProcesser('MiniLM')
const hypa = new HypaProcesser()
hypa.addText(texts)
const result = await hypa.similaritySearch(arg.query)
let message = ''
@@ -142,7 +142,7 @@ async function sendTxtFile(arg:sendFileArg) {
const lines = arg.file.split('\n').filter((a) => {
return a !== ''
})
const hypa = new HypaProcesser('MiniLM')
const hypa = new HypaProcesser()
hypa.addText(lines)
const result = await hypa.similaritySearch(arg.query)
let message = ''
@@ -157,7 +157,7 @@ async function sendTxtFile(arg:sendFileArg) {
}
async function sendXMLFile(arg:sendFileArg) {
const hypa = new HypaProcesser('MiniLM')
const hypa = new HypaProcesser()
let nodeTexts:string[] = []
const parser = new DOMParser();
const xmlDoc = parser.parseFromString(arg.file, "text/xml");

23
src/ts/process/index.svelte.ts
View File

@@ -1,10 +1,10 @@
import { get, writable } from "svelte/store";
import { type character, type MessageGenerationInfo, type Chat } from "../storage/database.svelte";
import { type character, type MessageGenerationInfo, type Chat, changeToPreset } from "../storage/database.svelte";
import { DBState } from '../stores.svelte';
import { CharEmotion, selectedCharID } from "../stores.svelte";
import { ChatTokenizer, tokenize, tokenizeNum } from "../tokenizer";
import { language } from "../../lang";
import { alertError } from "../alert";
import { alertError, alertToast } from "../alert";
import { loadLoreBookV3Prompt } from "./lorebook.svelte";
import { findCharacterbyId, getAuthorNoteDefaultText, getPersonaPrompt, getUserName, isLastCharPunctuation, trimUntilPunctuation } from "../util";
import { requestChatData } from "./request";
@@ -109,6 +109,23 @@ export async function sendChat(chatProcessIndex = -1,arg:{
}
doingChat.set(true)
if(chatProcessIndex === -1 && DBState.db.presetChain){
const names = DBState.db.presetChain.split(',').map((v) => v.trim())
const randomSelect = Math.floor(Math.random() * names.length)
const ele = names[randomSelect]
const findId = DBState.db.botPresets.findIndex((v) => {
return v.name === ele
})
if(findId === -1){
alertToast(`Cannot find preset: ${ele}`)
}
else{
changeToPreset(findId, true)
}
}
if(connectionOpen){
chatProcessStage.set(4)
const peerSafe = await peerSafeCheck()
@@ -1396,7 +1413,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{
}
if(DBState.db.emotionProcesser === 'embedding'){
const hypaProcesser = new HypaProcesser('MiniLM')
const hypaProcesser = new HypaProcesser()
await hypaProcesser.addText(emotionList.map((v) => 'emotion:' + v))
let searched = (await hypaProcesser.similaritySearchScored(result)).map((v) => {
v[0] = v[0].replace("emotion:",'')

2
src/ts/process/lua.ts
View File

@@ -213,7 +213,7 @@ export async function runLua(code:string, arg:{
if(!LuaLowLevelIds.has(id)){
return
}
const processer = new HypaProcesser('MiniLM')
const processer = new HypaProcesser()
await processer.addText(value)
return await processer.similaritySearch(source)
})

2
src/ts/process/memory/hanuraiMemory.ts
View File

@@ -13,7 +13,7 @@ export async function hanuraiMemory(chats:OpenAIChat[],arg:{
}){
const db = getDatabase()
const tokenizer = arg.tokenizer
const processer = new HypaProcesser('MiniLM')
const processer = new HypaProcesser()
let addTexts:string[] = []
const queryStartIndex=chats.length-maxRecentChatQuery
console.log(chats.length,maxRecentChatQuery,queryStartIndex)

60
src/ts/process/memory/hypamemory.ts
View File

@@ -1,22 +1,49 @@
import localforage from "localforage";
import {globalFetch} from "src/ts/globalApi.svelte";
import {runEmbedding} from "../transformers";
import {appendLastPath} from "src/ts/util";
import { globalFetch } from "src/ts/globalApi.svelte";
import { runEmbedding } from "../transformers";
import { alertError } from "src/ts/alert";
import { appendLastPath } from "src/ts/util";
import { getDatabase } from "src/ts/storage/database.svelte";
export type HypaModel = 'ada'|'MiniLM'|'nomic'|'custom'|'nomicGPU'|'bgeSmallEn'|'bgeSmallEnGPU'|'bgem3'|'bgem3GPU'|'openai3small'|'openai3large'
const localModels = {
models: {
'MiniLM':'Xenova/all-MiniLM-L6-v2',
'nomic':'nomic-ai/nomic-embed-text-v1.5',
'nomicGPU':'nomic-ai/nomic-embed-text-v1.5',
'bgeSmallEn': 'BAAI/bge-small-en-v1.5',
'bgeSmallEnGPU': 'BAAI/bge-small-en-v1.5',
'bgem3': 'BAAI/bge-m3',
'bgem3GPU': 'BAAI/bge-m3',
},
gpuModels:[
'nomicGPU',
'bgeSmallEnGPU',
'bgem3GPU'
]
}
export class HypaProcesser{
oaikey:string
vectors:memoryVector[]
forage:LocalForage
model:'ada'|'MiniLM'|'nomic'|'custom'
model:HypaModel
customEmbeddingUrl:string
constructor(model:'ada'|'MiniLM'|'nomic'|'custom',customEmbeddingUrl?:string){
constructor(model:HypaModel|'auto' = 'auto',customEmbeddingUrl?:string){
this.forage = localforage.createInstance({
name: "hypaVector"
})
this.vectors = []
this.model = model
if(model === 'auto'){
const db = getDatabase()
this.model = db.hypaModel || 'MiniLM'
}
else{
this.model = model
}
this.customEmbeddingUrl = customEmbeddingUrl
}
@@ -38,9 +65,9 @@ export class HypaProcesser{
async getEmbeds(input:string[]|string):Promise<VectorArray[]> {
if(this.model === 'MiniLM' || this.model === 'nomic'){
if(Object.keys(localModels.models).includes(this.model)){
const inputs:string[] = Array.isArray(input) ? input : [input]
let results:Float32Array[] = await runEmbedding(inputs, this.model === 'nomic' ? 'nomic-ai/nomic-embed-text-v1.5' : 'Xenova/all-MiniLM-L6-v2')
let results:Float32Array[] = await runEmbedding(inputs, localModels.models[this.model], localModels.gpuModels.includes(this.model) ? 'webgpu' : 'wasm')
return results
}
let gf = null;
@@ -57,14 +84,21 @@ export class HypaProcesser{
},
})
}
if(this.model === 'ada'){
if(this.model === 'ada' || this.model === 'openai3small' || this.model === 'openai3large'){
const db = getDatabase()
const models = {
'ada':'text-embedding-ada-002',
'openai3small':'text-embedding-3-small',
'openai3large':'text-embedding-3-large'
}
gf = await globalFetch("https://api.openai.com/v1/embeddings", {
headers: {
"Authorization": "Bearer " + this.oaikey
"Authorization": "Bearer " + db.supaMemoryKey || this.oaikey
},
body: {
"input": input,
"model": "text-embedding-ada-002"
"input": input,
"model": models[this.model]
}
})
}
@@ -138,7 +172,7 @@ export class HypaProcesser{
}
async similaritySearchScored(query: string) {
return await this.similaritySearchVectorWithScore((await this.getEmbeds(query))[0],)
return await this.similaritySearchVectorWithScore((await this.getEmbeds(query))[0],);
}
private async similaritySearchVectorWithScore(

42
src/ts/process/memory/hypav2.ts
View File

@@ -30,6 +30,7 @@ async function summary(
stringlizedChat: string
): Promise<{ success: boolean; data: string }> {
const db = getDatabase();
console.log("Summarizing");
if (db.supaModelType === "distilbart") {
try {
@@ -101,35 +102,27 @@ async function summary(
supaPrompt.replaceAll("{{slot}}", stringlizedChat)
);
const promptbody: OpenAIChat[] = parsedPrompt ?? [
const promptbody: OpenAIChat[] = (parsedPrompt ?? [
{
role: "user",
content: stringlizedChat,
},
{
role: "system",
content: supaPrompt,
},
];
console.log(
"Using submodel: ",
db.subModel,
"for supaMemory model"
);
const da = await requestChatData(
{
formated: promptbody,
bias: {},
useStreaming: false,
noMultiGen: true,
},
"memory"
);
if (
da.type === "fail" ||
da.type === "streaming" ||
da.type === "multiline"
) {
content: supaPrompt
}
]).map(message => ({
...message,
memo: "supaPrompt"
}));
console.log("Using submodel: ", db.subModel, "for supaMemory model");
const da = await requestChatData({
formated: promptbody,
bias: {},
useStreaming: false,
noMultiGen: true
}, 'memory');
if (da.type === 'fail' || da.type === 'streaming' || da.type === 'multiline') {
return {
success: false,
data: "SupaMemory: HTTP: " + da.result,
@@ -179,7 +172,7 @@ export async function regenerateSummary(
mainChunkIndex: number
) : Promise<void> {
const targetMainChunk = data.mainChunks[mainChunkIndex];
}
export async function hypaMemoryV2(
chats: OpenAIChat[],
@@ -209,6 +202,7 @@ export async function hypaMemoryV2(
currentTokens += allocatedTokens + chats.length * 4; // ChatML token counting from official openai documentation
let mainPrompt = "";
const lastTwoChats = chats.slice(-2);
// Error handling for infinite summarization attempts
let summarizationFailures = 0;
const maxSummarizationFailures = 3;

2
src/ts/process/models/modelString.ts
View File

@@ -4,7 +4,7 @@ export function getGenerationModelString(){
const db = getDatabase()
switch (db.aiModel){
case 'reverse_proxy':
return 'reverse_proxy-' + (db.reverseProxyOobaMode ? 'ooba' : db.proxyRequestModel)
return 'custom-' + (db.reverseProxyOobaMode ? 'ooba' : db.customProxyRequestModel)
case 'openrouter':
return 'openrouter-' + db.openrouterRequestModel
default:

84
src/ts/process/request.ts
View File

@@ -1364,7 +1364,6 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
let reformatedChat:GeminiChat[] = []
let pendingImage = ''
let systemPrompt = ''
if(formated[0].role === 'system'){
@@ -1374,10 +1373,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
for(let i=0;i<formated.length;i++){
const chat = formated[i]
if(chat.memo && chat.memo.startsWith('inlayImage')){
pendingImage = chat.content
continue
}
if(i === 0){
if(chat.role === 'user' || chat.role === 'assistant'){
reformatedChat.push({
@@ -1403,7 +1399,34 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
chat.role === 'assistant' ? 'MODEL' :
chat.role
if(prevChat.role === qRole){
if (chat.multimodals && chat.multimodals.length > 0 && chat.role === "user") {
let geminiParts: GeminiPart[] = [];
geminiParts.push({
text: chat.content,
});
for (const modal of chat.multimodals) {
if (modal.type === "image") {
const dataurl = modal.base64;
const base64 = dataurl.split(",")[1];
const mediaType = dataurl.split(";")[0].split(":")[1];
geminiParts.push({
inlineData: {
mimeType: mediaType,
data: base64,
}
});
}
}
reformatedChat.push({
role: "USER",
parts: geminiParts,
});
} else if (prevChat.role === qRole) {
reformatedChat[reformatedChat.length-1].parts[0].text += '\n' + chat.content
continue
}
@@ -1420,36 +1443,7 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
})
}
}
else if(chat.role === 'user' && pendingImage !== ''){
//conver image to jpeg so it can be inlined
const canv = document.createElement('canvas')
const img = new Image()
img.src = pendingImage
await img.decode()
canv.width = img.width
canv.height = img.height
const ctx = canv.getContext('2d')
ctx.drawImage(img, 0, 0)
const base64 = canv.toDataURL('image/jpeg').replace(/^data:image\/jpeg;base64,/, "")
const mimeType = 'image/jpeg'
pendingImage = ''
canv.remove()
img.remove()
reformatedChat.push({
role: "USER",
parts: [
{
text: chat.content,
},
{
inlineData: {
mimeType: mimeType,
data: base64
}
}]
})
}
else if(chat.role === 'assistant' || chat.role === 'user'){
reformatedChat.push({
role: chat.role === 'user' ? 'USER' : 'MODEL',
@@ -1578,14 +1572,24 @@ async function requestGoogleCloudVertex(arg:RequestDataArgumentExtended):Promise
}
}
const url = arg.customURL ?? (arg.modelInfo.format === LLMFormat.VertexAIGemini ?
`https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${arg.modelInfo.internalID}:streamGenerateContent`
: `https://generativelanguage.googleapis.com/v1beta/models/${arg.modelInfo.internalID}:generateContent?key=${db.google.accessToken}`)
let url = ''
if(arg.customURL){
const u = new URL(arg.customURL)
u.searchParams.set('key', db.proxyKey)
url = u.toString()
}
else if(arg.modelInfo.format === LLMFormat.VertexAIGemini){
url =`https://${REGION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${arg.modelInfo.internalID}:streamGenerateContent`
}
else{
url = `https://generativelanguage.googleapis.com/v1beta/models/${arg.modelInfo.internalID}:generateContent?key=${db.google.accessToken}`
}
const res = await globalFetch(url, {
headers: headers,
body: body,
chatId: arg.chatId,
abortSignal: arg.abortSignal
abortSignal: arg.abortSignal,
})
if(!res.ok){
@@ -2533,7 +2537,7 @@ async function requestWebLLM(arg:RequestDataArgumentExtended):Promise<requestDat
top_p: db.ooba.top_p,
repetition_penalty: db.ooba.repetition_penalty,
typical_p: db.ooba.typical_p,
})
} as any)
return {
type: 'success',
result: unstringlizeChat(v.generated_text as string, formated, currentChar?.name ?? '')

2
src/ts/process/scripts.ts
View File

@@ -321,7 +321,7 @@ export async function processScriptFull(char:character|groupChat|simpleCharacter
}
}
const processer = new HypaProcesser('MiniLM')
const processer = new HypaProcesser()
await processer.addText(assetNames)
const matches = data.matchAll(assetRegex)

26
src/ts/process/transformers.ts
View File

@@ -1,4 +1,4 @@
import {env, AutoTokenizer, pipeline, type SummarizationOutput, type TextGenerationConfig, type TextGenerationOutput, FeatureExtractionPipeline, TextToAudioPipeline, type ImageToTextOutput } from '@xenova/transformers';
import {env, AutoTokenizer, pipeline, type SummarizationOutput, type TextGenerationConfig, type TextGenerationOutput, FeatureExtractionPipeline, TextToAudioPipeline, type ImageToTextOutput } from '@huggingface/transformers';
import { unzip } from 'fflate';
import { globalFetch, loadAsset, saveAsset } from 'src/ts/globalApi.svelte';
import { selectSingleFile } from 'src/ts/util';
@@ -15,6 +15,7 @@ async function initTransformers(){
env.useBrowserCache = false
env.useFSCache = false
env.useCustomCache = true
env.allowLocalModels = true
env.customCache = {
put: async (url:URL|string, response:Response) => {
await tfCache.put(url, response)
@@ -33,10 +34,12 @@ async function initTransformers(){
console.log('transformers loaded')
}
export const runTransformers = async (baseText:string, model:string,config:TextGenerationConfig = {}) => {
export const runTransformers = async (baseText:string, model:string,config:TextGenerationConfig, device:'webgpu'|'wasm' = 'wasm') => {
await initTransformers()
let text = baseText
let generator = await pipeline('text-generation', model);
let generator = await pipeline('text-generation', model, {
device
});
let output = await generator(text, config) as TextGenerationOutput
const outputOne = output[0]
return outputOne
@@ -50,16 +53,25 @@ export const runSummarizer = async (text: string) => {
}
let extractor:FeatureExtractionPipeline = null
let lastEmbeddingModelQuery:string = ''
type EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5'
export const runEmbedding = async (texts: string[], model:EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array[]> => {
export const runEmbedding = async (texts: string[], model:EmbeddingModel = 'Xenova/all-MiniLM-L6-v2', device:'webgpu'|'wasm'):Promise<Float32Array[]> => {
await initTransformers()
if(!extractor){
extractor = await pipeline('feature-extraction', model);
console.log('running embedding')
let embeddingModelQuery = model + device
if(!extractor || embeddingModelQuery !== lastEmbeddingModelQuery){
extractor = await pipeline('feature-extraction', model, {
device: device,
progress_callback: (progress) => {
console.log(progress)
}
});
console.log('extractor loaded')
}
let result = await extractor(texts, { pooling: 'mean', normalize: true });
console.log(texts, result)
const data = result.data as Float32Array
console.log(data)
const lenPerText = data.length / texts.length
let res:Float32Array[] = []
for(let i = 0; i < texts.length; i++){

2
src/ts/process/triggers.ts
View File

@@ -459,7 +459,7 @@ export async function runTrigger(char:character,mode:triggerMode, arg:{
break
}
const processer = new HypaProcesser('MiniLM')
const processer = new HypaProcesser()
const effectValue = risuChatParser(effect.value,{chara:char})
const source = risuChatParser(effect.source,{chara:char})
await processer.addText(effectValue.split('§'))