Add Harunai Memory

src/ts/process/index.ts

@@ -26,6 +26,7 @@ import { runInlayScreen } from "./inlayScreen";
 import { runCharacterJS } from "../plugins/embedscript";
 import { addRerolls } from "./prereroll";
 import { runImageEmbedding } from "./transformers";
+import { hanuraiMemory } from "./memory/hanuraiMemory";
 
 export interface OpenAIChat{
     role: 'system'|'user'|'assistant'|'function'
@@ -647,22 +648,39 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
         index++
     }
 
-    if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){
+    
+    if(nowChatroom.supaMemory && (db.supaMemoryType !== 'none' || db.hanuraiEnable)){
         chatProcessStage.set(2)
-        const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, {
-            asHyper: db.hypaMemory
-        })
-        if(sp.error){
-            alertError(sp.error)
-            return false
+        if(db.hanuraiEnable){
+            const hn = await hanuraiMemory(chats, {
+                currentTokens,
+                maxContextTokens,
+                tokenizer
+            })
+
+            if(hn === false){
+                return false
+            }
+
+            chats = hn.chats
+            currentTokens = hn.tokens
+        }
+        else{
+            const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, {
+                asHyper: db.hypaMemory
+            })
+            if(sp.error){
+                alertError(sp.error)
+                return false
+            }
+            chats = sp.chats
+            currentTokens = sp.currentTokens
+            currentChat.supaMemoryData = sp.memory ?? currentChat.supaMemoryData
+            db.characters[selectedChar].chats[selectedChat].supaMemoryData = currentChat.supaMemoryData
+            console.log(currentChat.supaMemoryData)
+            DataBase.set(db)
+            currentChat.lastMemory = sp.lastId ?? currentChat.lastMemory;
         }
-        chats = sp.chats
-        currentTokens = sp.currentTokens
-        currentChat.supaMemoryData = sp.memory ?? currentChat.supaMemoryData
-        db.characters[selectedChar].chats[selectedChat].supaMemoryData = currentChat.supaMemoryData
-        console.log(currentChat.supaMemoryData)
-        DataBase.set(db)
-        currentChat.lastMemory = sp.lastId ?? currentChat.lastMemory
         chatProcessStage.set(1)
     }
     else{

src/ts/process/memory/hanuraiMemory.ts

@@ -0,0 +1,94 @@
+import { alertError } from "src/ts/alert";
+import type { OpenAIChat } from "..";
+import { HypaProcesser } from "./hypamemory";
+import { language } from "src/lang";
+import type { ChatTokenizer } from "src/ts/tokenizer";
+import { get } from "svelte/store";
+import { DataBase } from "src/ts/storage/database";
+
+export async function hanuraiMemory(chats:OpenAIChat[],arg:{
+    currentTokens:number,
+    maxContextTokens:number,
+    tokenizer:ChatTokenizer
+}){
+    const db = get(DataBase)
+    const tokenizer = arg.tokenizer
+    const processer = new HypaProcesser('nomic')
+    let addTexts:string[] = []
+    chats.map((chat) => {
+        if(!chat?.content?.trim()){
+            return
+        }
+        if(db.hanuraiSplit){
+            const splited = chat.content.split('\n\n')
+            for(const split of splited){
+                if(!split.trim()){
+                    continue
+                }
+                addTexts.push(`search_document: ${split.trim()}`)
+            }
+        }
+        addTexts.push(`search_document: ${chat.content?.trim()}`)
+    })
+    processer.addText(addTexts)
+
+    let scoredResults:{[key:string]:number} = {}
+    for(let i=1;i<5;i++){
+        const chat = chats[chats.length-i]
+        if(!chat?.content){
+            continue
+        }
+        const scoredArray = (await processer.similaritySearchScored('search_query: ' + chat.content)).map((result) => {
+            return [result[0],result[1]/i] as [string,number]
+        })
+        for(const scored of scoredArray){
+            if(scoredResults[scored[0]]){
+                scoredResults[scored[0]] += scored[1]
+            }else{
+                scoredResults[scored[0]] = scored[1]
+            }
+        }
+    }
+    const vectorResult = Object.entries(scoredResults).sort((a,b)=>a[1]-b[1])
+
+
+    let tokens = arg.currentTokens + db.hanuraiTokens
+
+    while(tokens < arg.maxContextTokens){
+        const poped = chats.pop()
+        if(!poped){
+            alertError(language.errors.toomuchtoken + "\n\nRequired Tokens: " + tokens)
+            return false
+        }
+        tokens -= await tokenizer.tokenizeChat(chats[0])
+    }
+
+    tokens -= db.hanuraiTokens
+
+    let resultTexts:string[] = []
+    for(const vector of vectorResult){
+        const chat = chats.find((chat) => chat.content === vector[0].substring(14))
+        if(chat){
+            continue
+        }
+        const tokenized = await tokenizer.tokenizeChat(chat) + 2
+        tokens += tokenized
+        if(tokens >= arg.maxContextTokens){
+            tokens -= tokenized
+            break
+        }
+        resultTexts.push(vector[0].substring(14))
+    }
+    console.log(resultTexts)
+    chats.unshift({
+        role: "system",
+        memo: "supaMemory",
+        content: resultTexts.join('\n\n'),
+    })
+    return {
+        tokens,
+        chats
+    }
+
+
+}

src/ts/process/memory/hypamemory.ts

@@ -92,7 +92,7 @@ export class HypaProcesser{
     async addText(texts:string[]) {
 
         for(let i=0;i<texts.length;i++){
-            const itm:memoryVector = await this.forage.getItem(texts[i])
+            const itm:memoryVector = await this.forage.getItem(texts[i] + '|' + this.model)
             if(itm){
                 itm.alreadySaved = true
                 this.vectors.push(itm)
@@ -121,7 +121,7 @@ export class HypaProcesser{
         for(let i=0;i<memoryVectors.length;i++){
             const vec = memoryVectors[i]
             if(!vec.alreadySaved){
-                await this.forage.setItem(texts[i], vec)
+                await this.forage.setItem(texts[i] + '|' + this.model, vec)
             }
         }
 

src/ts/process/memory/termMemory.ts

@@ -1,28 +0,0 @@
-import type { OpenAIChat } from "..";
-import { HypaProcesser } from "./hypamemory";
-
-export async function termMemory(chats:OpenAIChat[]){
-    const processer = new HypaProcesser('nomic')
-    processer.addText(chats.map(chat=>chat.content))
-
-    let scoredResults:{[key:string]:number}
-    for(let i=1;i<5;i++){
-        const chat = chats[chats.length-i]
-        if(!chat?.content){
-            continue
-        }
-        const scoredArray = (await processer.similaritySearchScored(chat.content)).map((result) => {
-            return [result[0],result[1]/i] as [string,number]
-        })
-        for(const scored of scoredArray){
-            if(scoredResults[scored[0]]){
-                scoredResults[scored[0]] += scored[1]
-            }else{
-                scoredResults[scored[0]] = scored[1]
-            }
-        }
-    }
-    const result = Object.entries(scoredResults).sort((a,b)=>a[1]-b[1])
-    return result.map(([content,score])=>(content)).join('\n\n')
-
-}

src/ts/process/transformers.ts

@@ -50,7 +50,8 @@ export const runSummarizer = async (text: string) => {
 }
 
 let extractor:FeatureExtractionPipeline = null
-export const runEmbedding = async (text: string, model:'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5' = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array> => {
+type EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5'
+export const runEmbedding = async (text: string, model:EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array> => {
     await initTransformers()
     if(!extractor){
         extractor = await pipeline('feature-extraction', model);