Add Harunai Memory

package.json

@@ -27,7 +27,7 @@
     "@smithy/protocol-http": "^3.0.12",
     "@smithy/signature-v4": "^2.0.19",
     "@tauri-apps/api": "1.5.3",
-    "@xenova/transformers": "^2.14.0",
+    "@xenova/transformers": "^2.17.1",
     "blueimp-md5": "^2.19.0",
     "body-parser": "^1.20.2",
     "buffer": "^6.0.3",

pnpm-lock.yaml

@@ -39,8 +39,8 @@ dependencies:
     specifier: 1.5.3
     version: 1.5.3
   '@xenova/transformers':
-    specifier: ^2.14.0
+    specifier: ^2.17.1
-    version: 2.14.0
+    version: 2.17.1
   blueimp-md5:
     specifier: ^2.19.0
     version: 2.19.0
@@ -701,11 +701,6 @@ packages:
     dev: true
     optional: true
 
-  /@huggingface/jinja@0.1.2:
-    resolution: {integrity: sha512-x5mpbfJt1nKmVep5WNP5VjNsjWApWNj8pPYI+uYMkBWH9bWUJmQmHt2lbf0VCoQd54Oq3XuFEh/UyoVh7rPxmg==}
-    engines: {node: '>=18'}
-    dev: false
-
   /@huggingface/jinja@0.2.2:
     resolution: {integrity: sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==}
     engines: {node: '>=18'}
@@ -1704,10 +1699,10 @@ packages:
     resolution: {integrity: sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==}
     dev: true
 
-  /@xenova/transformers@2.14.0:
+  /@xenova/transformers@2.17.1:
-    resolution: {integrity: sha512-rQ3O7SW5EM64b6XFZGx3XQ2cfiroefxUwU9ShfSpEZyhd082GvwNJJKndxgaukse1hZP1JUDoT0DfjDiq4IZiw==}
+    resolution: {integrity: sha512-zo702tQAFZXhzeD2GCYUNUqeqkoueOdiSbQWa4s0q7ZE4z8WBIwIsMMPGobpgdqjQ2u0Qulo08wuqVEUrBXjkQ==}
     dependencies:
-      '@huggingface/jinja': 0.1.2
+      '@huggingface/jinja': 0.2.2
       onnxruntime-web: 1.14.0
       sharp: 0.32.6
     optionalDependencies:

src/lang/en.ts

@@ -579,4 +579,5 @@ export const languageEnglish = {
     tokenizer: "Tokenizer",
     chatFormating: "Chat Formating",
     useInstructPrompt: "Use Instruction Prompt",
+    hanuraiMemory: "HanuraiMemory",
 }

src/lib/Setting/Pages/OtherBotSettings.svelte

@@ -207,32 +207,63 @@
     </SelectInput>
 </Arcodion>
 
-<Arcodion name={language.SuperMemory} styled>
+<Arcodion name={language.SuperMemory}/{language.hanuraiMemory} styled>
-    <span class="text-textcolor mt-4">{language.SuperMemory} {language.model}</span>
+    <span class="text-textcolor mt-4">{language.type}</span>
-    <SelectInput className="mt-2 mb-2" bind:value={$DataBase.supaMemoryType}>
+
+    <SelectInput value={
+        $DataBase.supaMemoryType !== 'none' ? 'supaMemory' :
+        $DataBase.hanuraiEnable ? 'hanuraiMemory' : 'none'
+    } on:change={(v) => {
+        //@ts-ignore
+        const value = v.target.value
+        if (value === 'supaMemory'){
+            $DataBase.supaMemoryType = 'distilbart'
+            $DataBase.hanuraiEnable = false
+        } else if (value === 'hanuraiMemory'){
+            $DataBase.supaMemoryType = 'none'
+            $DataBase.hanuraiEnable = true
+        } else {
+            $DataBase.supaMemoryType = 'none'
+            $DataBase.hanuraiEnable = false
+        }
+    }}>
         <OptionInput value="none" >None</OptionInput>
-        <OptionInput value="distilbart" >distilbart-cnn-6-6 (Free/Local)</OptionInput>
+        <OptionInput value="supaMemory" >{language.SuperMemory}</OptionInput>
-        <OptionInput value="instruct35" >OpenAI 3.5 Turbo Instruct</OptionInput>
+        <OptionInput value="hanuraiMemory" >{language.hanuraiMemory}</OptionInput>
-        <OptionInput value="subModel" >{language.submodel}</OptionInput>
     </SelectInput>
-    <span class="text-textcolor">{language.maxSupaChunkSize}</span>
+
-    <NumberInput size="sm" marginBottom bind:value={$DataBase.maxSupaChunkSize} min={100} />
+    {#if $DataBase.hanuraiEnable}
-    {#if $DataBase.supaMemoryType === 'davinci' || $DataBase.supaMemoryType === 'curie' || $DataBase.supaMemoryType === 'instruct35'}
+        <span>Chunk Size</span>
-        <span class="text-textcolor">{language.SuperMemory} OpenAI Key</span>
+        <NumberInput size="sm" marginBottom bind:value={$DataBase.hanuraiTokens} min={100} />
-        <TextInput size="sm" marginBottom bind:value={$DataBase.supaMemoryKey}/>
+        <div class="flex">
-    {/if}
+            <Check bind:check={$DataBase.hanuraiSplit} name="Text Spliting"/>
-    {#if $DataBase.supaMemoryType !== 'none'}
+        </div>
-        <span class="text-textcolor">{language.SuperMemory} Prompt</span>
+    {:else if $DataBase.supaMemoryType !== 'none'}
-        <TextInput size="sm" marginBottom bind:value={$DataBase.supaMemoryPrompt} placeholder="Leave it blank to use default"/>
+        <span class="text-textcolor mt-4">{language.SuperMemory} {language.model}</span>
-    {/if}
+        <SelectInput className="mt-2 mb-2" bind:value={$DataBase.supaMemoryType}>
-    {#if $DataBase.hypaMemory}
+            <OptionInput value="distilbart" >distilbart-cnn-6-6 (Free/Local)</OptionInput>
-        <span class="text-textcolor">{language.HypaMemory} Model</span>
+            <OptionInput value="instruct35" >OpenAI 3.5 Turbo Instruct</OptionInput>
-        <SelectInput className="mt-2 mb-2" bind:value={$DataBase.hypaModel}>
+            <OptionInput value="subModel" >{language.submodel}</OptionInput>
-            <OptionInput value="MiniLM" >MiniLM-L6-v2 (Free / Local)</OptionInput>
-            <OptionInput value="ada" >OpenAI Ada (Davinci / Curie Only)</OptionInput>
         </SelectInput>
+        <span class="text-textcolor">{language.maxSupaChunkSize}</span>
+        <NumberInput size="sm" marginBottom bind:value={$DataBase.maxSupaChunkSize} min={100} />
+        {#if $DataBase.supaMemoryType === 'davinci' || $DataBase.supaMemoryType === 'curie' || $DataBase.supaMemoryType === 'instruct35'}
+            <span class="text-textcolor">{language.SuperMemory} OpenAI Key</span>
+            <TextInput size="sm" marginBottom bind:value={$DataBase.supaMemoryKey}/>
+        {/if}
+        {#if $DataBase.supaMemoryType !== 'none'}
+            <span class="text-textcolor">{language.SuperMemory} Prompt</span>
+            <TextInput size="sm" marginBottom bind:value={$DataBase.supaMemoryPrompt} placeholder="Leave it blank to use default"/>
+        {/if}
+        {#if $DataBase.hypaMemory}
+            <span class="text-textcolor">{language.HypaMemory} Model</span>
+            <SelectInput className="mt-2 mb-2" bind:value={$DataBase.hypaModel}>
+                <OptionInput value="MiniLM" >MiniLM-L6-v2 (Free / Local)</OptionInput>
+                <OptionInput value="ada" >OpenAI Ada (Davinci / Curie Only)</OptionInput>
+            </SelectInput>
+        {/if}
+        <div class="flex">
+            <Check bind:check={$DataBase.hypaMemory} name={language.enable + ' ' + language.HypaMemory}/>
+        </div>
     {/if}
-    <div class="flex">
-        <Check bind:check={$DataBase.hypaMemory} name={language.enable + ' ' + language.HypaMemory}/>
-    </div>
 </Arcodion>

src/ts/process/index.ts

@@ -26,6 +26,7 @@ import { runInlayScreen } from "./inlayScreen";
 import { runCharacterJS } from "../plugins/embedscript";
 import { addRerolls } from "./prereroll";
 import { runImageEmbedding } from "./transformers";
+import { hanuraiMemory } from "./memory/hanuraiMemory";
 
 export interface OpenAIChat{
     role: 'system'|'user'|'assistant'|'function'
@@ -647,22 +648,39 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
         index++
     }
 
-    if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){
+    
+    if(nowChatroom.supaMemory && (db.supaMemoryType !== 'none' || db.hanuraiEnable)){
         chatProcessStage.set(2)
-        const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, {
+        if(db.hanuraiEnable){
-            asHyper: db.hypaMemory
+            const hn = await hanuraiMemory(chats, {
-        })
+                currentTokens,
-        if(sp.error){
+                maxContextTokens,
-            alertError(sp.error)
+                tokenizer
-            return false
+            })
+
+            if(hn === false){
+                return false
+            }
+
+            chats = hn.chats
+            currentTokens = hn.tokens
+        }
+        else{
+            const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, {
+                asHyper: db.hypaMemory
+            })
+            if(sp.error){
+                alertError(sp.error)
+                return false
+            }
+            chats = sp.chats
+            currentTokens = sp.currentTokens
+            currentChat.supaMemoryData = sp.memory ?? currentChat.supaMemoryData
+            db.characters[selectedChar].chats[selectedChat].supaMemoryData = currentChat.supaMemoryData
+            console.log(currentChat.supaMemoryData)
+            DataBase.set(db)
+            currentChat.lastMemory = sp.lastId ?? currentChat.lastMemory;
         }
-        chats = sp.chats
-        currentTokens = sp.currentTokens
-        currentChat.supaMemoryData = sp.memory ?? currentChat.supaMemoryData
-        db.characters[selectedChar].chats[selectedChat].supaMemoryData = currentChat.supaMemoryData
-        console.log(currentChat.supaMemoryData)
-        DataBase.set(db)
-        currentChat.lastMemory = sp.lastId ?? currentChat.lastMemory
         chatProcessStage.set(1)
     }
     else{

src/ts/process/memory/hanuraiMemory.ts

@@ -0,0 +1,94 @@
+import { alertError } from "src/ts/alert";
+import type { OpenAIChat } from "..";
+import { HypaProcesser } from "./hypamemory";
+import { language } from "src/lang";
+import type { ChatTokenizer } from "src/ts/tokenizer";
+import { get } from "svelte/store";
+import { DataBase } from "src/ts/storage/database";
+
+export async function hanuraiMemory(chats:OpenAIChat[],arg:{
+    currentTokens:number,
+    maxContextTokens:number,
+    tokenizer:ChatTokenizer
+}){
+    const db = get(DataBase)
+    const tokenizer = arg.tokenizer
+    const processer = new HypaProcesser('nomic')
+    let addTexts:string[] = []
+    chats.map((chat) => {
+        if(!chat?.content?.trim()){
+            return
+        }
+        if(db.hanuraiSplit){
+            const splited = chat.content.split('\n\n')
+            for(const split of splited){
+                if(!split.trim()){
+                    continue
+                }
+                addTexts.push(`search_document: ${split.trim()}`)
+            }
+        }
+        addTexts.push(`search_document: ${chat.content?.trim()}`)
+    })
+    processer.addText(addTexts)
+
+    let scoredResults:{[key:string]:number} = {}
+    for(let i=1;i<5;i++){
+        const chat = chats[chats.length-i]
+        if(!chat?.content){
+            continue
+        }
+        const scoredArray = (await processer.similaritySearchScored('search_query: ' + chat.content)).map((result) => {
+            return [result[0],result[1]/i] as [string,number]
+        })
+        for(const scored of scoredArray){
+            if(scoredResults[scored[0]]){
+                scoredResults[scored[0]] += scored[1]
+            }else{
+                scoredResults[scored[0]] = scored[1]
+            }
+        }
+    }
+    const vectorResult = Object.entries(scoredResults).sort((a,b)=>a[1]-b[1])
+
+
+    let tokens = arg.currentTokens + db.hanuraiTokens
+
+    while(tokens < arg.maxContextTokens){
+        const poped = chats.pop()
+        if(!poped){
+            alertError(language.errors.toomuchtoken + "\n\nRequired Tokens: " + tokens)
+            return false
+        }
+        tokens -= await tokenizer.tokenizeChat(chats[0])
+    }
+
+    tokens -= db.hanuraiTokens
+
+    let resultTexts:string[] = []
+    for(const vector of vectorResult){
+        const chat = chats.find((chat) => chat.content === vector[0].substring(14))
+        if(chat){
+            continue
+        }
+        const tokenized = await tokenizer.tokenizeChat(chat) + 2
+        tokens += tokenized
+        if(tokens >= arg.maxContextTokens){
+            tokens -= tokenized
+            break
+        }
+        resultTexts.push(vector[0].substring(14))
+    }
+    console.log(resultTexts)
+    chats.unshift({
+        role: "system",
+        memo: "supaMemory",
+        content: resultTexts.join('\n\n'),
+    })
+    return {
+        tokens,
+        chats
+    }
+
+
+}

src/ts/process/memory/hypamemory.ts

@@ -92,7 +92,7 @@ export class HypaProcesser{
     async addText(texts:string[]) {
 
         for(let i=0;i<texts.length;i++){
-            const itm:memoryVector = await this.forage.getItem(texts[i])
+            const itm:memoryVector = await this.forage.getItem(texts[i] + '|' + this.model)
             if(itm){
                 itm.alreadySaved = true
                 this.vectors.push(itm)
@@ -121,7 +121,7 @@ export class HypaProcesser{
         for(let i=0;i<memoryVectors.length;i++){
             const vec = memoryVectors[i]
             if(!vec.alreadySaved){
-                await this.forage.setItem(texts[i], vec)
+                await this.forage.setItem(texts[i] + '|' + this.model, vec)
             }
         }
 

src/ts/process/memory/termMemory.ts

@@ -1,28 +0,0 @@
-import type { OpenAIChat } from "..";
-import { HypaProcesser } from "./hypamemory";
-
-export async function termMemory(chats:OpenAIChat[]){
-    const processer = new HypaProcesser('nomic')
-    processer.addText(chats.map(chat=>chat.content))
-
-    let scoredResults:{[key:string]:number}
-    for(let i=1;i<5;i++){
-        const chat = chats[chats.length-i]
-        if(!chat?.content){
-            continue
-        }
-        const scoredArray = (await processer.similaritySearchScored(chat.content)).map((result) => {
-            return [result[0],result[1]/i] as [string,number]
-        })
-        for(const scored of scoredArray){
-            if(scoredResults[scored[0]]){
-                scoredResults[scored[0]] += scored[1]
-            }else{
-                scoredResults[scored[0]] = scored[1]
-            }
-        }
-    }
-    const result = Object.entries(scoredResults).sort((a,b)=>a[1]-b[1])
-    return result.map(([content,score])=>(content)).join('\n\n')
-
-}

src/ts/process/transformers.ts

@@ -50,7 +50,8 @@ export const runSummarizer = async (text: string) => {
 }
 
 let extractor:FeatureExtractionPipeline = null
-export const runEmbedding = async (text: string, model:'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5' = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array> => {
+type EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5'
+export const runEmbedding = async (text: string, model:EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array> => {
     await initTransformers()
     if(!extractor){
         extractor = await pipeline('feature-extraction', model);

src/ts/storage/database.ts

@@ -394,6 +394,9 @@ export function setDatabase(data:Database){
     data.instructChatTemplate ??= "chatml"
     data.openrouterProvider ??= ''
     data.useInstructPrompt ??= false
+    data.hanuraiEnable ??= false
+    data.hanuraiSplit ??= true
+    data.hanuraiTokens ??= 1000
 
     changeLanguage(data.language)
     DataBase.set(data)
@@ -642,6 +645,9 @@ export interface Database{
     JinjaTemplate:string
     openrouterProvider:string
     useInstructPrompt:boolean
+    hanuraiTokens:number
+    hanuraiSplit:boolean
+    hanuraiEnable:boolean
 }
 
 export interface customscript{
@@ -847,7 +853,6 @@ export interface botPreset{
     top_a?:number
     openrouterProvider?:string
     useInstructPrompt?:boolean
-
 }