From 2b92582df4f015214cabd4b02bf75e991514d267 Mon Sep 17 00:00:00 2001
From: Bo26fhmC5M <88071760+Bo26fhmC5M@users.noreply.github.com>
Date: Thu, 2 Jan 2025 18:47:16 +0900
Subject: [PATCH] fix: resolve chatMemos serialization issue in HypaV2Data

---
 src/ts/process/memory/hypav2.ts   | 65 ++++++++++++++++++++++++-------
 src/ts/storage/database.svelte.ts |  4 +-
 2 files changed, 52 insertions(+), 17 deletions(-)

diff --git a/src/ts/process/memory/hypav2.ts b/src/ts/process/memory/hypav2.ts
index d51074f8..917387bd 100644
--- a/src/ts/process/memory/hypav2.ts
+++ b/src/ts/process/memory/hypav2.ts
@@ -26,6 +26,16 @@ export interface HypaV2Data {
     }[];
 }
 
+// Reuse HypaV2Data and override only chatMemos in mainChunks
+export interface SerializableHypaV2Data extends Omit<HypaV2Data, 'mainChunks'> {
+    mainChunks: {
+        id: number;
+        text: string;
+        chatMemos: string[]; // Override Set<string> with string[]
+        lastChatMemo: string;
+    }[];
+}
+
 async function summary(
     stringlizedChat: string
 ): Promise<{ success: boolean; data: string }> {
@@ -294,6 +304,34 @@ export async function regenerateSummary(
     const targetMainChunk = data.mainChunks[mainChunkIndex];
 
 }
+
+function toSerializableHypaV2Data(data: HypaV2Data): SerializableHypaV2Data {
+    return {
+        ...data,
+        mainChunks: data.mainChunks.map(mainChunk => ({
+            ...mainChunk,
+            chatMemos: Array.from(mainChunk.chatMemos),
+        })),
+    };
+}
+    
+function toHypaV2Data(data: SerializableHypaV2Data): HypaV2Data {
+    // Handle corrupted data due to invalid json serialization
+    data.mainChunks.forEach((mainChunk) => {
+        if (!Array.isArray(mainChunk.chatMemos)) {
+            mainChunk.chatMemos = [];
+        }
+    });
+
+    return {
+        ...data,
+        mainChunks: data.mainChunks.map(mainChunk => ({
+            ...mainChunk,
+            chatMemos: new Set(mainChunk.chatMemos),
+        })),
+    };
+}
+
 export async function hypaMemoryV2(
     chats: OpenAIChat[],
     currentTokens: number,
@@ -305,26 +343,23 @@ export async function hypaMemoryV2(
     currentTokens: number;
     chats: OpenAIChat[];
     error?: string;
-    memory?: HypaV2Data;
+    memory?: SerializableHypaV2Data;
 }> {
     const db = getDatabase();
-
-    if(room.hypaV2Data && isOldHypaV2Data(room.hypaV2Data)){
-        console.log("Old HypaV2 data detected. Converting to new format...");
-        room.hypaV2Data = convertOldToNewHypaV2Data(room.hypaV2Data, chats);
-    }
-
-    const data: HypaV2Data = room.hypaV2Data ?? {
+    let data: HypaV2Data = {
         lastMainChunkID: 0,
         chunks: [],
         mainChunks: [],
     };
-    // JSON s
-    data.mainChunks.forEach(mainChunk => {
-            if (mainChunk.chatMemos && Array.isArray(mainChunk.chatMemos)) {
-                    mainChunk.chatMemos = new Set(mainChunk.chatMemos);
-                }
-    });
+
+    if (room.hypaV2Data) {
+        if (isOldHypaV2Data(room.hypaV2Data)) {
+        console.log("Old HypaV2 data detected. Converting to new format...");
+        data = convertOldToNewHypaV2Data(room.hypaV2Data, chats);
+        } else {
+            data = toHypaV2Data(room.hypaV2Data);
+        }
+    }
 
     // Clean invalid HypaV2 data
     cleanInvalidChunks(chats, data);
@@ -590,6 +625,6 @@ export async function hypaMemoryV2(
     return {
         currentTokens: currentTokens,
         chats: unsummarizedChats,
-        memory: data,
+        memory: toSerializableHypaV2Data(data),
     };
 }
\ No newline at end of file
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index 978c0100..8f7665ff 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -1255,7 +1255,7 @@ export interface Chat{
     localLore: loreBook[]
     sdData?:string
     supaMemoryData?:string
-    hypaV2Data?:HypaV2Data
+    hypaV2Data?:SerializableHypaV2Data
     lastMemory?:string
     suggestMessages?:string[]
     isStreaming?:boolean
@@ -1611,7 +1611,7 @@ import { encode as encodeMsgpack, decode as decodeMsgpack } from "msgpackr";
 import * as fflate from "fflate";
 import type { OnnxModelFiles } from '../process/transformers';
 import type { RisuModule } from '../process/modules';
-import type { HypaV2Data } from '../process/memory/hypav2';
+import type { SerializableHypaV2Data } from '../process/memory/hypav2';
 import { decodeRPack, encodeRPack } from '../rpack/rpack_bg';
 import { DBState, selectedCharID } from '../stores.svelte';
 import { LLMFlags, LLMFormat } from '../model/modellist';