fix: HypaV2 serialization and token calculation (#701)

# PR Checklist
- [x] Have you checked if it works normally in all models? *Ignore this
if it doesn't use models.*
- [x] Have you checked if it works normally in all web, local, and node
hosted versions? If it doesn't, have you blocked it in those versions?
- [x] Have you added type definitions?

# Description
This PR addresses two issues in HypaV2:

1. Fixes JSON serialization of chatMemos in HypaV2Data by converting Set
to string array
2. Corrects token calculation by subtracting tokens from previously
summarized chats

Changes:
- Added SerializableHypaV2Data interface with string[] instead of
Set<string>
- Added conversion functions between HypaV2Data and
SerializableHypaV2Data
- Fixed token calculation by properly accounting for removed chats

Note:
I'm not entirely confident about the token calculation changes. A
thorough review would be appreciated, particularly regarding:
- Whether token subtraction for removed chats is the correct approach
- Potential edge cases in the current implementation 

Please review these changes, especially the token calculation logic, to
ensure it aligns with the intended behavior of the memory system.

src/ts/process/memory/hypav2.ts

@@ -26,6 +26,16 @@ export interface HypaV2Data {
     }[];
 }
 
+// Reuse HypaV2Data and override only chatMemos in mainChunks
+export interface SerializableHypaV2Data extends Omit<HypaV2Data, 'mainChunks'> {
+    mainChunks: {
+        id: number;
+        text: string;
+        chatMemos: string[]; // Override Set<string> with string[]
+        lastChatMemo: string;
+    }[];
+}
+
 async function summary(
     stringlizedChat: string
 ): Promise<{ success: boolean; data: string }> {
@@ -294,6 +304,34 @@ export async function regenerateSummary(
     const targetMainChunk = data.mainChunks[mainChunkIndex];
 
 }
+
+function toSerializableHypaV2Data(data: HypaV2Data): SerializableHypaV2Data {
+    return {
+        ...data,
+        mainChunks: data.mainChunks.map(mainChunk => ({
+            ...mainChunk,
+            chatMemos: Array.from(mainChunk.chatMemos),
+        })),
+    };
+}
+    
+function toHypaV2Data(data: SerializableHypaV2Data): HypaV2Data {
+    // Handle corrupted data due to invalid json serialization
+    data.mainChunks.forEach((mainChunk) => {
+        if (!Array.isArray(mainChunk.chatMemos)) {
+            mainChunk.chatMemos = [];
+        }
+    });
+
+    return {
+        ...data,
+        mainChunks: data.mainChunks.map(mainChunk => ({
+            ...mainChunk,
+            chatMemos: new Set(mainChunk.chatMemos),
+        })),
+    };
+}
+
 export async function hypaMemoryV2(
     chats: OpenAIChat[],
     currentTokens: number,
@@ -305,26 +343,23 @@ export async function hypaMemoryV2(
     currentTokens: number;
     chats: OpenAIChat[];
     error?: string;
-    memory?: HypaV2Data;
+    memory?: SerializableHypaV2Data;
 }> {
     const db = getDatabase();
-
-    if(room.hypaV2Data && isOldHypaV2Data(room.hypaV2Data)){
-        console.log("Old HypaV2 data detected. Converting to new format...");
-        room.hypaV2Data = convertOldToNewHypaV2Data(room.hypaV2Data, chats);
-    }
-
-    const data: HypaV2Data = room.hypaV2Data ?? {
+    let data: HypaV2Data = {
         lastMainChunkID: 0,
         chunks: [],
         mainChunks: [],
     };
-    // JSON s
-    data.mainChunks.forEach(mainChunk => {
-            if (mainChunk.chatMemos && Array.isArray(mainChunk.chatMemos)) {
-                    mainChunk.chatMemos = new Set(mainChunk.chatMemos);
-                }
-    });
+
+    if (room.hypaV2Data) {
+        if (isOldHypaV2Data(room.hypaV2Data)) {
+        console.log("Old HypaV2 data detected. Converting to new format...");
+        data = convertOldToNewHypaV2Data(room.hypaV2Data, chats);
+        } else {
+            data = toHypaV2Data(room.hypaV2Data);
+        }
+    }
 
     // Clean invalid HypaV2 data
     cleanInvalidChunks(chats, data);
@@ -346,6 +381,12 @@ export async function hypaMemoryV2(
         const lastChatIndex = chats.findIndex(chat => chat.memo === lastChatMemo);
         if (lastChatIndex !== -1) {
             idx = lastChatIndex + 1;
+
+            // Subtract tokens of removed chats
+            const removedChats = chats.slice(0, lastChatIndex + 1);
+            for (const chat of removedChats) {
+                currentTokens -= await tokenizer.tokenizeChat(chat);
+            }
         }
     }
     // Starting chat index of new mainChunk to be generated
@@ -590,6 +631,6 @@ export async function hypaMemoryV2(
     return {
         currentTokens: currentTokens,
         chats: unsummarizedChats,
-        memory: data,
+        memory: toSerializableHypaV2Data(data),
     };
 }

src/ts/storage/database.svelte.ts

@@ -1255,7 +1255,7 @@ export interface Chat{
     localLore: loreBook[]
     sdData?:string
     supaMemoryData?:string
-    hypaV2Data?:HypaV2Data
+    hypaV2Data?:SerializableHypaV2Data
     lastMemory?:string
     suggestMessages?:string[]
     isStreaming?:boolean
@@ -1611,7 +1611,7 @@ import { encode as encodeMsgpack, decode as decodeMsgpack } from "msgpackr";
 import * as fflate from "fflate";
 import type { OnnxModelFiles } from '../process/transformers';
 import type { RisuModule } from '../process/modules';
-import type { HypaV2Data } from '../process/memory/hypav2';
+import type { SerializableHypaV2Data } from '../process/memory/hypav2';
 import { decodeRPack, encodeRPack } from '../rpack/rpack_bg';
 import { DBState, selectedCharID } from '../stores.svelte';
 import { LLMFlags, LLMFormat } from '../model/modellist';