From cd65eaf4008f610f0dce9a7f196d373744213278 Mon Sep 17 00:00:00 2001
From: Bo26fhmC5M <88071760+Bo26fhmC5M@users.noreply.github.com>
Date: Sun, 19 Jan 2025 18:05:11 +0900
Subject: [PATCH] feat: add process regex script option in HypaV3 modal

---
 src/lib/Others/HypaV3Modal.svelte             | 289 +++++++++++-------
 src/lib/Setting/Pages/OtherBotSettings.svelte |   5 +-
 src/ts/storage/database.svelte.ts             |   4 +-
 3 files changed, 188 insertions(+), 110 deletions(-)

diff --git a/src/lib/Others/HypaV3Modal.svelte b/src/lib/Others/HypaV3Modal.svelte
index 686707a2..8a762704 100644
--- a/src/lib/Others/HypaV3Modal.svelte
+++ b/src/lib/Others/HypaV3Modal.svelte
@@ -1,4 +1,5 @@
 <script lang="ts">
+  import { tick } from "svelte";
   import {
     SettingsIcon,
     Trash2Icon,
@@ -9,7 +10,6 @@
     ScissorsLineDashed,
     CheckIcon,
   } from "lucide-svelte";
-  import { tick } from "svelte";
   import TextAreaInput from "../../lib/UI/GUI/TextAreaInput.svelte";
   import {
     alertConfirm,
@@ -23,8 +23,13 @@
     settingsOpen,
     SettingsMenuIndex,
   } from "../../ts/stores.svelte";
-  import { summarize } from "../../ts/process/memory/hypav3";
   import { type OpenAIChat } from "../../ts/process/index.svelte";
+  import {
+    processScript,
+    processScriptFull,
+    risuChatParser,
+  } from "../../ts/process/scripts";
+  import { summarize } from "../../ts/process/memory/hypav3";
   import { type Message } from "../../ts/storage/database.svelte";
   import { translateHTML } from "../../ts/translator/translator";
 
@@ -80,33 +85,6 @@
     );
   }
 
-  function getFirstMessage(): string | null {
-    const char = DBState.db.characters[$selectedCharID];
-    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
-
-    return chat.fmIndex === -1
-      ? char.firstMessage
-      : char.alternateGreetings?.[chat.fmIndex]
-        ? char.alternateGreetings[chat.fmIndex]
-        : null;
-  }
-
-  function getMessageFromChatMemo(chatMemo: string | null): Message | null {
-    const char = DBState.db.characters[$selectedCharID];
-    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
-    const firstMessage = getFirstMessage();
-
-    if (!firstMessage) {
-      return null;
-    }
-
-    if (chatMemo == null) {
-      return { role: "char", data: firstMessage };
-    }
-
-    return chat.message.find((m) => m.chatId === chatMemo) || null;
-  }
-
   async function toggleTranslate(
     summaryIndex: number,
     regenerate?: boolean
@@ -167,16 +145,19 @@
 
     try {
       const summary = hypaV3DataState.summaries[summaryIndex];
-      const toSummarize: OpenAIChat[] = summary.chatMemos.map((chatMemo) => {
-        const message = getMessageFromChatMemo(chatMemo);
+      const toSummarize: OpenAIChat[] = await Promise.all(
+        summary.chatMemos.map(async (chatMemo) => {
+          // Processed message
+          const message = await getProcessedMessageFromChatMemo(chatMemo);
 
-        return {
-          role: (message.role === "char"
-            ? "assistant"
-            : message.role) as OpenAIChat["role"],
-          content: message.data,
-        };
-      });
+          return {
+            role: (message.role === "char"
+              ? "assistant"
+              : message.role) as OpenAIChat["role"],
+            content: message.data,
+          };
+        })
+      );
 
       const summarizeResult = await summarize(toSummarize);
 
@@ -226,6 +207,65 @@
     summaryUIState.isRerolledTranslating = false;
   }
 
+  async function getProcessedMessageFromChatMemo(
+    chatMemo: string | null
+  ): Promise<Message | null> {
+    const unprocessed = getMessageFromChatMemo(chatMemo);
+
+    if (!unprocessed) {
+      return null;
+    }
+
+    return DBState.db.hypaV3Settings.processRegexScript
+      ? await processRegexScript(unprocessed)
+      : unprocessed;
+  }
+
+  function getMessageFromChatMemo(chatMemo: string | null): Message | null {
+    const char = DBState.db.characters[$selectedCharID];
+    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
+
+    if (chatMemo == null) {
+      const firstMessage = getFirstMessage();
+
+      return firstMessage ? { role: "char", data: firstMessage } : null;
+    }
+
+    return chat.message.find((m) => m.chatId === chatMemo) || null;
+  }
+
+  function getFirstMessage(): string | null {
+    const char = DBState.db.characters[$selectedCharID];
+    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
+
+    return chat.fmIndex === -1
+      ? char.firstMessage
+      : char.alternateGreetings?.[chat.fmIndex]
+        ? char.alternateGreetings[chat.fmIndex]
+        : null;
+  }
+
+  async function processRegexScript(msg: Message): Promise<Message> {
+    const char = DBState.db.characters[$selectedCharID];
+    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
+    const newData: string = (
+      await processScriptFull(
+        char,
+        risuChatParser(msg.data, { chara: char, role: msg.role }),
+        "editprocess",
+        -1,
+        {
+          chatRole: msg.role,
+        }
+      )
+    ).data;
+
+    return {
+      ...msg,
+      data: newData,
+    };
+  }
+
   function isMessageExpanded(
     summaryIndex: number,
     chatMemo: string | null
@@ -265,7 +305,8 @@
       return;
     }
 
-    const message = getMessageFromChatMemo(
+    // Processed message
+    const message = await getProcessedMessageFromChatMemo(
       expandedMessageUIState.selectedChatMemo
     );
 
@@ -292,15 +333,33 @@
     expandedMessageUIState.isTranslating = false;
   }
 
-  function getNextMessageToSummarize(): Message | null {
-    const char = DBState.db.characters[$selectedCharID];
-    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
-    const firstMessage = getFirstMessage();
+  async function translate(
+    text: string,
+    regenerate?: boolean
+  ): Promise<string> {
+    try {
+      return await translateHTML(text, false, "", -1, regenerate);
+    } catch (error) {
+      return `Translation failed: ${error}`;
+    }
+  }
 
-    if (!firstMessage) {
+  async function getProcessedNextSummarizationTarget(): Promise<Message | null> {
+    const unprocessed = getNextSummarizationTarget();
+
+    if (!unprocessed) {
       return null;
     }
 
+    return DBState.db.hypaV3Settings.processRegexScript
+      ? await processRegexScript(unprocessed)
+      : unprocessed;
+  }
+
+  function getNextSummarizationTarget(): Message | null {
+    const char = DBState.db.characters[$selectedCharID];
+    const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
+
     // Summaries exist
     if (hypaV3DataState.summaries.length > 0) {
       const lastSummary = hypaV3DataState.summaries.at(-1);
@@ -317,8 +376,11 @@
       }
     }
 
-    // No summaries
-    if (firstMessage?.trim() === "") {
+    const firstMessage = getFirstMessage();
+
+    // When no summaries exist OR couldn't find last connected message
+    // Check if first message is available
+    if (!firstMessage || firstMessage.trim() === "") {
       if (chat.message.length > 0) {
         return chat.message[0];
       }
@@ -327,25 +389,14 @@
     }
 
     // will summarize first message
-    return { role: "char", chatId: "first message", data: firstMessage };
-  }
-
-  async function translate(
-    text: string,
-    regenerate?: boolean
-  ): Promise<string> {
-    try {
-      return await translateHTML(text, false, "", -1, regenerate);
-    } catch (error) {
-      return `Translation failed: ${error}`;
-    }
+    return { role: "char", chatId: "first", data: firstMessage };
   }
 
   function isHypaV2ConversionPossible(): boolean {
     const char = DBState.db.characters[$selectedCharID];
     const chat = char.chats[DBState.db.characters[$selectedCharID].chatPage];
 
-    return chat.hypaV3Data?.summaries?.length === 0 && chat.hypaV2Data !== null;
+    return chat.hypaV3Data?.summaries?.length === 0 && chat.hypaV2Data != null;
   }
 
   function convertHypaV2ToV3(): { success: boolean; error?: string } {
@@ -559,30 +610,34 @@
         {#if hypaV3DataState.summaries.length === 0}
           <!-- Conversion Section -->
           {#if isHypaV2ConversionPossible()}
-            <div class="mt-4 flex flex-col items-center gap-2">
-              <span class="text-textcolor2 text-center p-4"
-                >No summaries yet, but you can convert HypaV2 data to V3.</span
-              >
-              <button
-                class="px-4 py-2 bg-zinc-800 text-zinc-200 rounded-md hover:bg-zinc-700 transition-colors"
-                onclick={async () => {
-                  const conversionResult = convertHypaV2ToV3();
+            <div
+              class="flex flex-col p-4 rounded-lg border border-zinc-700 bg-zinc-800/50"
+            >
+              <div class="mt-4 flex flex-col items-center gap-2">
+                <span class="text-textcolor2 text-center p-4"
+                  >No summaries yet, but you may convert HypaV2 data to V3.</span
+                >
+                <button
+                  class="px-4 py-2 bg-zinc-800 text-zinc-200 rounded-md hover:bg-zinc-700 transition-colors"
+                  onclick={async () => {
+                    const conversionResult = convertHypaV2ToV3();
 
-                  if (conversionResult.success) {
-                    await alertNormalWait(
-                      "Successfully converted HypaV2 data to V3"
-                    );
-                  } else {
-                    await alertNormalWait(
-                      `Failed to convert HypaV2 data to V3: ${conversionResult.error}`
-                    );
-                  }
+                    if (conversionResult.success) {
+                      await alertNormalWait(
+                        "Successfully converted HypaV2 data to V3"
+                      );
+                    } else {
+                      await alertNormalWait(
+                        `Failed to convert HypaV2 data to V3: ${conversionResult.error}`
+                      );
+                    }
 
-                  showHypaV3Alert();
-                }}
-              >
-                Convert to V3
-              </button>
+                    showHypaV3Alert();
+                  }}
+                >
+                  Convert to V3
+                </button>
+              </div>
             </div>
           {:else}
             <span class="text-textcolor2 text-center p-4">No summaries yet</span
@@ -779,26 +834,32 @@
                   {/each}
                 </div>
 
-                <!-- Selected Message Content (if selected) -->
+                <!-- Selected Message (if selected) -->
                 {#if expandedMessageUIState?.summaryIndex === i}
-                  {@const message = getMessageFromChatMemo(
-                    expandedMessageUIState.selectedChatMemo
-                  )}
                   <div class="mt-4">
-                    {#if message}
-                      <!-- Role -->
-                      <div class="text-sm text-textcolor2 mb-2 block">
-                        {message.role}'s Message
+                    <!-- Processed Message -->
+                    {#await getProcessedMessageFromChatMemo(expandedMessageUIState.selectedChatMemo) then expandedMessage}
+                      {#if expandedMessage}
+                        <!-- Role -->
+                        <div class="text-sm text-textcolor2 mb-2 block">
+                          {expandedMessage.role}'s Message
+                        </div>
+                        <!-- Content -->
+                        <div
+                          class="p-2 max-h-48 overflow-y-auto bg-zinc-800 rounded-md whitespace-pre-wrap"
+                        >
+                          {expandedMessage.data}
+                        </div>
+                      {:else}
+                        <div class="text-sm text-red-500">
+                          Message not found
+                        </div>
+                      {/if}
+                    {:catch error}
+                      <div class="text-sm text-red-500 mb-2">
+                        Error loading expanded message: {error.message}
                       </div>
-                      <!-- Content -->
-                      <div
-                        class="p-2 max-h-48 overflow-y-auto bg-zinc-800 rounded-md whitespace-pre-wrap"
-                      >
-                        {message.data}
-                      </div>
-                    {:else}
-                      <div class="text-sm text-red-500">Message not found</div>
-                    {/if}
+                    {/await}
 
                     <!-- Message Translation -->
                     {#if expandedMessageUIState.translation}
@@ -822,15 +883,18 @@
           {/if}
         {/each}
 
-        <!-- Next message to be summarized -->
-        {#if true}
-          {@const nextMessage = getNextMessageToSummarize()}
+        <!-- Next Summarization Target -->
+        {#await getProcessedNextSummarizationTarget() then nextMessage}
           {#if nextMessage}
+            {@const chatId =
+              nextMessage.chatId === "first"
+                ? "First Message"
+                : nextMessage.chatId == null
+                  ? "No Message ID"
+                  : nextMessage.chatId}
             <div class="mt-4">
               <span class="text-sm text-textcolor2 mb-2 block">
-                HypaV3 will summarize [{nextMessage.chatId
-                  ? nextMessage.chatId
-                  : "no message id"}]
+                HypaV3 will summarize [{chatId}]
               </span>
               <div
                 class="p-2 max-h-48 overflow-y-auto bg-zinc-800 rounded-md whitespace-pre-wrap"
@@ -838,11 +902,20 @@
                 {nextMessage.data}
               </div>
             </div>
-          {:else if !getFirstMessage()}
-            <div class="text-sm text-red-500">
-              WARN: Selected first message is null
-            </div>
           {/if}
+        {:catch error}
+          <div class="text-sm text-red-500 mb-2">
+            Error loading next message: {error.message}
+          </div>
+        {/await}
+
+        <!-- No First Message -->
+        {#if !getFirstMessage()}
+          <div class="mt-4">
+            <div class="text-sm text-red-500 mb-2">
+              WARN: Selected first message is empty
+            </div>
+          </div>
         {/if}
       </div>
     </div>
diff --git a/src/lib/Setting/Pages/OtherBotSettings.svelte b/src/lib/Setting/Pages/OtherBotSettings.svelte
index 8259f29b..f8a59880 100644
--- a/src/lib/Setting/Pages/OtherBotSettings.svelte
+++ b/src/lib/Setting/Pages/OtherBotSettings.svelte
@@ -519,7 +519,10 @@
             </div>
             <div class="flex mb-2">
                 <Check name="Preserve Orphaned Memory" bind:check={DBState.db.hypaV3Settings.preserveOrphanedMemory} />
-            </div>        
+            </div>
+            <div class="flex mb-2">
+                <Check name="Process Regex Script (Modify Request Data)" bind:check={DBState.db.hypaV3Settings.processRegexScript} />
+            </div> 
         {:else if (DBState.db.supaModelType !== 'none' && DBState.db.hypav2 === false && DBState.db.hypaV3 === false)}
             <span class="mb-2 text-textcolor2 text-sm text-wrap break-words max-w-full">{language.supaDesc}</span>
             <span class="text-textcolor mt-4">{language.SuperMemory} {language.model}</span>
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index c2a0e149..9f9facc8 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -477,7 +477,8 @@ export function setDatabase(data:Database){
         recentMemoryRatio: data.hypaV3Settings?.recentMemoryRatio ?? 0.4,
         similarMemoryRatio: data.hypaV3Settings?.similarMemoryRatio ?? 0.4,
         enableSimilarityCorrection: data.hypaV3Settings?.enableSimilarityCorrection ?? false,
-        preserveOrphanedMemory: data.hypaV3Settings?.preserveOrphanedMemory ?? false
+        preserveOrphanedMemory: data.hypaV3Settings?.preserveOrphanedMemory ?? false,
+        processRegexScript: data.hypaV3Settings?.processRegexScript ?? false
     }
     changeLanguage(data.language)
     setDatabaseLite(data)
@@ -890,6 +891,7 @@ export interface Database{
         similarMemoryRatio: number
         enableSimilarityCorrection: boolean
         preserveOrphanedMemory: boolean
+        processRegexScript: boolean
     }
 }