From 297568e704742279c88d815d4c4f000e79bb1b0f Mon Sep 17 00:00:00 2001
From: kwaroran <kwaworan@gmail.com>
Date: Mon, 17 Jul 2023 16:30:55 +0900
Subject: [PATCH] [feat] new fast parser

---
 package.json                             |   2 +
 pnpm-lock.yaml                           |  16 +++
 src/lib/ChatScreens/BackgroundDom.svelte |   4 +-
 src/styles.css                           |   4 +
 src/ts/parser.ts                         | 120 +++++++++++++++++++----
 5 files changed, 126 insertions(+), 20 deletions(-)

diff --git a/package.json b/package.json
index cd0a0051..ebdc23df 100644
--- a/package.json
+++ b/package.json
@@ -18,6 +18,7 @@
     "@dqbd/tiktoken": "^1.0.4",
     "@mlc-ai/web-tokenizers": "^0.1.0",
     "@tauri-apps/api": "1.4.0",
+    "@types/marked": "^5.0.1",
     "blueimp-md5": "^2.19.0",
     "body-parser": "^1.20.2",
     "buffer": "^6.0.3",
@@ -33,6 +34,7 @@
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
     "lucide-svelte": "^0.260.0",
+    "marked": "^5.1.1",
     "ml-distance": "^4.0.1",
     "mobile-drag-drop": "3.0.0-rc.0",
     "msgpackr": "^1.9.5",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 21c766e1..d63ae1cf 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -17,6 +17,9 @@ dependencies:
   '@tauri-apps/api':
     specifier: 1.4.0
     version: 1.4.0
+  '@types/marked':
+    specifier: ^5.0.1
+    version: 5.0.1
   blueimp-md5:
     specifier: ^2.19.0
     version: 2.19.0
@@ -62,6 +65,9 @@ dependencies:
   lucide-svelte:
     specifier: ^0.260.0
     version: 0.260.0(svelte@3.58.0)
+  marked:
+    specifier: ^5.1.1
+    version: 5.1.1
   ml-distance:
     specifier: ^4.0.1
     version: 4.0.1
@@ -805,6 +811,10 @@ packages:
     resolution: {integrity: sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==}
     dev: true
 
+  /@types/marked@5.0.1:
+    resolution: {integrity: sha512-Y3pAUzHKh605fN6fvASsz5FDSWbZcs/65Q6xYRmnIP9ZIYz27T4IOmXfH9gWJV1dpi7f1e7z7nBGUTx/a0ptpA==}
+    dev: false
+
   /@types/node@18.15.11:
     resolution: {integrity: sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==}
 
@@ -1922,6 +1932,12 @@ packages:
       '@jridgewell/sourcemap-codec': 1.4.14
     dev: true
 
+  /marked@5.1.1:
+    resolution: {integrity: sha512-bTmmGdEINWmOMDjnPWDxGPQ4qkDLeYorpYbEtFOXzOruTwUE671q4Guiuchn4N8h/v6NGd7916kXsm3Iz4iUSg==}
+    engines: {node: '>= 18'}
+    hasBin: true
+    dev: false
+
   /media-typer@0.3.0:
     resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
     engines: {node: '>= 0.6'}
diff --git a/src/lib/ChatScreens/BackgroundDom.svelte b/src/lib/ChatScreens/BackgroundDom.svelte
index 999511a3..d073ff4c 100644
--- a/src/lib/ChatScreens/BackgroundDom.svelte
+++ b/src/lib/ChatScreens/BackgroundDom.svelte
@@ -1,5 +1,5 @@
 <script lang="ts">
-    import { ParseMarkdown } from "src/ts/parser";
+    import { ParseMarkdown, risuChatParser } from "src/ts/parser";
     import { DataBase, type Database, type character, type groupChat } from "src/ts/storage/database";
     import { selectedCharID } from "src/ts/stores";
     import { onDestroy } from "svelte";
@@ -43,7 +43,7 @@
 
 {#if backgroundHTML}
     <div class="absolute top-0 left-0 w-full h-full">
-        {#await ParseMarkdown(backgroundHTML, currentChar, 'back') then md} 
+        {#await ParseMarkdown(risuChatParser(backgroundHTML, {chara:currentChar}), currentChar, 'back') then md} 
             {@html md}
         {/await}
     </div>
diff --git a/src/styles.css b/src/styles.css
index 3398f55e..d94d53d3 100644
--- a/src/styles.css
+++ b/src/styles.css
@@ -45,6 +45,10 @@ html, body{
   color: var(--FontColorItalicBold);
 }
 
+.chattext em strong{
+  color: var(--FontColorItalicBold);
+}
+
 ::-webkit-scrollbar {
   width: 5px;
   height: 5px;
diff --git a/src/ts/parser.ts b/src/ts/parser.ts
index b20fd357..aa5e86f1 100644
--- a/src/ts/parser.ts
+++ b/src/ts/parser.ts
@@ -1,5 +1,7 @@
 import DOMPurify from 'isomorphic-dompurify';
 import showdown from 'showdown';
+import { Marked } from 'marked';
+
 import { DataBase, type Database, type character, type groupChat } from './storage/database';
 import { getFileSrc } from './storage/globalApi';
 import { processScript, processScriptFull } from './process/scripts';
@@ -7,13 +9,23 @@ import { get } from 'svelte/store';
 import css from '@adobe/css-tools'
 import { selectedCharID } from './stores';
 import { calcString } from './process/infunctions';
+import { findCharacterbyId } from './util';
 
-const convertor = new showdown.Converter({
+const convertora = new showdown.Converter({
     simpleLineBreaks: true,
     strikethrough: true,
     tables: true
 })
 
+const mconverted = new Marked({
+    gfm: true,
+    breaks: true,
+    silent: true,
+    tokenizer: {
+
+    }
+})
+
 const safeConvertor = new showdown.Converter({
     simpleLineBreaks: true,
     strikethrough: true,
@@ -90,7 +102,7 @@ export async function ParseMarkdown(data:string, char:(character | groupChat) =
     if(firstParsed !== data && char && char.type !== 'group'){
         data = await parseAdditionalAssets(data, char, mode)
     }
-    return DOMPurify.sanitize(convertor.makeHtml(data), {
+    return DOMPurify.sanitize(mconverted.parse(data), {
         ADD_TAGS: ["iframe"],
         ADD_ATTR: ["allow", "allowfullscreen", "frameborder", "scrolling"],
     })
@@ -242,15 +254,30 @@ function wppParser(data:string){
    return characterDetails;
 }
 
+
 const rgx = /(?:{{|<)(.+?)(?:}}|>)/gm
 export function risuChatParser(da:string, arg:{
     chatID?:number
     db?:Database
-    chara?:string|character
+    chara?:string|character|groupChat
 } = {}):string{
     const chatID = arg.chatID ?? -1
     const db = arg.db ?? get(DataBase)
-    return da.replace(rgx, (v, p1:string) => {
+    const aChara = arg.chara
+    let chara:character|string = null
+
+    if(aChara){
+        if(typeof(aChara) !== 'string' && aChara.type === 'group'){
+            const gc = findCharacterbyId(aChara.chats[aChara.chatPage].message.at(-1).saying ?? '')
+            if(gc.name !== 'Unknown Character'){
+                chara = gc
+            }
+        }
+        else{
+            chara = aChara
+        }
+    }
+    const matcher = (p1:string) => {
         const lowerCased = p1.toLocaleLowerCase()
         switch(lowerCased){
             case 'previous_char_chat':{
@@ -285,7 +312,6 @@ export function risuChatParser(da:string, arg:{
             }
             case 'char':
             case 'bot':{
-                const chara = arg.chara
                 if(chara){
                     if(typeof(chara) === 'string'){
                         return chara
@@ -303,21 +329,21 @@ export function risuChatParser(da:string, arg:{
             }
             case 'personality':
             case 'char_persona':{
-                const argChara = arg.chara
-                const chara = (argChara && typeof(argChara) !== 'string') ? argChara : (db.characters[get(selectedCharID)])
-                if(chara.type === 'group'){
+                const argChara = chara
+                const achara = (argChara && typeof(argChara) !== 'string') ? argChara : (db.characters[get(selectedCharID)])
+                if(achara.type === 'group'){
                     return ""
                 }
-                return chara.personality
+                return achara.personality
             }
             case 'persona':
             case 'user_persona':{
-                const argChara = arg.chara
-                const chara = (argChara && typeof(argChara) !== 'string') ? argChara : (db.characters[get(selectedCharID)])
-                if(chara.type === 'group'){
+                const argChara = chara
+                const achara = (argChara && typeof(argChara) !== 'string') ? argChara : (db.characters[get(selectedCharID)])
+                if(achara.type === 'group'){
                     return ""
                 }
-                return chara.personality
+                return achara.personality
             }
             case 'ujb':
             case 'global_note':{
@@ -350,7 +376,7 @@ export function risuChatParser(da:string, arg:{
                     return `<button style="padding" x-risu-prompt="${arra[2]}">${arra[1]}</button>`
                 }
                 case 'risu':{
-                    return `<img src="/logo2.png" />`
+                    return `<img src="/logo2.png" style="height:${v || 45}px;width:${v || 45}px" />`
                 }
             }
         }
@@ -365,11 +391,69 @@ export function risuChatParser(da:string, arg:{
                 return arr[randomIndex]
             }
         }
-        return v
-    })
+        return null
+    }
+    
+    let pointer = 0;
+    let nested:string[] = [""]
+    let pf = performance.now()
+    let v = new Uint8Array(255)
+    while(pointer < da.length){
+        switch(da[pointer]){
+            case '{':{
+                if(da[pointer + 1] !== '{'){
+                    nested[0] += da[pointer]
+                    break
+                }
+                pointer++
+                nested.unshift('')
+                v[nested.length] = 1
+                break
+            }
+            case '<':{
+                nested.unshift('')
+                v[nested.length] = 2
+                break
+            }
+            case '}':{
+                if(da[pointer + 1] !== '}' || nested.length === 1 || v[nested.length] !== 1){
+                    nested[0] += da[pointer]
+                    break
+                }
+                pointer++
+                const dat = nested.shift()
+                const mc = matcher(dat)
+                nested[0] += mc ?? `{{${dat}}}`
+                break
+            }
+            case '>':{
+                if(nested.length === 1 || v[nested.length] !== 2){
+                    break
+                }
+                const dat = nested.shift()
+                const mc = matcher(dat)
+                nested[0] += mc ?? `<${dat}>`
+                break
+            }
+            default:{
+                nested[0] += da[pointer]
+                break
+            }
+        }
+        pointer++
+    }
+    if(nested.length === 1){
+        return nested[0]
+    }
+    let result = ''
+    while(nested.length > 1){
+        let dat = (v[nested.length] === 1) ? '{{' : "<"
+        dat += nested.shift()
+        result = dat + result
+    }
+    return nested[0] + result
 }
 
-
 function getVarChat(targetIndex = -1){
     const db = get(DataBase)
     const selchar = db.characters[get(selectedCharID)]
@@ -430,7 +514,7 @@ function getVarChat(targetIndex = -1){
                 break
             }
             case "min":{
-                if(parseInt(vars[rule.key]) > parseInt(rule.arg)){
+                if(parseInt(vars[rule.key]) < parseInt(rule.arg)){
                     vars[rule.key] = rule.arg
                 }
                 break