diff --git a/package.json b/package.json index 12376a35..826f974c 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "@smithy/protocol-http": "^3.0.12", "@smithy/signature-v4": "^2.0.19", "@tauri-apps/api": "1.5.3", + "@types/markdown-it": "^14.1.1", "@xenova/transformers": "^2.17.1", "blueimp-md5": "^2.19.0", "body-parser": "^1.20.2", @@ -50,6 +51,7 @@ "localforage": "^1.10.0", "lodash": "^4.17.21", "lucide-svelte": "^0.292.0", + "markdown-it": "^14.1.0", "marked": "^5.1.2", "ml-distance": "^4.0.1", "mobile-drag-drop": "3.0.0-rc.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0cdf5c07..29001994 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: '@tauri-apps/api': specifier: 1.5.3 version: 1.5.3 + '@types/markdown-it': + specifier: ^14.1.1 + version: 14.1.1 '@xenova/transformers': specifier: ^2.17.1 version: 2.17.1 @@ -110,6 +113,9 @@ importers: lucide-svelte: specifier: ^0.292.0 version: 0.292.0(svelte@4.2.8) + markdown-it: + specifier: ^14.1.0 + version: 14.1.0 marked: specifier: ^5.1.2 version: 5.1.2 @@ -1028,6 +1034,9 @@ packages: '@types/libsodium-wrappers@0.7.13': resolution: {integrity: sha512-KeAKtlObirLJk/na6jHBFEdTDjDfFS6Vcr0eG2FjiHKn3Nw8axJFfIu0Y9TpwaauRldQBj/pZm/MHtK76r6OWg==} + '@types/linkify-it@5.0.0': + resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==} + '@types/lodash.isequal@4.5.8': resolution: {integrity: sha512-uput6pg4E/tj2LGxCZo9+y27JNyB2OZuuI/T5F+ylVDYuqICLG2/ktjxx0v6GvVntAf8TvEzeQLcV0ffRirXuA==} @@ -1037,9 +1046,15 @@ packages: '@types/long@4.0.2': resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} + '@types/markdown-it@14.1.1': + resolution: {integrity: sha512-4NpsnpYl2Gt1ljyBGrKMxFYAYvpqbnnkgP/i/g+NLpjEUa3obn1XJCur9YbEXKDAkaXqsR1LbDnGEJ0MmKFxfg==} + '@types/marked@5.0.2': resolution: {integrity: sha512-OucS4KMHhFzhz27KxmWg7J+kIYqyqoW5kdIEI319hqARQQUTqhao3M/F+uFnDXD0Rg72iDDZxZNxq5gvctmLlg==} + '@types/mdurl@2.0.0': + resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==} + '@types/minimist@1.2.5': resolution: {integrity: sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==} @@ -1174,6 +1189,9 @@ packages: arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + aria-query@5.3.0: resolution: {integrity: sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==} @@ -2264,6 +2282,9 @@ packages: lines-and-columns@1.2.4: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + linkify-it@5.0.0: + resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} + load-json-file@4.0.0: resolution: {integrity: sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==} engines: {node: '>=4'} @@ -2332,6 +2353,10 @@ packages: resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==} engines: {node: '>=8'} + markdown-it@14.1.0: + resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==} + hasBin: true + marked@5.1.2: resolution: {integrity: sha512-ahRPGXJpjMjwSOlBoTMZAK7ATXkli5qCPxZ21TG44rx1KEo44bii4ekgTDQPNRQ4Kh7JMb9Ub1PVk1NxRSsorg==} engines: {node: '>= 16'} @@ -2340,6 +2365,9 @@ packages: mdn-data@2.0.30: resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==} + mdurl@2.0.0: + resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} + media-typer@0.3.0: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} @@ -2883,6 +2911,10 @@ packages: pump@3.0.0: resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + punycode.js@2.3.1: + resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} + engines: {node: '>=6'} + punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -3461,6 +3493,9 @@ packages: engines: {node: '>=14.17'} hasBin: true + uc.micro@2.1.0: + resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} + uglify-js@3.17.4: resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==} engines: {node: '>=0.8.0'} @@ -4475,6 +4510,8 @@ snapshots: '@types/libsodium-wrappers@0.7.13': {} + '@types/linkify-it@5.0.0': {} + '@types/lodash.isequal@4.5.8': dependencies: '@types/lodash': 4.14.202 @@ -4483,8 +4520,15 @@ snapshots: '@types/long@4.0.2': {} + '@types/markdown-it@14.1.1': + dependencies: + '@types/linkify-it': 5.0.0 + '@types/mdurl': 2.0.0 + '@types/marked@5.0.2': {} + '@types/mdurl@2.0.0': {} + '@types/minimist@1.2.5': {} '@types/node@18.19.7': @@ -4612,6 +4656,8 @@ snapshots: arg@5.0.2: {} + argparse@2.0.1: {} + aria-query@5.3.0: dependencies: dequal: 2.0.3 @@ -5822,6 +5868,10 @@ snapshots: lines-and-columns@1.2.4: {} + linkify-it@5.0.0: + dependencies: + uc.micro: 2.1.0 + load-json-file@4.0.0: dependencies: graceful-fs: 4.2.11 @@ -5881,10 +5931,21 @@ snapshots: map-obj@4.3.0: {} + markdown-it@14.1.0: + dependencies: + argparse: 2.0.1 + entities: 4.5.0 + linkify-it: 5.0.0 + mdurl: 2.0.0 + punycode.js: 2.3.1 + uc.micro: 2.1.0 + marked@5.1.2: {} mdn-data@2.0.30: {} + mdurl@2.0.0: {} + media-typer@0.3.0: {} meow@8.1.2: @@ -6453,6 +6514,8 @@ snapshots: end-of-stream: 1.4.4 once: 1.4.0 + punycode.js@2.3.1: {} + punycode@2.3.1: {} q@1.5.1: {} @@ -7112,6 +7175,8 @@ snapshots: typescript@5.3.3: {} + uc.micro@2.1.0: {} + uglify-js@3.17.4: optional: true diff --git a/src/ts/parser.ts b/src/ts/parser.ts index 0b516e1c..4f356c2a 100644 --- a/src/ts/parser.ts +++ b/src/ts/parser.ts @@ -1,6 +1,5 @@ import DOMPurify from 'isomorphic-dompurify'; -import { Marked } from 'marked'; - +import markdownit from 'markdown-it' import { DataBase, setDatabase, type Database, type Message, type character, type customscript, type groupChat, type triggerscript } from './storage/database'; import { getFileSrc } from './storage/globalApi'; import { processScriptFull } from './process/scripts'; @@ -18,24 +17,12 @@ import { requestChatData } from './process/request'; import type { OpenAIChat } from './process'; import { alertInput, alertNormal } from './alert'; -const mconverted = new Marked({ - gfm: true, +const mconverted = markdownit({ + html: true, breaks: true, - silent: true, - tokenizer: { - del(src) { - const cap = /^~~~(?=\S)([\s\S]*?\S)~~~/.exec(src); - if (cap) { - return { - type: 'del', - raw: cap[0], - text: cap[1], - tokens: this.lexer.inlineTokens(cap[1]) - }; - } - } - } + linkify: false }) +mconverted.disable(['code']) @@ -194,7 +181,7 @@ export async function ParseMarkdown(data:string, charArg:(character|simpleCharac data = encodeStyle(data) if(mode === 'normal'){ data = risuFormater(data) - data = mconverted.parse(data) + data = mconverted.render(data) } return decodeStyle(DOMPurify.sanitize(data, { ADD_TAGS: ["iframe", "style", "risu-style", "x-em"], @@ -212,12 +199,12 @@ export function postTranslationParse(data:string){ } } - data = mconverted.parse(lines.join('\n')) + data = mconverted.render(lines.join('\n')) return data } export function parseMarkdownSafe(data:string) { - return DOMPurify.sanitize(mconverted.parse(data), { + return DOMPurify.sanitize(mconverted.render(data), { FORBID_TAGS: ["a", "style"], FORBID_ATTR: ["style", "href", "class"] }) @@ -2291,4 +2278,32 @@ export async function promptTypeParser(prompt:string):Promise -1 -const mconverted = new Marked({ - gfm: true, - breaks: true, - silent: true, - tokenizer: { - - } -}) - export interface Messagec extends Message{ index: number } @@ -585,33 +575,6 @@ export function getNodetextToSentence(node: Node): string { return result; } -export function applyMarkdownToNode(node: Node) { - if (node.nodeType === Node.TEXT_NODE) { - const text = node.textContent; - if (text) { - let markdown = mconverted.parseInline(text); - if (markdown !== text) { - const span = document.createElement('span'); - span.innerHTML = markdown; - - // inherit inline style from the parent node - const parentStyle = (node.parentNode as HTMLElement)?.style; - if(parentStyle){ - for(let i=0;i