mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

change markdown engine to markdown-lt

Browse Source
This commit is contained in:
kwaroran
2024-07-02 05:04:37 +09:00
parent 45b16da8a5
commit f38b88e208
5 changed files with 105 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -28,6 +28,7 @@
"@smithy/protocol-http": "^3.0.12",
"@smithy/signature-v4": "^2.0.19",
"@tauri-apps/api": "1.5.3",
"@types/markdown-it": "^14.1.1",
"@xenova/transformers": "^2.17.1",
"blueimp-md5": "^2.19.0",
"body-parser": "^1.20.2",
@@ -50,6 +51,7 @@
"localforage": "^1.10.0",
"lodash": "^4.17.21",
"lucide-svelte": "^0.292.0",
"markdown-it": "^14.1.0",
"marked": "^5.1.2",
"ml-distance": "^4.0.1",
"mobile-drag-drop": "3.0.0-rc.0",

65
pnpm-lock.yaml generated
View File

@@ -44,6 +44,9 @@ importers:
'@tauri-apps/api':
specifier: 1.5.3
version: 1.5.3
'@types/markdown-it':
specifier: ^14.1.1
version: 14.1.1
'@xenova/transformers':
specifier: ^2.17.1
version: 2.17.1
@@ -110,6 +113,9 @@ importers:
lucide-svelte:
specifier: ^0.292.0
version: 0.292.0(svelte@4.2.8)
markdown-it:
specifier: ^14.1.0
version: 14.1.0
marked:
specifier: ^5.1.2
version: 5.1.2
@@ -1028,6 +1034,9 @@ packages:
'@types/libsodium-wrappers@0.7.13':
resolution: {integrity: sha512-KeAKtlObirLJk/na6jHBFEdTDjDfFS6Vcr0eG2FjiHKn3Nw8axJFfIu0Y9TpwaauRldQBj/pZm/MHtK76r6OWg==}
'@types/linkify-it@5.0.0':
resolution: {integrity: sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==}
'@types/lodash.isequal@4.5.8':
resolution: {integrity: sha512-uput6pg4E/tj2LGxCZo9+y27JNyB2OZuuI/T5F+ylVDYuqICLG2/ktjxx0v6GvVntAf8TvEzeQLcV0ffRirXuA==}
@@ -1037,9 +1046,15 @@ packages:
'@types/long@4.0.2':
resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==}
'@types/markdown-it@14.1.1':
resolution: {integrity: sha512-4NpsnpYl2Gt1ljyBGrKMxFYAYvpqbnnkgP/i/g+NLpjEUa3obn1XJCur9YbEXKDAkaXqsR1LbDnGEJ0MmKFxfg==}
'@types/marked@5.0.2':
resolution: {integrity: sha512-OucS4KMHhFzhz27KxmWg7J+kIYqyqoW5kdIEI319hqARQQUTqhao3M/F+uFnDXD0Rg72iDDZxZNxq5gvctmLlg==}
'@types/mdurl@2.0.0':
resolution: {integrity: sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==}
'@types/minimist@1.2.5':
resolution: {integrity: sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==}
@@ -1174,6 +1189,9 @@ packages:
arg@5.0.2:
resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==}
argparse@2.0.1:
resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==}
aria-query@5.3.0:
resolution: {integrity: sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==}
@@ -2264,6 +2282,9 @@ packages:
lines-and-columns@1.2.4:
resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
linkify-it@5.0.0:
resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==}
load-json-file@4.0.0:
resolution: {integrity: sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==}
engines: {node: '>=4'}
@@ -2332,6 +2353,10 @@ packages:
resolution: {integrity: sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==}
engines: {node: '>=8'}
markdown-it@14.1.0:
resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==}
hasBin: true
marked@5.1.2:
resolution: {integrity: sha512-ahRPGXJpjMjwSOlBoTMZAK7ATXkli5qCPxZ21TG44rx1KEo44bii4ekgTDQPNRQ4Kh7JMb9Ub1PVk1NxRSsorg==}
engines: {node: '>= 16'}
@@ -2340,6 +2365,9 @@ packages:
mdn-data@2.0.30:
resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==}
mdurl@2.0.0:
resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==}
media-typer@0.3.0:
resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==}
engines: {node: '>= 0.6'}
@@ -2883,6 +2911,10 @@ packages:
pump@3.0.0:
resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==}
punycode.js@2.3.1:
resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==}
engines: {node: '>=6'}
punycode@2.3.1:
resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==}
engines: {node: '>=6'}
@@ -3461,6 +3493,9 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
uc.micro@2.1.0:
resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==}
uglify-js@3.17.4:
resolution: {integrity: sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==}
engines: {node: '>=0.8.0'}
@@ -4475,6 +4510,8 @@ snapshots:
'@types/libsodium-wrappers@0.7.13': {}
'@types/linkify-it@5.0.0': {}
'@types/lodash.isequal@4.5.8':
dependencies:
'@types/lodash': 4.14.202
@@ -4483,8 +4520,15 @@ snapshots:
'@types/long@4.0.2': {}
'@types/markdown-it@14.1.1':
dependencies:
'@types/linkify-it': 5.0.0
'@types/mdurl': 2.0.0
'@types/marked@5.0.2': {}
'@types/mdurl@2.0.0': {}
'@types/minimist@1.2.5': {}
'@types/node@18.19.7':
@@ -4612,6 +4656,8 @@ snapshots:
arg@5.0.2: {}
argparse@2.0.1: {}
aria-query@5.3.0:
dependencies:
dequal: 2.0.3
@@ -5822,6 +5868,10 @@ snapshots:
lines-and-columns@1.2.4: {}
linkify-it@5.0.0:
dependencies:
uc.micro: 2.1.0
load-json-file@4.0.0:
dependencies:
graceful-fs: 4.2.11
@@ -5881,10 +5931,21 @@ snapshots:
map-obj@4.3.0: {}
markdown-it@14.1.0:
dependencies:
argparse: 2.0.1
entities: 4.5.0
linkify-it: 5.0.0
mdurl: 2.0.0
punycode.js: 2.3.1
uc.micro: 2.1.0
marked@5.1.2: {}
mdn-data@2.0.30: {}
mdurl@2.0.0: {}
media-typer@0.3.0: {}
meow@8.1.2:
@@ -6453,6 +6514,8 @@ snapshots:
end-of-stream: 1.4.4
once: 1.4.0
punycode.js@2.3.1: {}
punycode@2.3.1: {}
q@1.5.1: {}
@@ -7112,6 +7175,8 @@ snapshots:
typescript@5.3.3: {}
uc.micro@2.1.0: {}
uglify-js@3.17.4:
optional: true

57
src/ts/parser.ts
View File

@@ -1,6 +1,5 @@
import DOMPurify from 'isomorphic-dompurify';
import { Marked } from 'marked';
import markdownit from 'markdown-it'
import { DataBase, setDatabase, type Database, type Message, type character, type customscript, type groupChat, type triggerscript } from './storage/database';
import { getFileSrc } from './storage/globalApi';
import { processScriptFull } from './process/scripts';
@@ -18,24 +17,12 @@ import { requestChatData } from './process/request';
import type { OpenAIChat } from './process';
import { alertInput, alertNormal } from './alert';
const mconverted = new Marked({
gfm: true,
const mconverted = markdownit({
html: true,
breaks: true,
silent: true,
tokenizer: {
del(src) {
const cap = /^~~~(?=\S)([\s\S]*?\S)~~~/.exec(src);
if (cap) {
return {
type: 'del',
raw: cap[0],
text: cap[1],
tokens: this.lexer.inlineTokens(cap[1])
};
}
}
}
linkify: false
})
mconverted.disable(['code'])
@@ -194,7 +181,7 @@ export async function ParseMarkdown(data:string, charArg:(character|simpleCharac
data = encodeStyle(data)
if(mode === 'normal'){
data = risuFormater(data)
data = mconverted.parse(data)
data = mconverted.render(data)
}
return decodeStyle(DOMPurify.sanitize(data, {
ADD_TAGS: ["iframe", "style", "risu-style", "x-em"],
@@ -212,12 +199,12 @@ export function postTranslationParse(data:string){
}
}
data = mconverted.parse(lines.join('\n'))
data = mconverted.render(lines.join('\n'))
return data
}
export function parseMarkdownSafe(data:string) {
return DOMPurify.sanitize(mconverted.parse(data), {
return DOMPurify.sanitize(mconverted.render(data), {
FORBID_TAGS: ["a", "style"],
FORBID_ATTR: ["style", "href", "class"]
})
@@ -2291,4 +2278,32 @@ export async function promptTypeParser(prompt:string):Promise<string | PromptPar
}
return prompt
}
export function applyMarkdownToNode(node: Node) {
if (node.nodeType === Node.TEXT_NODE) {
const text = node.textContent;
if (text) {
let markdown = mconverted.render(text);
if (markdown !== text) {
const span = document.createElement('span');
span.innerHTML = markdown;
// inherit inline style from the parent node
const parentStyle = (node.parentNode as HTMLElement)?.style;
if(parentStyle){
for(let i=0;i<parentStyle.length;i++){
span.style.setProperty(parentStyle[i], parentStyle.getPropertyValue(parentStyle[i]))
}
}
(node as Element)?.replaceWith(span);
return
}
}
} else {
for (const child of node.childNodes) {
applyMarkdownToNode(child);
}
}
}

4
src/ts/translator/translator.ts
View File

@@ -5,10 +5,10 @@ import { globalFetch, isTauri } from "../storage/globalApi"
import { alertError } from "../alert"
import { requestChatData } from "../process/request"
import { doingChat } from "../process"
import type { simpleCharacterArgument } from "../parser"
import { applyMarkdownToNode, type simpleCharacterArgument } from "../parser"
import { selectedCharID } from "../stores"
import { getModuleRegexScripts } from "../process/modules"
import { getNodetextToSentence, sleep, applyMarkdownToNode } from "../util"
import { getNodetextToSentence, sleep } from "../util"
import { processScriptFull } from "../process/scripts"
import { Capacitor } from "@capacitor/core"

37
src/ts/util.ts
View File

@@ -8,19 +8,9 @@ import { basename } from "@tauri-apps/api/path"
import { createBlankChar, getCharImage } from "./characters"
import { appWindow } from '@tauri-apps/api/window';
import { isTauri } from "./storage/globalApi"
import { Marked } from "marked"
export const isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1
const mconverted = new Marked({
gfm: true,
breaks: true,
silent: true,
tokenizer: {
}
})
export interface Messagec extends Message{
index: number
}
@@ -585,33 +575,6 @@ export function getNodetextToSentence(node: Node): string {
return result;
}
export function applyMarkdownToNode(node: Node) {
if (node.nodeType === Node.TEXT_NODE) {
const text = node.textContent;
if (text) {
let markdown = mconverted.parseInline(text);
if (markdown !== text) {
const span = document.createElement('span');
span.innerHTML = markdown;
// inherit inline style from the parent node
const parentStyle = (node.parentNode as HTMLElement)?.style;
if(parentStyle){
for(let i=0;i<parentStyle.length;i++){
span.style.setProperty(parentStyle[i], parentStyle.getPropertyValue(parentStyle[i]))
}
}
(node as Element)?.replaceWith(span);
return
}
}
} else {
for (const child of node.childNodes) {
applyMarkdownToNode(child);
}
}
}
export const TagList = [
{