From 25b507a01a36f779d4f63e884382d47c5f5d29d2 Mon Sep 17 00:00:00 2001 From: Dong Hyun Kim <146043537+ThePotioner@users.noreply.github.com> Date: Thu, 6 Feb 2025 18:33:43 +0900 Subject: [PATCH 1/5] refactor(parser): import only the required asset when in a node environment - refactor parseAdditionalAssets() - Modified the getClostMatch() to work well with Object to which it is dynamically added. --- src/ts/parser.svelte.ts | 142 +++++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 54 deletions(-) diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts index c6912d5e..4a235208 100644 --- a/src/ts/parser.svelte.ts +++ b/src/ts/parser.svelte.ts @@ -295,6 +295,14 @@ async function renderHighlightableMarkdown(data:string) { export const assetRegex = /{{(raw|path|img|image|video|audio|bg|emotion|asset|video-img|source)::(.+?)}}/gms +async function replaceAsync(string, regexp, replacerFunction) { + const replacements = await Promise.all( + Array.from(string.matchAll(regexp), + match => replacerFunction(...match as any))); + let i = 0; + return string.replace(regexp, () => replacements[i++]); +} + async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|character, mode:'normal'|'back', mode2:'unset'|'pre'|'post' = 'unset'){ const assetWidthString = (DBState.db.assetWidth && DBState.db.assetWidth !== -1 || DBState.db.assetWidth === 0) ? `max-width:${DBState.db.assetWidth}rem;` : '' @@ -306,15 +314,6 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c path:string }} = {} - if(char.additionalAssets){ - for(const asset of char.additionalAssets){ - const assetPath = await getFileSrc(asset[1]) - assetPaths[asset[0].toLocaleLowerCase()] = { - path: assetPath, - ext: asset[2] - } - } - } if(char.emotionImages){ for(const emo of char.emotionImages){ const emoPath = await getFileSrc(emo[1]) @@ -335,8 +334,23 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c } const videoExtention = ['mp4', 'webm', 'avi', 'm4p', 'm4v'] let needsSourceAccess = false - data = data.replaceAll(assetRegex, (full:string, type:string, name:string) => { + data = await replaceAsync(data, assetRegex, async (full:string, type:string, name:string) => { name = name.toLocaleLowerCase() + if(char.additionalAssets){ + for(const asset of char.additionalAssets){ + if (trimmer(asset[0].toLocaleLowerCase()) === trimmer(name)){ + console.log("In parseAdditionalAssets() > Catch Asset : " + asset[0] + "\nIn parseAdditionalAssets() > Asset Path : " + asset[1]) + const assetPath = await getFileSrc(asset[1]) + assetPaths[name] = { + path: assetPath, + ext: asset[2] + } + } + } + } + console.log("In parseAdditionalAssets() > Name: " + name) + console.log("In parseAdditionalAssets() > Char Obj: ", char) + console.log("In parseAdditionalAssets() > Char Type: " + char.type) if(type === 'emotion'){ const path = emoPaths[name]?.path if(!path){ @@ -361,7 +375,7 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c return '' } - path = getClosestMatch(name, assetPaths) + path = await getClosestMatch(char, name, assetPaths) if(!path){ return '' @@ -411,63 +425,84 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c return data } -function getClosestMatch(name:string, assetPaths:{[key:string]:{path:string, ext?:string}}){ - - if(Object.keys(assetPaths).length === 0){ - return null - } - - //Levenshtein distance, new with 1d array - const dest = (a:string, b:string) => { - const h = a.length + 1 - const w = b.length + 1 - let d = new Int16Array(h * w) - for(let i=0;i DBState.db.assetMaxDifference){ return null } + + const assetPath = await getFileSrc(targetPath) + assetPaths[closest] = { + path: assetPath, + ext: targetExt + } + + console.log("In getClosestMatch() > assetPaths[closest]: " + assetPaths[closest]) + console.log("In getClosestMatch() > closet, closetDist: " + closest, closestDist) + return assetPaths[closest] } +//Levenshtein distance, new with 1d array +function getDistance(a:string, b:string) { + const h = a.length + 1 + const w = b.length + 1 + let d = new Int16Array(h * w) + for(let i=0;i { try { let text = Buffer.from(txt, 'hex').toString('utf-8') From b3587fc950428b7584b0c8cf29abd101d4598bbb Mon Sep 17 00:00:00 2001 From: Dong Hyun Kim <146043537+ThePotioner@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:38:57 +0900 Subject: [PATCH 2/5] refactor(parsor): cleanup & add getAssetSrc() - Modified getAssetSrc() to handle Emotional, Module, and AdditionalAssets. - Reduced if-for statement depth for improved readability. --- src/ts/parser.svelte.ts | 79 ++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts index 4a235208..622b7ade 100644 --- a/src/ts/parser.svelte.ts +++ b/src/ts/parser.svelte.ts @@ -15,6 +15,7 @@ import type { OpenAIChat } from './process/index.svelte'; import hljs from 'highlight.js/lib/core' import 'highlight.js/styles/atom-one-dark.min.css' import { language } from 'src/lang'; +import type { key } from 'localforage'; const markdownItOptions = { html: true, @@ -303,6 +304,33 @@ async function replaceAsync(string, regexp, replacerFunction) { return string.replace(regexp, () => replacements[i++]); } +async function getAssetSrc(assetArr: string[][], name: string, assetPaths?: {[key: string]:{path: string, ext?: string}}, emoPaths?: {[key: string]:{path: string, ext?: string}}) { + name = name.toLocaleLowerCase() + if (assetPaths) { + for(const asset of assetArr){ + if (trimmer(asset[0].toLocaleLowerCase()) !== trimmer(name)) continue; + console.log("In getAssets() > Catch Asset : " + asset[0]) + console.log("In getAssets() > Asset Path : " + asset[1]) + const assetPath = await getFileSrc(asset[1]) + assetPaths[name] = { + path: assetPath, + ext: asset[2] + } + } + } + if (emoPaths) { + for(const emo of assetArr){ + if (trimmer(emo[0].toLocaleLowerCase()) !== trimmer(name)) continue; + console.log("In getAssets() > Catch Asset(emo) : " + emo[0]) + console.log("In getAssets() > Asset Path(emo) : " + emo[1]) + const emoPath = await getFileSrc(emo[1]) + emoPaths[name] = { + path: emoPath, + } + } + } +} + async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|character, mode:'normal'|'back', mode2:'unset'|'pre'|'post' = 'unset'){ const assetWidthString = (DBState.db.assetWidth && DBState.db.assetWidth !== -1 || DBState.db.assetWidth === 0) ? `max-width:${DBState.db.assetWidth}rem;` : '' @@ -314,39 +342,19 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c path:string }} = {} - if(char.emotionImages){ - for(const emo of char.emotionImages){ - const emoPath = await getFileSrc(emo[1]) - emoPaths[emo[0].toLocaleLowerCase()] = { - path: emoPath, - } - } - } - const moduleAssets = getModuleAssets() - if(moduleAssets.length > 0){ - for(const asset of moduleAssets){ - const assetPath = await getFileSrc(asset[1]) - assetPaths[asset[0].toLocaleLowerCase()] = { - path: assetPath, - ext: asset[2] - } - } - } const videoExtention = ['mp4', 'webm', 'avi', 'm4p', 'm4v'] let needsSourceAccess = false + data = await replaceAsync(data, assetRegex, async (full:string, type:string, name:string) => { - name = name.toLocaleLowerCase() - if(char.additionalAssets){ - for(const asset of char.additionalAssets){ - if (trimmer(asset[0].toLocaleLowerCase()) === trimmer(name)){ - console.log("In parseAdditionalAssets() > Catch Asset : " + asset[0] + "\nIn parseAdditionalAssets() > Asset Path : " + asset[1]) - const assetPath = await getFileSrc(asset[1]) - assetPaths[name] = { - path: assetPath, - ext: asset[2] - } - } - } + const moduleAssets = getModuleAssets() + if (char.additionalAssets) { + await getAssetSrc(char.additionalAssets, name, assetPaths); + } + if (char.emotionImages) { + await getAssetSrc(char.emotionImages, name, assetPaths, emoPaths); + } + if (moduleAssets.length > 0) { + await getAssetSrc(moduleAssets, name, assetPaths); } console.log("In parseAdditionalAssets() > Name: " + name) console.log("In parseAdditionalAssets() > Char Obj: ", char) @@ -444,14 +452,6 @@ async function getClosestMatch(char: simpleCharacterArgument|character, name:str targetExt = asset[2] } } - - // for(const key in assetPaths){ - // const dist = getDistance(trimmedName, trimmer(key)) - // if(dist < closestDist){ - // closest = key - // closestDist = dist - // } - // } if(closestDist > DBState.db.assetMaxDifference){ return null @@ -463,7 +463,7 @@ async function getClosestMatch(char: simpleCharacterArgument|character, name:str ext: targetExt } - console.log("In getClosestMatch() > assetPaths[closest]: " + assetPaths[closest]) + console.log("In getClosestMatch() > assetPaths[closest]: ", assetPaths[closest]) console.log("In getClosestMatch() > closet, closetDist: " + closest, closestDist) return assetPaths[closest] @@ -499,7 +499,6 @@ function trimmer(str:string){ } } - return str.trim().replace(/[_ -.]/g, '') } From 997b4f70ce6e57760cd1c0e0aae0478835748a44 Mon Sep 17 00:00:00 2001 From: Dong Hyun Kim <146043537+ThePotioner@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:41:40 +0900 Subject: [PATCH 3/5] refactor(parser): cleanup & add getEmoSrc() --- src/ts/parser.svelte.ts | 51 +++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts index 622b7ade..e3c5be18 100644 --- a/src/ts/parser.svelte.ts +++ b/src/ts/parser.svelte.ts @@ -293,7 +293,6 @@ async function renderHighlightableMarkdown(data:string) { } - export const assetRegex = /{{(raw|path|img|image|video|audio|bg|emotion|asset|video-img|source)::(.+?)}}/gms async function replaceAsync(string, regexp, replacerFunction) { @@ -304,29 +303,24 @@ async function replaceAsync(string, regexp, replacerFunction) { return string.replace(regexp, () => replacements[i++]); } -async function getAssetSrc(assetArr: string[][], name: string, assetPaths?: {[key: string]:{path: string, ext?: string}}, emoPaths?: {[key: string]:{path: string, ext?: string}}) { +async function getAssetSrc(assetArr: string[][], name: string, assetPaths: {[key: string]:{path: string, ext?: string}}) { name = name.toLocaleLowerCase() - if (assetPaths) { - for(const asset of assetArr){ - if (trimmer(asset[0].toLocaleLowerCase()) !== trimmer(name)) continue; - console.log("In getAssets() > Catch Asset : " + asset[0]) - console.log("In getAssets() > Asset Path : " + asset[1]) - const assetPath = await getFileSrc(asset[1]) - assetPaths[name] = { - path: assetPath, - ext: asset[2] - } + for (const asset of assetArr) { + if (trimmer(asset[0].toLocaleLowerCase()) !== trimmer(name)) continue; + const assetPath = await getFileSrc(asset[1]) + assetPaths[asset[0].toLocaleLowerCase()] = { + path: assetPath, + ext: asset[2] } + return; } - if (emoPaths) { - for(const emo of assetArr){ - if (trimmer(emo[0].toLocaleLowerCase()) !== trimmer(name)) continue; - console.log("In getAssets() > Catch Asset(emo) : " + emo[0]) - console.log("In getAssets() > Asset Path(emo) : " + emo[1]) - const emoPath = await getFileSrc(emo[1]) - emoPaths[name] = { - path: emoPath, - } +} + +async function getEmoSrc(emoArr: string[][], emoPaths: {[key: string]:{path: string}}) { + for (const emo of emoArr) { + const emoPath = await getFileSrc(emo[1]) + emoPaths[emo[0].toLocaleLowerCase()] = { + path: emoPath, } } } @@ -342,6 +336,8 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c path:string }} = {} + if (char.emotionImages) getEmoSrc(char.emotionImages, emoPaths) + const videoExtention = ['mp4', 'webm', 'avi', 'm4p', 'm4v'] let needsSourceAccess = false @@ -350,15 +346,10 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c if (char.additionalAssets) { await getAssetSrc(char.additionalAssets, name, assetPaths); } - if (char.emotionImages) { - await getAssetSrc(char.emotionImages, name, assetPaths, emoPaths); - } if (moduleAssets.length > 0) { await getAssetSrc(moduleAssets, name, assetPaths); } - console.log("In parseAdditionalAssets() > Name: " + name) - console.log("In parseAdditionalAssets() > Char Obj: ", char) - console.log("In parseAdditionalAssets() > Char Type: " + char.type) + if(type === 'emotion'){ const path = emoPaths[name]?.path if(!path){ @@ -366,6 +357,7 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c } return `${path}` } + if(type === 'source'){ needsSourceAccess = true switch(name){ @@ -377,7 +369,9 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c } } } + let path = assetPaths[name] + if(!path){ if(DBState.db.legacyMediaFindings){ return '' @@ -462,9 +456,6 @@ async function getClosestMatch(char: simpleCharacterArgument|character, name:str path: assetPath, ext: targetExt } - - console.log("In getClosestMatch() > assetPaths[closest]: ", assetPaths[closest]) - console.log("In getClosestMatch() > closet, closetDist: " + closest, closestDist) return assetPaths[closest] } From ffd904ab5c2b0ba03963e3f9ffb0c506e230f045 Mon Sep 17 00:00:00 2001 From: Dong Hyun Kim <146043537+ThePotioner@users.noreply.github.com> Date: Fri, 7 Feb 2025 10:23:15 +0900 Subject: [PATCH 4/5] fix(parser): fix typo & cleanup --- src/ts/parser.svelte.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts index e3c5be18..33e14f5d 100644 --- a/src/ts/parser.svelte.ts +++ b/src/ts/parser.svelte.ts @@ -298,21 +298,21 @@ export const assetRegex = /{{(raw|path|img|image|video|audio|bg|emotion|asset|vi async function replaceAsync(string, regexp, replacerFunction) { const replacements = await Promise.all( Array.from(string.matchAll(regexp), - match => replacerFunction(...match as any))); + match => replacerFunction(...match as any))) let i = 0; - return string.replace(regexp, () => replacements[i++]); + return string.replace(regexp, () => replacements[i++]) } async function getAssetSrc(assetArr: string[][], name: string, assetPaths: {[key: string]:{path: string, ext?: string}}) { name = name.toLocaleLowerCase() for (const asset of assetArr) { - if (trimmer(asset[0].toLocaleLowerCase()) !== trimmer(name)) continue; + if (trimmer(asset[0].toLocaleLowerCase()) !== trimmer(name)) continue const assetPath = await getFileSrc(asset[1]) assetPaths[asset[0].toLocaleLowerCase()] = { path: assetPath, ext: asset[2] } - return; + return } } @@ -336,7 +336,7 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c path:string }} = {} - if (char.emotionImages) getEmoSrc(char.emotionImages, emoPaths) + if (char.emotionImages) await getEmoSrc(char.emotionImages, emoPaths) const videoExtention = ['mp4', 'webm', 'avi', 'm4p', 'm4v'] let needsSourceAccess = false @@ -344,10 +344,10 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c data = await replaceAsync(data, assetRegex, async (full:string, type:string, name:string) => { const moduleAssets = getModuleAssets() if (char.additionalAssets) { - await getAssetSrc(char.additionalAssets, name, assetPaths); + await getAssetSrc(char.additionalAssets, name, assetPaths) } if (moduleAssets.length > 0) { - await getAssetSrc(moduleAssets, name, assetPaths); + await getAssetSrc(moduleAssets, name, assetPaths) } if(type === 'emotion'){ From e3f205cc310727903cdbcda2f146567032a07106 Mon Sep 17 00:00:00 2001 From: Dong Hyun Kim <146043537+ThePotioner@users.noreply.github.com> Date: Fri, 7 Feb 2025 10:43:03 +0900 Subject: [PATCH 5/5] fix(parsor): remove incorrectly imported packages --- src/ts/parser.svelte.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts index 33e14f5d..6cd15b20 100644 --- a/src/ts/parser.svelte.ts +++ b/src/ts/parser.svelte.ts @@ -15,7 +15,6 @@ import type { OpenAIChat } from './process/index.svelte'; import hljs from 'highlight.js/lib/core' import 'highlight.js/styles/atom-one-dark.min.css' import { language } from 'src/lang'; -import type { key } from 'localforage'; const markdownItOptions = { html: true,