mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into patch-25

Browse Source
This commit is contained in:
kwaroran
2025-02-09 16:24:49 +09:00
committed by GitHub
parent fb941148e0 4a4a47b418
commit 7e71d03d21
14 changed files with 1030 additions and 471 deletions
Download Patch File Download Diff File Expand all files Collapse all files

171
src/ts/parser.svelte.ts
View File

@@ -292,9 +292,38 @@ async function renderHighlightableMarkdown(data:string) {
}
export const assetRegex = /{{(raw|path|img|image|video|audio|bg|emotion|asset|video-img|source)::(.+?)}}/gms
async function replaceAsync(string, regexp, replacerFunction) {
const replacements = await Promise.all(
Array.from(string.matchAll(regexp),
match => replacerFunction(...match as any)))
let i = 0;
return string.replace(regexp, () => replacements[i++])
}
async function getAssetSrc(assetArr: string[][], name: string, assetPaths: {[key: string]:{path: string, ext?: string}}) {
name = name.toLocaleLowerCase()
for (const asset of assetArr) {
if (trimmer(asset[0].toLocaleLowerCase()) !== trimmer(name)) continue
const assetPath = await getFileSrc(asset[1])
assetPaths[asset[0].toLocaleLowerCase()] = {
path: assetPath,
ext: asset[2]
}
return
}
}
async function getEmoSrc(emoArr: string[][], emoPaths: {[key: string]:{path: string}}) {
for (const emo of emoArr) {
const emoPath = await getFileSrc(emo[1])
emoPaths[emo[0].toLocaleLowerCase()] = {
path: emoPath,
}
}
}
async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|character, mode:'normal'|'back', mode2:'unset'|'pre'|'post' = 'unset'){
const assetWidthString = (DBState.db.assetWidth && DBState.db.assetWidth !== -1 || DBState.db.assetWidth === 0) ? `max-width:${DBState.db.assetWidth}rem;` : ''
@@ -306,37 +335,20 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c
path:string
}} = {}
if(char.additionalAssets){
for(const asset of char.additionalAssets){
const assetPath = await getFileSrc(asset[1])
assetPaths[asset[0].toLocaleLowerCase()] = {
path: assetPath,
ext: asset[2]
}
}
}
if(char.emotionImages){
for(const emo of char.emotionImages){
const emoPath = await getFileSrc(emo[1])
emoPaths[emo[0].toLocaleLowerCase()] = {
path: emoPath,
}
}
}
const moduleAssets = getModuleAssets()
if(moduleAssets.length > 0){
for(const asset of moduleAssets){
const assetPath = await getFileSrc(asset[1])
assetPaths[asset[0].toLocaleLowerCase()] = {
path: assetPath,
ext: asset[2]
}
}
}
if (char.emotionImages) await getEmoSrc(char.emotionImages, emoPaths)
const videoExtention = ['mp4', 'webm', 'avi', 'm4p', 'm4v']
let needsSourceAccess = false
data = data.replaceAll(assetRegex, (full:string, type:string, name:string) => {
name = name.toLocaleLowerCase()
data = await replaceAsync(data, assetRegex, async (full:string, type:string, name:string) => {
const moduleAssets = getModuleAssets()
if (char.additionalAssets) {
await getAssetSrc(char.additionalAssets, name, assetPaths)
}
if (moduleAssets.length > 0) {
await getAssetSrc(moduleAssets, name, assetPaths)
}
if(type === 'emotion'){
const path = emoPaths[name]?.path
if(!path){
@@ -344,6 +356,7 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c
}
return `<img src="${path}" alt="${path}" style="${assetWidthString} "/>`
}
if(type === 'source'){
needsSourceAccess = true
switch(name){
@@ -355,13 +368,15 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c
}
}
}
let path = assetPaths[name]
if(!path){
if(DBState.db.legacyMediaFindings){
return ''
}
path = getClosestMatch(name, assetPaths)
path = await getClosestMatch(char, name, assetPaths)
if(!path){
return ''
@@ -411,63 +426,72 @@ async function parseAdditionalAssets(data:string, char:simpleCharacterArgument|c
return data
}
function getClosestMatch(name:string, assetPaths:{[key:string]:{path:string, ext?:string}}){
if(Object.keys(assetPaths).length === 0){
return null
}
//Levenshtein distance, new with 1d array
const dest = (a:string, b:string) => {
const h = a.length + 1
const w = b.length + 1
let d = new Int16Array(h * w)
for(let i=0;i<h;i++){
d[i * w] = i
}
for(let i=0;i<w;i++){
d[i] = i
}
for(let i=1; i<h; i++){
for(let j=1;j<w;j++){
d[i * w + j] = Math.min(
d[(i-1) * w + j-1] + (a.charAt(i-1)===b.charAt(j-1) ? 0 : 1),
d[(i-1) * w + j]+1, d[i * w + j-1]+1
)
}
}
return d[h * w - 1]
}
function trimmer(str:string){
const ext = ['webp', 'png', 'jpg', 'jpeg', 'gif', 'mp4', 'webm', 'avi', 'm4p', 'm4v', 'mp3', 'wav', 'ogg']
for(const e of ext){
if(str.endsWith('.' + e)){
str = str.substring(0, str.length - e.length - 1)
}
}
return str.trim().replace(/[_ -.]/g, '')
}
async function getClosestMatch(char: simpleCharacterArgument|character, name:string, assetPaths:{[key:string]:{path:string, ext?:string}}){
if(!char.additionalAssets) return null
let closest = ''
let closestDist = 999999
let targetPath = ''
let targetExt = ''
const trimmedName = trimmer(name)
for(const key in assetPaths){
const dist = dest(trimmedName, trimmer(key))
for(const asset of char.additionalAssets) {
const key = asset[0].toLocaleLowerCase()
const dist = getDistance(trimmedName, trimmer(key))
if(dist < closestDist){
closest = key
closestDist = dist
targetPath = asset[1]
targetExt = asset[2]
}
}
if(closestDist > DBState.db.assetMaxDifference){
return null
}
const assetPath = await getFileSrc(targetPath)
assetPaths[closest] = {
path: assetPath,
ext: targetExt
}
return assetPaths[closest]
}
//Levenshtein distance, new with 1d array
function getDistance(a:string, b:string) {
const h = a.length + 1
const w = b.length + 1
let d = new Int16Array(h * w)
for(let i=0;i<h;i++){
d[i * w] = i
}
for(let i=0;i<w;i++){
d[i] = i
}
for(let i=1; i<h; i++){
for(let j=1;j<w;j++){
d[i * w + j] = Math.min(
d[(i-1) * w + j-1] + (a.charAt(i-1)===b.charAt(j-1) ? 0 : 1),
d[(i-1) * w + j]+1, d[i * w + j-1]+1
)
}
}
return d[h * w - 1]
}
function trimmer(str:string){
const ext = ['webp', 'png', 'jpg', 'jpeg', 'gif', 'mp4', 'webm', 'avi', 'm4p', 'm4v', 'mp3', 'wav', 'ogg']
for(const e of ext){
if(str.endsWith('.' + e)){
str = str.substring(0, str.length - e.length - 1)
}
}
return str.trim().replace(/[_ -.]/g, '')
}
async function parseInlayAssets(data:string){
const inlayMatch = data.match(/{{(inlay|inlayed)::(.+?)}}/g)
if(inlayMatch){
@@ -605,7 +629,6 @@ function decodeStyleRule(rule:CssAtRuleAST){
}
function decodeStyle(text:string){
return text.replaceAll(styleDecodeRegex, (full, txt:string) => {
try {
let text = Buffer.from(txt, 'hex').toString('utf-8')

67
src/ts/process/memory/hypav3.ts
View File

@@ -421,6 +421,12 @@ export async function hypaMemoryV3(
continue;
}
if (db.hypaV3Settings.doNotSummarizeUserMessage && chat.role === "user") {
console.log(`[HypaV3] Skipping user role at index ${i}`);
continue;
}
toSummarize.push(chat);
}
@@ -436,23 +442,25 @@ export async function hypaMemoryV3(
}
// Attempt summarization
const summarizeResult = await retryableSummarize(toSummarize);
if (toSummarize.length > 0) {
const summarizeResult = await retryableSummarize(toSummarize);
if (!summarizeResult.success) {
return {
currentTokens,
chats,
error: `[HypaV3] Summarization failed after maximum retries: ${summarizeResult.data}`,
memory: toSerializableHypaV3Data(data),
};
if (!summarizeResult.success) {
return {
currentTokens,
chats,
error: `[HypaV3] Summarization failed after maximum retries: ${summarizeResult.data}`,
memory: toSerializableHypaV3Data(data),
};
}
data.summaries.push({
text: summarizeResult.data,
chatMemos: new Set(toSummarize.map((chat) => chat.memo)),
isImportant: false,
});
}
data.summaries.push({
text: summarizeResult.data,
chatMemos: new Set(toSummarize.map((chat) => chat.memo)),
isImportant: false,
});
currentTokens -= toSummarizeTokens;
startIdx = endIdx;
}
@@ -469,6 +477,37 @@ export async function hypaMemoryV3(
availableMemoryTokens
);
// Early return if no summaries
if (data.summaries.length === 0) {
// Generate final memory prompt
const memory = encapsulateMemoryPrompt("");
const newChats: OpenAIChat[] = [
{
role: "system",
content: memory,
memo: "supaMemory",
},
...chats.slice(startIdx),
];
console.log(
"[HypaV3] Exiting function:",
"\nCurrent Tokens:",
currentTokens,
"\nAll chats, including memory prompt:",
newChats,
"\nMemory Data:",
data
);
return {
currentTokens,
chats: newChats,
memory: toSerializableHypaV3Data(data),
};
}
const selectedSummaries: Summary[] = [];
const randomMemoryRatio =
1 -

6
src/ts/storage/database.svelte.ts
View File

@@ -474,13 +474,14 @@ export function setDatabase(data:Database){
data.reasoningEffort ??= 0
data.hypaV3Settings = {
memoryTokensRatio: data.hypaV3Settings?.memoryTokensRatio ?? 0.2,
extraSummarizationRatio: data.hypaV3Settings?.extraSummarizationRatio ?? 0.2,
extraSummarizationRatio: data.hypaV3Settings?.extraSummarizationRatio ?? 0,
maxChatsPerSummary: data.hypaV3Settings?.maxChatsPerSummary ?? 4,
recentMemoryRatio: data.hypaV3Settings?.recentMemoryRatio ?? 0.4,
similarMemoryRatio: data.hypaV3Settings?.similarMemoryRatio ?? 0.4,
enableSimilarityCorrection: data.hypaV3Settings?.enableSimilarityCorrection ?? false,
preserveOrphanedMemory: data.hypaV3Settings?.preserveOrphanedMemory ?? false,
processRegexScript: data.hypaV3Settings?.processRegexScript ?? false
processRegexScript: data.hypaV3Settings?.processRegexScript ?? false,
doNotSummarizeUserMessage: data.hypaV3Settings?.doNotSummarizeUserMessage ?? false
}
changeLanguage(data.language)
setDatabaseLite(data)
@@ -894,6 +895,7 @@ export interface Database{
enableSimilarityCorrection: boolean
preserveOrphanedMemory: boolean
processRegexScript: boolean
doNotSummarizeUserMessage: boolean
},
OaiCompAPIKeys: {[key:string]:string}
inlayErrorResponse:boolean

4
src/ts/translator/translator.ts
View File

@@ -231,7 +231,9 @@ export async function translateHTML(html: string, reverse:boolean, charArg:simpl
let DoingChat = get(doingChat)
if(DoingChat){
if(isExpTranslator()){
return html
if(!(db.translatorType === 'llm' && await getLLMCache(html) !== null)){
return html
}
}
}
if(db.translatorType === 'llm'){