Merge remote-tracking branch 'origin/main'

# Conflicts: # src/ts/process/stableDiff.ts
2025-05-03 16:41:56 +09:00
parent 5b0ca460a5 6db9bd08f1
commit 133156d7d4
33 changed files with 725 additions and 647 deletions
--- a/src/ts/globalApi.svelte.ts
+++ b/src/ts/globalApi.svelte.ts
@@ -87,7 +87,16 @@ export async function downloadFile(name:string, dat:Uint8Array|ArrayBuffer|strin
        await writeFile(name, data, {baseDir: BaseDirectory.Download})
    }
    else{
-        downloadURL(`data:png/image;base64,${Buffer.from(data).toString('base64')}`, name)
+        const blob = new Blob([data], { type: 'application/octet-stream' })
+        const url = URL.createObjectURL(blob)
+
+        downloadURL(url, name)
+
+        setTimeout(() => {
+            URL.revokeObjectURL(url)
+        }, 10000)
+
+        
    }
 }

--- a/src/ts/parser.svelte.ts
+++ b/src/ts/parser.svelte.ts
@@ -1,4 +1,4 @@
-import DOMPurify from 'isomorphic-dompurify';
+import DOMPurify from 'dompurify';
 import markdownit from 'markdown-it'
 import { getCurrentCharacter, type Database, type Message, type character, type customscript, type groupChat, type triggerscript } from './storage/database.svelte';
 import { DBState } from './stores.svelte';
@@ -650,6 +650,11 @@ function decodeStyleRule(rule:CssAtRuleAST){
            rule.rules[i] = decodeStyleRule(rule.rules[i])
        }
    }
+    if(rule.type === 'import'){
+       if(rule.import.startsWith('data:')){
+            rule.import = 'data:,'
+       }
+    }
    return rule
 }

@@ -1234,6 +1239,11 @@ function basicMatcher (p1:string,matcherArg:matcherArg,vars:{[key:string]:string
            case 'screen_height':{
                return get(SizeStore).h.toString()
            }
+            case 'cbr':
+            case 'cnl':
+            case 'cnewline':{
+                return '\\n'
+            }
        }
        const arra = p1.split("::")
        if(arra.length > 1){
@@ -1672,6 +1682,38 @@ function basicMatcher (p1:string,matcherArg:matcherArg,vars:{[key:string]:string
                case 'hash':{
                    return ((pickHashRand(0, arra[1]) * 10000000) + 1).toFixed(0).padStart(7, '0')
                }
+                case 'randint':{
+                    const min = Number(arra[1])
+                    const max = Number(arra[2])
+                    if(isNaN(min) || isNaN(max)){
+                        return 'NaN'
+                    }
+                    return (Math.floor(Math.random() * (max - min + 1)) + min).toString()
+                }
+                case 'cbr':
+                case 'cnl':
+                case 'cnewline':{
+                    return '\\n'.repeat(Number(arra[1]))
+                }
+                case 'dice':{
+                    const notation = arra[1].split('d')
+                    const num = Number(notation[0])
+                    const sides = Number(notation[1])
+                    if(isNaN(num) || isNaN(sides)){
+                        return 'NaN'
+                    }
+                    let total = 0
+                    for(let i = 0; i < num; i++){
+                        total += Math.floor(Math.random() * sides) + 1
+                    }
+                    return total.toString()
+                }
+                case 'fromhex':{
+                    return Number.parseInt(arra[1], 16).toString()
+                }
+                case 'tohex':{
+                    return Number.parseInt(arra[1]).toString(16)
+                }
            }
        }
        if(p1.startsWith('random')){
@@ -1872,7 +1914,7 @@ const legacyBlockMatcher = (p1:string,matcherArg:matcherArg) => {
    return null
 }

-type blockMatch = 'ignore'|'parse'|'nothing'|'parse-pure'|'pure'|'each'|'function'|'pure-display'
+type blockMatch = 'ignore'|'parse'|'nothing'|'parse-pure'|'pure'|'each'|'function'|'pure-display'|'normalize'

 function parseArray(p1:string):string[]{
    try {
@@ -1918,6 +1960,10 @@ function blockStartMatcher(p1:string,matcherArg:matcherArg):{type:blockMatch,typ
    if(p1 === '#pure_display' || p1 === '#puredisplay'){
        return {type:'pure-display'}
    }
+    if(p1 === '#code'){
+        return {type:'normalize'}
+
+    }
    if(p1.startsWith('#each')){
        let t2 = p1.substring(5).trim()
        if(t2.startsWith('as ')){
@@ -1958,6 +2004,34 @@ function blockEndMatcher(p1:string,type:{type:blockMatch,type2?:string},matcherA
        case 'parse-pure':{
            return p1
        }
+        case 'normalize':{
+            return p1Trimed.trim().replaceAll('\n','').replaceAll('\t','')
+            .replaceAll(/\\u([0-9A-Fa-f]{4})/g, (match, p1) => {
+                return String.fromCharCode(parseInt(p1, 16))
+            })
+            .replaceAll(/\\(.)/g, (match, p1) => {
+                switch(p1){
+                    case 'n':
+                        return '\n'
+                    case 'r':
+                        return '\r'
+                    case 't':
+                        return '\t'
+                    case 'b':
+                        return '\b'
+                    case 'f':
+                        return '\f'
+                    case 'v':
+                        return '\v'
+                    case 'a':
+                        return '\a'
+                    case 'x':
+                        return '\x00'
+                    default:
+                        return p1
+                }
+            })
+        }
        default:{
            return ''
        }
@@ -2119,7 +2193,7 @@ export function risuChatParser(da:string, arg:{
                        break
                    }
                }
-                if(dat.startsWith('/')){
+                if(dat.startsWith('/') && !dat.startsWith('//')){
                    if(stackType[nested.length] === 5){
                        const blockType = blockNestType.get(nested.length)
                        if( blockType.type === 'ignore' || blockType.type === 'pure' ||
--- a/src/ts/process/index.svelte.ts
+++ b/src/ts/process/index.svelte.ts
@@ -59,9 +59,15 @@ export interface OpenAIChatFull extends OpenAIChat{
    }
 }

+export interface requestTokenPart{
+    name:string
+    tokens:number
+}
+
 export const doingChat = writable(false)
 export const chatProcessStage = writable(0)
 export const abortChat = writable(false)
+export let requestTokenParts:{[key:string]:requestTokenPart[]} = {}
 export let previewFormated:OpenAIChat[] = []
 export let previewBody:string = ''

--- a/src/ts/process/lorebook.svelte.ts
+++ b/src/ts/process/lorebook.svelte.ts
@@ -55,7 +55,8 @@ export async function loadLoreBookV3Prompt(){
    const recursiveScanning = char.loreSettings?.recursiveScanning ?? true
    let recursivePrompt:{
        prompt: string,
-        source: string
+        source: string,
+        data: string
    }[] = []
    let matchLog:{
        prompt: string,
@@ -75,23 +76,27 @@ export async function loadLoreBookV3Prompt(){
        let mList:{
            source:string
            prompt:string
+            data:string
        }[] = sliced.map((msg, i) => {
            if(msg.role === 'user'){
                return {
                    source: `message ${i} by user`,
-                    prompt: `\x01{{${DBState.db.username}}}:` + msg.data + '\x01'
+                    prompt: `\x01{{${DBState.db.username}}}:` + msg.data + '\x01',
+                    data: msg.data
                }
            }
            else{
                return {
                    source: `message ${i} by char`,
-                    prompt: `\x01{{${msg.name ?? (msg.saying ? findCharacterbyId(msg.saying)?.name : null) ?? char.name}}}:` + msg.data + '\x01'
+                    prompt: `\x01{{${msg.name ?? (msg.saying ? findCharacterbyId(msg.saying)?.name : null) ?? char.name}}}:` + msg.data + '\x01',
+                    data: msg.data
                }
            }
        }).concat(recursivePrompt.map((msg) => {
            return {
                source: 'lorebook ' + msg.source,
-                prompt: msg.prompt
+                prompt: msg.prompt,
+                data: msg.data
            }
        }))

@@ -106,7 +111,7 @@ export async function loadLoreBookV3Prompt(){
                        arg.keys[0] = regexString.replace('/'+regexFlag,'')
                        try {
                            const regex = new RegExp(arg.keys[0],regexFlag)
-                            const d = regex.test(mText.prompt)
+                            const d = regex.test(mText.data)
                            if(d){
                                matchLog.push({
                                    prompt: mText.prompt,
@@ -127,7 +132,8 @@ export async function loadLoreBookV3Prompt(){
        mList = mList.map((m) => {
            return {
                source: m.source,
-                prompt: m.prompt.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,'')
+                prompt: m.prompt.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,''),
+                data: m.data.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,'')
            }
        })

@@ -135,7 +141,7 @@ export async function loadLoreBookV3Prompt(){
        let allModeMatched = true

        for(const m of mList){
-            let mText = m.prompt
+            let mText = m.data
            if(arg.fullWordMatching){
                const splited = mText.split(' ')
                for(const key of arg.keys){
@@ -510,7 +516,7 @@ export async function importLoreBook(mode:'global'|'local'|'sglobal'){
    }
 }

-interface CCLorebook{
+export interface CCLorebook{
    key:string[]
    comment:string
    content:string
--- a/src/ts/process/lua.ts
+++ b/src/ts/process/lua.ts
@@ -13,11 +13,14 @@ import { v4 } from "uuid";
 import { getModuleTriggers } from "./modules";
 import { Mutex } from "../mutex";
 import { tokenize } from "../tokenizer";
+import { fetchNative } from "../globalApi.svelte";

 let luaFactory:LuaFactory
 let LuaSafeIds = new Set<string>()
 let LuaEditDisplayIds = new Set<string>()
 let LuaLowLevelIds = new Set<string>()
+let lastRequestResetTime = 0
+let lastRequestsCount = 0

 interface LuaEngineState {
    code?: string;
@@ -205,6 +208,75 @@ export async function runLua(code:string, arg:{
                return await processer.similaritySearch(source)
            })

+            luaEngine.global.set('request', async (id:string, url:string) => {
+                if(!LuaLowLevelIds.has(id)){
+                    return
+                }
+
+                if(lastRequestResetTime + 60000 < Date.now()){
+                    lastRequestsCount = 0
+                    lastRequestResetTime = Date.now()
+                }
+                
+                if(lastRequestsCount > 5){
+                    return {
+                        status: 429,
+                        data: 'Too many requests. you can request 5 times per minute'
+                    }
+                }
+
+                lastRequestsCount++
+
+                try {
+                    //for security and other reasons, only get request in 120 char is allowed
+                    if(url.length > 120){
+                        return {
+                            status: 413,
+                            data: 'URL to large. max is 120 characters'
+                        }
+                    }
+
+                    if(!url.startsWith('https://')){
+                        return {
+                            status: 400,
+                            data: "Only https requests are allowed"
+                        }
+                    }
+
+                    const bannedURL = [
+                        "https://realm.risuai.net",
+                        "https://risuai.net",
+                        "https://risuai.xyz"
+                    ]
+
+                    for(const burl of bannedURL){
+
+                        if(url.startsWith(burl)){
+                            return {
+                                status: 400,
+                                data: "request to " + url + ' is not allowed'
+                            }
+                        }
+                    }
+
+                    //browser fetch
+                    const d = await fetchNative(url, {
+                        method: "GET"
+                    })
+                    const text = await d.text()
+                    return {
+                        status: d.status,
+                        data: text
+                    }
+
+                } catch (error) {
+                    return {
+                        status: 400,
+                        data: 'internal error'
+                    }
+                }
+            })
+
            luaEngine.global.set('generateImage', async (id:string, value:string, negValue:string = '') => {
                if(!LuaLowLevelIds.has(id)){
                    return
--- a/src/ts/process/scripts.ts
+++ b/src/ts/process/scripts.ts
@@ -74,9 +74,7 @@ function generateScriptCacheKey(scripts: customscript[], data: string, mode: Scr
        if(script.type !== mode){
            continue
        }
-        hash += `${script.flag?.includes('<cbs>') ? 
-            risuChatParser(script.in, { chatID: chatID, cbsConditions }) : 
-            script.in}|||${risuChatParser(script.out, { chatID: chatID, cbsConditions})}|||${script.flag ?? ''}|||${script.ableFlag ? 1 : 0}`;
+        hash += `${script.flag?.includes('<cbs>') ? risuChatParser(script.in, { chatID: chatID, cbsConditions }) : script.in}|||${script.out}${chatID}|||${script.flag ?? ''}|||${script.ableFlag ? 1 : 0}`;
    }
    return hash;
 }
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -12,7 +12,7 @@ import { defaultColorScheme, type ColorScheme } from '../gui/colorscheme';
 import type { PromptItem, PromptSettings } from '../process/prompt';
 import type { OobaChatCompletionRequestParams } from '../model/ooba';

-export let appVer = "158.2.1"
+export let appVer = "159.0.0"
 export let webAppSubVer = ''


@@ -255,8 +255,10 @@ export function setDatabase(data:Database){
            width:512,
            height:768,
            sampler:"k_dpmpp_sde",
+            noise_schedule:"native",
            steps:28,
            scale:5,
+            cfg_rescale: 0,
            sm:true,
            sm_dyn:false,
            noise:0.0,
@@ -1026,6 +1028,8 @@ export interface Database{
        flags: LLMFlags[]
    }[]
    igpPrompt:string
+    useTokenizerCaching:boolean
+    showMenuHypaMemoryModal:boolean
 }

 interface SeparateParameters{
@@ -1411,8 +1415,10 @@ export interface NAIImgConfig{
    width:number,
    height:number,
    sampler:string,
+    noise_schedule:string,
    steps:number,
    scale:number,
+    cfg_rescale:number,
    sm:boolean,
    sm_dyn:boolean,
    noise:number,
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -47,18 +47,21 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
    const modelInfo = getModelInfo(db.aiModel);
    const pluginTokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer ?? "none";

-    const cacheKey = getHash(
-        data,
-        db.aiModel,
-        db.customTokenizer,
-        db.currentPluginProvider,
-        db.googleClaudeTokenizing,
-        modelInfo,
-        pluginTokenizer
-    );
-    const cachedResult = encodeCache.get(cacheKey);
-    if (cachedResult !== undefined) {
-        return cachedResult;
+    let cacheKey = ''
+    if(db.useTokenizerCaching){
+        cacheKey = getHash(
+            data,
+            db.aiModel,
+            db.customTokenizer,
+            db.currentPluginProvider,
+            db.googleClaudeTokenizing,
+            modelInfo,
+            pluginTokenizer
+        );
+        const cachedResult = encodeCache.get(cacheKey);
+        if (cachedResult !== undefined) {
+            return cachedResult;
+        }
    }

    let result: number[] | Uint32Array | Int32Array;
@@ -86,9 +89,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
            default:
                result = await tikJS(data, 'o200k_base'); break;
        }
-    }
-    
-    if(db.aiModel === 'custom' && pluginTokenizer){
+    } else if (db.aiModel === 'custom' && pluginTokenizer) {
        switch(pluginTokenizer){
            case 'mistral':
                result = await tokenizeWebTokenizers(data, 'mistral'); break;
@@ -117,32 +118,37 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
        }
    } 
    
-    if(modelInfo.tokenizer === LLMTokenizer.NovelList){
-        result = await tokenizeWebTokenizers(data, 'novellist');
-    } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
-        result = await tokenizeWebTokenizers(data, 'claude');
-    } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
-        result = await tokenizeWebTokenizers(data, 'novelai');
-    } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
-        result = await tokenizeWebTokenizers(data, 'mistral');
-    } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
-        result = await tokenizeWebTokenizers(data, 'llama');
-    } else if(modelInfo.tokenizer === LLMTokenizer.Local){
-        result = await tokenizeGGUFModel(data);
-    } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
-        result = await tikJS(data, 'o200k_base');
-    } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
-        result = await tokenizeGoogleCloud(data);
-    } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
-        result = await gemmaTokenize(data);
-    } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
-        result = await tokenizeWebTokenizers(data, 'DeepSeek');
-    } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
-        result = await tokenizeWebTokenizers(data, 'cohere');
-    } else {
-        result = await tikJS(data);
+    // Fallback
+    if (result === undefined) {
+        if(modelInfo.tokenizer === LLMTokenizer.NovelList){
+            result = await tokenizeWebTokenizers(data, 'novellist');
+        } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
+            result = await tokenizeWebTokenizers(data, 'claude');
+        } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
+            result = await tokenizeWebTokenizers(data, 'novelai');
+        } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
+            result = await tokenizeWebTokenizers(data, 'mistral');
+        } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
+            result = await tokenizeWebTokenizers(data, 'llama');
+        } else if(modelInfo.tokenizer === LLMTokenizer.Local){
+            result = await tokenizeGGUFModel(data);
+        } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
+            result = await tikJS(data, 'o200k_base');
+        } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
+            result = await tokenizeGoogleCloud(data);
+        } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
+            result = await gemmaTokenize(data);
+        } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
+            result = await tokenizeWebTokenizers(data, 'DeepSeek');
+        } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
+            result = await tokenizeWebTokenizers(data, 'cohere');
+        } else {
+            result = await tikJS(data);
+        }
+    }
+    if(db.useTokenizerCaching){
+        encodeCache.set(cacheKey, result);
    }
-    encodeCache.set(cacheKey, result);

    return result;
 }
@@ -203,6 +209,7 @@ async function gemmaTokenize(text:string) {

 async function tikJS(text:string, model='cl100k_base') {
    if(!tikParser || lastTikModel !== model){
+        tikParser?.free()
        if(model === 'cl100k_base'){
            const {Tiktoken} = await import('@dqbd/tiktoken')
            const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");