diff --git a/src/lib/Setting/Pages/UserSettings.svelte b/src/lib/Setting/Pages/UserSettings.svelte
index 95a4a6c5..2cdcb012 100644
--- a/src/lib/Setting/Pages/UserSettings.svelte
+++ b/src/lib/Setting/Pages/UserSettings.svelte
@@ -19,7 +19,7 @@
{
- if(e.origin.startsWith("https://sv.risuai.xyz") || e.origin.startsWith("http://127.0.0.1")){
+ if(e.origin.startsWith("https://sv.risuai.xyz") || e.origin.startsWith("http://127.0.0.1") || e.origin === window.location.origin){
if(e.data.msg.type === 'drive'){
await loadRisuAccountData()
DBState.db.account.data.refresh_token = e.data.msg.data.refresh_token
diff --git a/src/lib/SideBars/CharConfig.svelte b/src/lib/SideBars/CharConfig.svelte
index 7af6beca..c4c90a6e 100644
--- a/src/lib/SideBars/CharConfig.svelte
+++ b/src/lib/SideBars/CharConfig.svelte
@@ -1101,20 +1101,16 @@
}}
className="mt-4"
>
- {language.HypaMemory} V2 Data
+ {language.hypaMemoryV2Modal}
{:else if DBState.db.supaModelType !== 'none' && DBState.db.hypaV3}
{:else if DBState.db.characters[$selectedCharID].chats[DBState.db.characters[$selectedCharID].chatPage].supaMemoryData && DBState.db.characters[$selectedCharID].chats[DBState.db.characters[$selectedCharID].chatPage].supaMemoryData.length > 4 || DBState.db.characters[$selectedCharID].supaMemory}
{language.SuperMemory}
diff --git a/src/lib/SideBars/LoreBook/LoreBookList.svelte b/src/lib/SideBars/LoreBook/LoreBookList.svelte
index 89b1f455..213eec65 100644
--- a/src/lib/SideBars/LoreBook/LoreBookList.svelte
+++ b/src/lib/SideBars/LoreBook/LoreBookList.svelte
@@ -11,9 +11,10 @@
globalMode?: boolean;
submenu?: number;
lorePlus?: boolean;
+ externalLoreBooks?: loreBook[];
}
- let { globalMode = false, submenu = 0, lorePlus = false }: Props = $props();
+ let { globalMode = false, submenu = 0, lorePlus = false, externalLoreBooks = null }: Props = $props();
let stb: Sortable = null
let ele: HTMLDivElement = $state()
let sorted = $state(0)
@@ -31,6 +32,13 @@
})
DBState.db.loreBook[DBState.db.loreBookPage].data = newLore
}
+ else if(externalLoreBooks){
+ let newLore:loreBook[] = []
+ idx.forEach((i) => {
+ newLore.push(externalLoreBooks[i])
+ })
+ externalLoreBooks = newLore
+ }
else if(submenu === 1){
let newLore:loreBook[] = []
idx.forEach((i) => {
@@ -97,6 +105,18 @@
}} onOpen={onOpen} onClose={onClose}/>
{/each}
{/if}
+ {:else if externalLoreBooks}
+ {#if externalLoreBooks.length === 0}
+ No Lorebook
+ {:else}
+ {#each externalLoreBooks as book, i}
+ {
+ let lore = externalLoreBooks
+ lore.splice(i, 1)
+ externalLoreBooks = lore
+ }} onOpen={onOpen} onClose={onClose}/>
+ {/each}
+ {/if}
{:else if submenu === 0}
{#if DBState.db.characters[$selectedCharID].globalLore.length === 0}
No Lorebook
diff --git a/src/lib/UI/GUI/LazyPortal.svelte b/src/lib/UI/GUI/LazyPortal.svelte
new file mode 100644
index 00000000..5d70245d
--- /dev/null
+++ b/src/lib/UI/GUI/LazyPortal.svelte
@@ -0,0 +1,63 @@
+
\ No newline at end of file
diff --git a/src/ts/globalApi.svelte.ts b/src/ts/globalApi.svelte.ts
index 72c2feee..ed4d607d 100644
--- a/src/ts/globalApi.svelte.ts
+++ b/src/ts/globalApi.svelte.ts
@@ -87,7 +87,16 @@ export async function downloadFile(name:string, dat:Uint8Array|ArrayBuffer|strin
await writeFile(name, data, {baseDir: BaseDirectory.Download})
}
else{
- downloadURL(`data:png/image;base64,${Buffer.from(data).toString('base64')}`, name)
+ const blob = new Blob([data], { type: 'application/octet-stream' })
+ const url = URL.createObjectURL(blob)
+
+ downloadURL(url, name)
+
+ setTimeout(() => {
+ URL.revokeObjectURL(url)
+ }, 10000)
+
+
}
}
diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts
index 3ad61a3b..b9767916 100644
--- a/src/ts/parser.svelte.ts
+++ b/src/ts/parser.svelte.ts
@@ -1,4 +1,4 @@
-import DOMPurify from 'isomorphic-dompurify';
+import DOMPurify from 'dompurify';
import markdownit from 'markdown-it'
import { getCurrentCharacter, type Database, type Message, type character, type customscript, type groupChat, type triggerscript } from './storage/database.svelte';
import { DBState } from './stores.svelte';
@@ -650,6 +650,11 @@ function decodeStyleRule(rule:CssAtRuleAST){
rule.rules[i] = decodeStyleRule(rule.rules[i])
}
}
+ if(rule.type === 'import'){
+ if(rule.import.startsWith('data:')){
+ rule.import = 'data:,'
+ }
+ }
return rule
}
@@ -1234,6 +1239,11 @@ function basicMatcher (p1:string,matcherArg:matcherArg,vars:{[key:string]:string
case 'screen_height':{
return get(SizeStore).h.toString()
}
+ case 'cbr':
+ case 'cnl':
+ case 'cnewline':{
+ return '\\n'
+ }
}
const arra = p1.split("::")
if(arra.length > 1){
@@ -1672,6 +1682,38 @@ function basicMatcher (p1:string,matcherArg:matcherArg,vars:{[key:string]:string
case 'hash':{
return ((pickHashRand(0, arra[1]) * 10000000) + 1).toFixed(0).padStart(7, '0')
}
+ case 'randint':{
+ const min = Number(arra[1])
+ const max = Number(arra[2])
+ if(isNaN(min) || isNaN(max)){
+ return 'NaN'
+ }
+ return (Math.floor(Math.random() * (max - min + 1)) + min).toString()
+ }
+ case 'cbr':
+ case 'cnl':
+ case 'cnewline':{
+ return '\\n'.repeat(Number(arra[1]))
+ }
+ case 'dice':{
+ const notation = arra[1].split('d')
+ const num = Number(notation[0])
+ const sides = Number(notation[1])
+ if(isNaN(num) || isNaN(sides)){
+ return 'NaN'
+ }
+ let total = 0
+ for(let i = 0; i < num; i++){
+ total += Math.floor(Math.random() * sides) + 1
+ }
+ return total.toString()
+ }
+ case 'fromhex':{
+ return Number.parseInt(arra[1], 16).toString()
+ }
+ case 'tohex':{
+ return Number.parseInt(arra[1]).toString(16)
+ }
}
}
if(p1.startsWith('random')){
@@ -1872,7 +1914,7 @@ const legacyBlockMatcher = (p1:string,matcherArg:matcherArg) => {
return null
}
-type blockMatch = 'ignore'|'parse'|'nothing'|'parse-pure'|'pure'|'each'|'function'|'pure-display'
+type blockMatch = 'ignore'|'parse'|'nothing'|'parse-pure'|'pure'|'each'|'function'|'pure-display'|'normalize'
function parseArray(p1:string):string[]{
try {
@@ -1918,6 +1960,10 @@ function blockStartMatcher(p1:string,matcherArg:matcherArg):{type:blockMatch,typ
if(p1 === '#pure_display' || p1 === '#puredisplay'){
return {type:'pure-display'}
}
+ if(p1 === '#code'){
+ return {type:'normalize'}
+
+ }
if(p1.startsWith('#each')){
let t2 = p1.substring(5).trim()
if(t2.startsWith('as ')){
@@ -1958,6 +2004,34 @@ function blockEndMatcher(p1:string,type:{type:blockMatch,type2?:string},matcherA
case 'parse-pure':{
return p1
}
+ case 'normalize':{
+ return p1Trimed.trim().replaceAll('\n','').replaceAll('\t','')
+ .replaceAll(/\\u([0-9A-Fa-f]{4})/g, (match, p1) => {
+ return String.fromCharCode(parseInt(p1, 16))
+ })
+ .replaceAll(/\\(.)/g, (match, p1) => {
+ switch(p1){
+ case 'n':
+ return '\n'
+ case 'r':
+ return '\r'
+ case 't':
+ return '\t'
+ case 'b':
+ return '\b'
+ case 'f':
+ return '\f'
+ case 'v':
+ return '\v'
+ case 'a':
+ return '\a'
+ case 'x':
+ return '\x00'
+ default:
+ return p1
+ }
+ })
+ }
default:{
return ''
}
@@ -2119,7 +2193,7 @@ export function risuChatParser(da:string, arg:{
break
}
}
- if(dat.startsWith('/')){
+ if(dat.startsWith('/') && !dat.startsWith('//')){
if(stackType[nested.length] === 5){
const blockType = blockNestType.get(nested.length)
if( blockType.type === 'ignore' || blockType.type === 'pure' ||
diff --git a/src/ts/process/index.svelte.ts b/src/ts/process/index.svelte.ts
index 08e02c07..65bb4157 100644
--- a/src/ts/process/index.svelte.ts
+++ b/src/ts/process/index.svelte.ts
@@ -59,9 +59,15 @@ export interface OpenAIChatFull extends OpenAIChat{
}
}
+export interface requestTokenPart{
+ name:string
+ tokens:number
+}
+
export const doingChat = writable(false)
export const chatProcessStage = writable(0)
export const abortChat = writable(false)
+export let requestTokenParts:{[key:string]:requestTokenPart[]} = {}
export let previewFormated:OpenAIChat[] = []
export let previewBody:string = ''
diff --git a/src/ts/process/lorebook.svelte.ts b/src/ts/process/lorebook.svelte.ts
index 5e578be7..d428c6e1 100644
--- a/src/ts/process/lorebook.svelte.ts
+++ b/src/ts/process/lorebook.svelte.ts
@@ -55,7 +55,8 @@ export async function loadLoreBookV3Prompt(){
const recursiveScanning = char.loreSettings?.recursiveScanning ?? true
let recursivePrompt:{
prompt: string,
- source: string
+ source: string,
+ data: string
}[] = []
let matchLog:{
prompt: string,
@@ -75,23 +76,27 @@ export async function loadLoreBookV3Prompt(){
let mList:{
source:string
prompt:string
+ data:string
}[] = sliced.map((msg, i) => {
if(msg.role === 'user'){
return {
source: `message ${i} by user`,
- prompt: `\x01{{${DBState.db.username}}}:` + msg.data + '\x01'
+ prompt: `\x01{{${DBState.db.username}}}:` + msg.data + '\x01',
+ data: msg.data
}
}
else{
return {
source: `message ${i} by char`,
- prompt: `\x01{{${msg.name ?? (msg.saying ? findCharacterbyId(msg.saying)?.name : null) ?? char.name}}}:` + msg.data + '\x01'
+ prompt: `\x01{{${msg.name ?? (msg.saying ? findCharacterbyId(msg.saying)?.name : null) ?? char.name}}}:` + msg.data + '\x01',
+ data: msg.data
}
}
}).concat(recursivePrompt.map((msg) => {
return {
source: 'lorebook ' + msg.source,
- prompt: msg.prompt
+ prompt: msg.prompt,
+ data: msg.data
}
}))
@@ -106,7 +111,7 @@ export async function loadLoreBookV3Prompt(){
arg.keys[0] = regexString.replace('/'+regexFlag,'')
try {
const regex = new RegExp(arg.keys[0],regexFlag)
- const d = regex.test(mText.prompt)
+ const d = regex.test(mText.data)
if(d){
matchLog.push({
prompt: mText.prompt,
@@ -127,7 +132,8 @@ export async function loadLoreBookV3Prompt(){
mList = mList.map((m) => {
return {
source: m.source,
- prompt: m.prompt.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,'')
+ prompt: m.prompt.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,''),
+ data: m.data.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,'')
}
})
@@ -135,7 +141,7 @@ export async function loadLoreBookV3Prompt(){
let allModeMatched = true
for(const m of mList){
- let mText = m.prompt
+ let mText = m.data
if(arg.fullWordMatching){
const splited = mText.split(' ')
for(const key of arg.keys){
@@ -510,7 +516,7 @@ export async function importLoreBook(mode:'global'|'local'|'sglobal'){
}
}
-interface CCLorebook{
+export interface CCLorebook{
key:string[]
comment:string
content:string
diff --git a/src/ts/process/lua.ts b/src/ts/process/lua.ts
index 727cfb59..bc737239 100644
--- a/src/ts/process/lua.ts
+++ b/src/ts/process/lua.ts
@@ -13,11 +13,14 @@ import { v4 } from "uuid";
import { getModuleTriggers } from "./modules";
import { Mutex } from "../mutex";
import { tokenize } from "../tokenizer";
+import { fetchNative } from "../globalApi.svelte";
let luaFactory:LuaFactory
let LuaSafeIds = new Set()
let LuaEditDisplayIds = new Set()
let LuaLowLevelIds = new Set()
+let lastRequestResetTime = 0
+let lastRequestsCount = 0
interface LuaEngineState {
code?: string;
@@ -205,6 +208,75 @@ export async function runLua(code:string, arg:{
return await processer.similaritySearch(source)
})
+ luaEngine.global.set('request', async (id:string, url:string) => {
+ if(!LuaLowLevelIds.has(id)){
+ return
+ }
+
+ if(lastRequestResetTime + 60000 < Date.now()){
+ lastRequestsCount = 0
+ lastRequestResetTime = Date.now()
+ }
+
+ if(lastRequestsCount > 5){
+ return {
+ status: 429,
+ data: 'Too many requests. you can request 5 times per minute'
+ }
+ }
+
+ lastRequestsCount++
+
+ try {
+ //for security and other reasons, only get request in 120 char is allowed
+ if(url.length > 120){
+ return {
+ status: 413,
+ data: 'URL to large. max is 120 characters'
+ }
+ }
+
+ if(!url.startsWith('https://')){
+ return {
+ status: 400,
+ data: "Only https requests are allowed"
+ }
+ }
+
+ const bannedURL = [
+ "https://realm.risuai.net",
+ "https://risuai.net",
+ "https://risuai.xyz"
+ ]
+
+ for(const burl of bannedURL){
+
+ if(url.startsWith(burl)){
+ return {
+ status: 400,
+ data: "request to " + url + ' is not allowed'
+ }
+ }
+ }
+
+ //browser fetch
+ const d = await fetchNative(url, {
+ method: "GET"
+ })
+ const text = await d.text()
+ return {
+ status: d.status,
+ data: text
+ }
+
+ } catch (error) {
+ return {
+ status: 400,
+ data: 'internal error'
+ }
+ }
+ })
+
luaEngine.global.set('generateImage', async (id:string, value:string, negValue:string = '') => {
if(!LuaLowLevelIds.has(id)){
return
diff --git a/src/ts/process/scripts.ts b/src/ts/process/scripts.ts
index 3c95ab81..ed4c9ed7 100644
--- a/src/ts/process/scripts.ts
+++ b/src/ts/process/scripts.ts
@@ -74,9 +74,7 @@ function generateScriptCacheKey(scripts: customscript[], data: string, mode: Scr
if(script.type !== mode){
continue
}
- hash += `${script.flag?.includes('') ?
- risuChatParser(script.in, { chatID: chatID, cbsConditions }) :
- script.in}|||${risuChatParser(script.out, { chatID: chatID, cbsConditions})}|||${script.flag ?? ''}|||${script.ableFlag ? 1 : 0}`;
+ hash += `${script.flag?.includes('') ? risuChatParser(script.in, { chatID: chatID, cbsConditions }) : script.in}|||${script.out}${chatID}|||${script.flag ?? ''}|||${script.ableFlag ? 1 : 0}`;
}
return hash;
}
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index e721c9b0..0d18f95e 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -12,7 +12,7 @@ import { defaultColorScheme, type ColorScheme } from '../gui/colorscheme';
import type { PromptItem, PromptSettings } from '../process/prompt';
import type { OobaChatCompletionRequestParams } from '../model/ooba';
-export let appVer = "158.2.1"
+export let appVer = "159.0.0"
export let webAppSubVer = ''
@@ -255,8 +255,10 @@ export function setDatabase(data:Database){
width:512,
height:768,
sampler:"k_dpmpp_sde",
+ noise_schedule:"native",
steps:28,
scale:5,
+ cfg_rescale: 0,
sm:true,
sm_dyn:false,
noise:0.0,
@@ -1026,6 +1028,8 @@ export interface Database{
flags: LLMFlags[]
}[]
igpPrompt:string
+ useTokenizerCaching:boolean
+ showMenuHypaMemoryModal:boolean
}
interface SeparateParameters{
@@ -1411,8 +1415,10 @@ export interface NAIImgConfig{
width:number,
height:number,
sampler:string,
+ noise_schedule:string,
steps:number,
scale:number,
+ cfg_rescale:number,
sm:boolean,
sm_dyn:boolean,
noise:number,
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index 3211c878..25fb442d 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -47,18 +47,21 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
const modelInfo = getModelInfo(db.aiModel);
const pluginTokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer ?? "none";
- const cacheKey = getHash(
- data,
- db.aiModel,
- db.customTokenizer,
- db.currentPluginProvider,
- db.googleClaudeTokenizing,
- modelInfo,
- pluginTokenizer
- );
- const cachedResult = encodeCache.get(cacheKey);
- if (cachedResult !== undefined) {
- return cachedResult;
+ let cacheKey = ''
+ if(db.useTokenizerCaching){
+ cacheKey = getHash(
+ data,
+ db.aiModel,
+ db.customTokenizer,
+ db.currentPluginProvider,
+ db.googleClaudeTokenizing,
+ modelInfo,
+ pluginTokenizer
+ );
+ const cachedResult = encodeCache.get(cacheKey);
+ if (cachedResult !== undefined) {
+ return cachedResult;
+ }
}
let result: number[] | Uint32Array | Int32Array;
@@ -86,9 +89,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
default:
result = await tikJS(data, 'o200k_base'); break;
}
- }
-
- if(db.aiModel === 'custom' && pluginTokenizer){
+ } else if (db.aiModel === 'custom' && pluginTokenizer) {
switch(pluginTokenizer){
case 'mistral':
result = await tokenizeWebTokenizers(data, 'mistral'); break;
@@ -117,32 +118,37 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
}
}
- if(modelInfo.tokenizer === LLMTokenizer.NovelList){
- result = await tokenizeWebTokenizers(data, 'novellist');
- } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
- result = await tokenizeWebTokenizers(data, 'claude');
- } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
- result = await tokenizeWebTokenizers(data, 'novelai');
- } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
- result = await tokenizeWebTokenizers(data, 'mistral');
- } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
- result = await tokenizeWebTokenizers(data, 'llama');
- } else if(modelInfo.tokenizer === LLMTokenizer.Local){
- result = await tokenizeGGUFModel(data);
- } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
- result = await tikJS(data, 'o200k_base');
- } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
- result = await tokenizeGoogleCloud(data);
- } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
- result = await gemmaTokenize(data);
- } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
- result = await tokenizeWebTokenizers(data, 'DeepSeek');
- } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
- result = await tokenizeWebTokenizers(data, 'cohere');
- } else {
- result = await tikJS(data);
+ // Fallback
+ if (result === undefined) {
+ if(modelInfo.tokenizer === LLMTokenizer.NovelList){
+ result = await tokenizeWebTokenizers(data, 'novellist');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
+ result = await tokenizeWebTokenizers(data, 'claude');
+ } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
+ result = await tokenizeWebTokenizers(data, 'novelai');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
+ result = await tokenizeWebTokenizers(data, 'mistral');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
+ result = await tokenizeWebTokenizers(data, 'llama');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Local){
+ result = await tokenizeGGUFModel(data);
+ } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
+ result = await tikJS(data, 'o200k_base');
+ } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
+ result = await tokenizeGoogleCloud(data);
+ } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
+ result = await gemmaTokenize(data);
+ } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
+ result = await tokenizeWebTokenizers(data, 'DeepSeek');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
+ result = await tokenizeWebTokenizers(data, 'cohere');
+ } else {
+ result = await tikJS(data);
+ }
+ }
+ if(db.useTokenizerCaching){
+ encodeCache.set(cacheKey, result);
}
- encodeCache.set(cacheKey, result);
return result;
}
@@ -203,6 +209,7 @@ async function gemmaTokenize(text:string) {
async function tikJS(text:string, model='cl100k_base') {
if(!tikParser || lastTikModel !== model){
+ tikParser?.free()
if(model === 'cl100k_base'){
const {Tiktoken} = await import('@dqbd/tiktoken')
const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");
diff --git a/version.json b/version.json
index 3f89e9b8..4e3840fb 100644
--- a/version.json
+++ b/version.json
@@ -1 +1 @@
-{"version":"158.2.1"}
\ No newline at end of file
+{"version":"159.0.0"}
\ No newline at end of file