diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json
index d4aa0621..3df199a8 100644
--- a/src-tauri/tauri.conf.json
+++ b/src-tauri/tauri.conf.json
@@ -29,7 +29,7 @@
},
"productName": "RisuAI",
"mainBinaryName": "RisuAI",
- "version": "158.2.1",
+ "version": "159.0.0",
"identifier": "co.aiclient.risu",
"plugins": {
"updater": {
diff --git a/src/lang/en.ts b/src/lang/en.ts
index f410a812..d20ba4be 100644
--- a/src/lang/en.ts
+++ b/src/lang/en.ts
@@ -1117,6 +1117,7 @@ export const languageEnglish = {
doNotChangeFallbackModels: "Do Not Change Fallback Models on Preset Change",
customModels: "Custom Models",
igpPrompt: "IGP Prompt",
+ useTokenizerCaching: "Tokenizer Caching",
hypaMemoryV2Modal: "Hypa V2 Modal",
hypaMemoryV3Modal: "Hypa V3 Modal",
showMenuHypaMemoryModal: "Show Menu Hypa Modal",
diff --git a/src/lib/Setting/Pages/AdvancedSettings.svelte b/src/lib/Setting/Pages/AdvancedSettings.svelte
index 805cd9dc..49a50881 100644
--- a/src/lib/Setting/Pages/AdvancedSettings.svelte
+++ b/src/lib/Setting/Pages/AdvancedSettings.svelte
@@ -207,6 +207,10 @@
+
+
+
+
{#if DBState.db.useExperimental}
diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts
index 3ad61a3b..54824838 100644
--- a/src/ts/parser.svelte.ts
+++ b/src/ts/parser.svelte.ts
@@ -650,6 +650,11 @@ function decodeStyleRule(rule:CssAtRuleAST){
rule.rules[i] = decodeStyleRule(rule.rules[i])
}
}
+ if(rule.type === 'import'){
+ if(rule.import.startsWith('data:')){
+ rule.import = 'data:,'
+ }
+ }
return rule
}
diff --git a/src/ts/process/index.svelte.ts b/src/ts/process/index.svelte.ts
index 08e02c07..65bb4157 100644
--- a/src/ts/process/index.svelte.ts
+++ b/src/ts/process/index.svelte.ts
@@ -59,9 +59,15 @@ export interface OpenAIChatFull extends OpenAIChat{
}
}
+export interface requestTokenPart{
+ name:string
+ tokens:number
+}
+
export const doingChat = writable(false)
export const chatProcessStage = writable(0)
export const abortChat = writable(false)
+export let requestTokenParts:{[key:string]:requestTokenPart[]} = {}
export let previewFormated:OpenAIChat[] = []
export let previewBody:string = ''
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index 6c7cc284..57978e9e 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -12,7 +12,7 @@ import { defaultColorScheme, type ColorScheme } from '../gui/colorscheme';
import type { PromptItem, PromptSettings } from '../process/prompt';
import type { OobaChatCompletionRequestParams } from '../model/ooba';
-export let appVer = "158.2.1"
+export let appVer = "159.0.0"
export let webAppSubVer = ''
@@ -1022,6 +1022,7 @@ export interface Database{
flags: LLMFlags[]
}[]
igpPrompt:string
+ useTokenizerCaching:boolean
showMenuHypaMemoryModal:boolean
}
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index 3211c878..e71528a4 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -6,27 +6,9 @@ import { supportsInlayImage } from "./process/files/inlays";
import { risuChatParser } from "./parser.svelte";
import { tokenizeGGUFModel } from "./process/models/local";
import { globalFetch } from "./globalApi.svelte";
-import { getModelInfo, LLMTokenizer, type LLMModel } from "./model/modellist";
+import { getModelInfo, LLMTokenizer } from "./model/modellist";
import { pluginV2 } from "./plugins/plugins";
import type { GemmaTokenizer } from "@huggingface/transformers";
-import { LRUMap } from 'mnemonist';
-
-const MAX_CACHE_SIZE = 1500;
-
-const encodeCache = new LRUMap(MAX_CACHE_SIZE);
-
-function getHash(
- data: string,
- aiModel: string,
- customTokenizer: string,
- currentPluginProvider: string,
- googleClaudeTokenizing: boolean,
- modelInfo: LLMModel,
- pluginTokenizer: string
-): string {
- const combined = `${data}::${aiModel}::${customTokenizer}::${currentPluginProvider}::${googleClaudeTokenizing ? '1' : '0'}::${modelInfo.tokenizer}::${pluginTokenizer}`;
- return combined;
-}
export const tokenizerList = [
@@ -43,108 +25,100 @@ export const tokenizerList = [
] as const
export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
- const db = getDatabase();
- const modelInfo = getModelInfo(db.aiModel);
- const pluginTokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer ?? "none";
-
- const cacheKey = getHash(
- data,
- db.aiModel,
- db.customTokenizer,
- db.currentPluginProvider,
- db.googleClaudeTokenizing,
- modelInfo,
- pluginTokenizer
- );
- const cachedResult = encodeCache.get(cacheKey);
- if (cachedResult !== undefined) {
- return cachedResult;
- }
-
- let result: number[] | Uint32Array | Int32Array;
-
+ let db = getDatabase()
if(db.aiModel === 'openrouter' || db.aiModel === 'reverse_proxy'){
switch(db.customTokenizer){
case 'mistral':
- result = await tokenizeWebTokenizers(data, 'mistral'); break;
+ return await tokenizeWebTokenizers(data, 'mistral')
case 'llama':
- result = await tokenizeWebTokenizers(data, 'llama'); break;
+ return await tokenizeWebTokenizers(data, 'llama')
case 'novelai':
- result = await tokenizeWebTokenizers(data, 'novelai'); break;
+ return await tokenizeWebTokenizers(data, 'novelai')
case 'claude':
- result = await tokenizeWebTokenizers(data, 'claude'); break;
+ return await tokenizeWebTokenizers(data, 'claude')
case 'novellist':
- result = await tokenizeWebTokenizers(data, 'novellist'); break;
+ return await tokenizeWebTokenizers(data, 'novellist')
case 'llama3':
- result = await tokenizeWebTokenizers(data, 'llama'); break;
+ return await tokenizeWebTokenizers(data, 'llama')
case 'gemma':
- result = await gemmaTokenize(data); break;
+ return await gemmaTokenize(data)
case 'cohere':
- result = await tokenizeWebTokenizers(data, 'cohere'); break;
+ return await tokenizeWebTokenizers(data, 'cohere')
case 'deepseek':
- result = await tokenizeWebTokenizers(data, 'DeepSeek'); break;
+ return await tokenizeWebTokenizers(data, 'DeepSeek')
default:
- result = await tikJS(data, 'o200k_base'); break;
+ return await tikJS(data, 'o200k_base')
}
}
-
- if(db.aiModel === 'custom' && pluginTokenizer){
- switch(pluginTokenizer){
- case 'mistral':
- result = await tokenizeWebTokenizers(data, 'mistral'); break;
- case 'llama':
- result = await tokenizeWebTokenizers(data, 'llama'); break;
- case 'novelai':
- result = await tokenizeWebTokenizers(data, 'novelai'); break;
- case 'claude':
- result = await tokenizeWebTokenizers(data, 'claude'); break;
- case 'novellist':
- result = await tokenizeWebTokenizers(data, 'novellist'); break;
- case 'llama3':
- result = await tokenizeWebTokenizers(data, 'llama'); break;
- case 'gemma':
- result = await gemmaTokenize(data); break;
- case 'cohere':
- result = await tokenizeWebTokenizers(data, 'cohere'); break;
- case 'o200k_base':
- result = await tikJS(data, 'o200k_base'); break;
- case 'cl100k_base':
- result = await tikJS(data, 'cl100k_base'); break;
- case 'custom':
- result = await pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizerFunc?.(data) ?? [0]; break;
- default:
- result = await tikJS(data, 'o200k_base'); break;
- }
- }
-
- if(modelInfo.tokenizer === LLMTokenizer.NovelList){
- result = await tokenizeWebTokenizers(data, 'novellist');
- } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
- result = await tokenizeWebTokenizers(data, 'claude');
- } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
- result = await tokenizeWebTokenizers(data, 'novelai');
- } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
- result = await tokenizeWebTokenizers(data, 'mistral');
- } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
- result = await tokenizeWebTokenizers(data, 'llama');
- } else if(modelInfo.tokenizer === LLMTokenizer.Local){
- result = await tokenizeGGUFModel(data);
- } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
- result = await tikJS(data, 'o200k_base');
- } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
- result = await tokenizeGoogleCloud(data);
- } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
- result = await gemmaTokenize(data);
- } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
- result = await tokenizeWebTokenizers(data, 'DeepSeek');
- } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
- result = await tokenizeWebTokenizers(data, 'cohere');
- } else {
- result = await tikJS(data);
- }
- encodeCache.set(cacheKey, result);
- return result;
+ const modelInfo = getModelInfo(db.aiModel)
+
+ if(db.aiModel === 'custom' && pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer){
+ const tokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer
+ switch(tokenizer){
+ case 'mistral':
+ return await tokenizeWebTokenizers(data, 'mistral')
+ case 'llama':
+ return await tokenizeWebTokenizers(data, 'llama')
+ case 'novelai':
+ return await tokenizeWebTokenizers(data, 'novelai')
+ case 'claude':
+ return await tokenizeWebTokenizers(data, 'claude')
+ case 'novellist':
+ return await tokenizeWebTokenizers(data, 'novellist')
+ case 'llama3':
+ return await tokenizeWebTokenizers(data, 'llama')
+ case 'gemma':
+ return await gemmaTokenize(data)
+ case 'cohere':
+ return await tokenizeWebTokenizers(data, 'cohere')
+ case 'o200k_base':
+ return await tikJS(data, 'o200k_base')
+ case 'cl100k_base':
+ return await tikJS(data, 'cl100k_base')
+ case 'custom':
+ return await pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizerFunc?.(data) ?? [0]
+ default:
+ return await tikJS(data, 'o200k_base')
+ }
+ }
+
+ if(modelInfo.tokenizer === LLMTokenizer.NovelList){
+ const nv= await tokenizeWebTokenizers(data, 'novellist')
+ return nv
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.Claude){
+ return await tokenizeWebTokenizers(data, 'claude')
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
+ return await tokenizeWebTokenizers(data, 'novelai')
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.Mistral){
+ return await tokenizeWebTokenizers(data, 'mistral')
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.Llama){
+ return await tokenizeWebTokenizers(data, 'llama')
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.Local){
+ return await tokenizeGGUFModel(data)
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
+ return await tikJS(data, 'o200k_base')
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
+ return await tokenizeGoogleCloud(data)
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
+ return await gemmaTokenize(data)
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
+ return await tokenizeWebTokenizers(data, 'DeepSeek')
+ }
+ if(modelInfo.tokenizer === LLMTokenizer.Cohere){
+ return await tokenizeWebTokenizers(data, 'cohere')
+ }
+
+ return await tikJS(data)
}
type tokenizerType = 'novellist'|'claude'|'novelai'|'llama'|'mistral'|'llama3'|'gemma'|'cohere'|'googleCloud'|'DeepSeek'
diff --git a/version.json b/version.json
index 3f89e9b8..4e3840fb 100644
--- a/version.json
+++ b/version.json
@@ -1 +1 @@
-{"version":"158.2.1"}
\ No newline at end of file
+{"version":"159.0.0"}
\ No newline at end of file