diff --git a/src/lib/Setting/Pages/UserSettings.svelte b/src/lib/Setting/Pages/UserSettings.svelte
index 95a4a6c5..2cdcb012 100644
--- a/src/lib/Setting/Pages/UserSettings.svelte
+++ b/src/lib/Setting/Pages/UserSettings.svelte
@@ -19,7 +19,7 @@
{
- if(e.origin.startsWith("https://sv.risuai.xyz") || e.origin.startsWith("http://127.0.0.1")){
+ if(e.origin.startsWith("https://sv.risuai.xyz") || e.origin.startsWith("http://127.0.0.1") || e.origin === window.location.origin){
if(e.data.msg.type === 'drive'){
await loadRisuAccountData()
DBState.db.account.data.refresh_token = e.data.msg.data.refresh_token
diff --git a/src/lib/SideBars/CharConfig.svelte b/src/lib/SideBars/CharConfig.svelte
index 7af6beca..c4c90a6e 100644
--- a/src/lib/SideBars/CharConfig.svelte
+++ b/src/lib/SideBars/CharConfig.svelte
@@ -1101,20 +1101,16 @@
}}
className="mt-4"
>
- {language.HypaMemory} V2 Data
+ {language.hypaMemoryV2Modal}
{:else if DBState.db.supaModelType !== 'none' && DBState.db.hypaV3}
{:else if DBState.db.characters[$selectedCharID].chats[DBState.db.characters[$selectedCharID].chatPage].supaMemoryData && DBState.db.characters[$selectedCharID].chats[DBState.db.characters[$selectedCharID].chatPage].supaMemoryData.length > 4 || DBState.db.characters[$selectedCharID].supaMemory}
{language.SuperMemory}
diff --git a/src/lib/SideBars/LoreBook/LoreBookList.svelte b/src/lib/SideBars/LoreBook/LoreBookList.svelte
index 89b1f455..213eec65 100644
--- a/src/lib/SideBars/LoreBook/LoreBookList.svelte
+++ b/src/lib/SideBars/LoreBook/LoreBookList.svelte
@@ -11,9 +11,10 @@
globalMode?: boolean;
submenu?: number;
lorePlus?: boolean;
+ externalLoreBooks?: loreBook[];
}
- let { globalMode = false, submenu = 0, lorePlus = false }: Props = $props();
+ let { globalMode = false, submenu = 0, lorePlus = false, externalLoreBooks = null }: Props = $props();
let stb: Sortable = null
let ele: HTMLDivElement = $state()
let sorted = $state(0)
@@ -31,6 +32,13 @@
})
DBState.db.loreBook[DBState.db.loreBookPage].data = newLore
}
+ else if(externalLoreBooks){
+ let newLore:loreBook[] = []
+ idx.forEach((i) => {
+ newLore.push(externalLoreBooks[i])
+ })
+ externalLoreBooks = newLore
+ }
else if(submenu === 1){
let newLore:loreBook[] = []
idx.forEach((i) => {
@@ -97,6 +105,18 @@
}} onOpen={onOpen} onClose={onClose}/>
{/each}
{/if}
+ {:else if externalLoreBooks}
+ {#if externalLoreBooks.length === 0}
+ No Lorebook
+ {:else}
+ {#each externalLoreBooks as book, i}
+ {
+ let lore = externalLoreBooks
+ lore.splice(i, 1)
+ externalLoreBooks = lore
+ }} onOpen={onOpen} onClose={onClose}/>
+ {/each}
+ {/if}
{:else if submenu === 0}
{#if DBState.db.characters[$selectedCharID].globalLore.length === 0}
No Lorebook
diff --git a/src/lib/UI/GUI/LazyPortal.svelte b/src/lib/UI/GUI/LazyPortal.svelte
new file mode 100644
index 00000000..5d70245d
--- /dev/null
+++ b/src/lib/UI/GUI/LazyPortal.svelte
@@ -0,0 +1,63 @@
+
\ No newline at end of file
diff --git a/src/ts/globalApi.svelte.ts b/src/ts/globalApi.svelte.ts
index 72c2feee..ed4d607d 100644
--- a/src/ts/globalApi.svelte.ts
+++ b/src/ts/globalApi.svelte.ts
@@ -87,7 +87,16 @@ export async function downloadFile(name:string, dat:Uint8Array|ArrayBuffer|strin
await writeFile(name, data, {baseDir: BaseDirectory.Download})
}
else{
- downloadURL(`data:png/image;base64,${Buffer.from(data).toString('base64')}`, name)
+ const blob = new Blob([data], { type: 'application/octet-stream' })
+ const url = URL.createObjectURL(blob)
+
+ downloadURL(url, name)
+
+ setTimeout(() => {
+ URL.revokeObjectURL(url)
+ }, 10000)
+
+
}
}
diff --git a/src/ts/parser.svelte.ts b/src/ts/parser.svelte.ts
index 54824838..b9767916 100644
--- a/src/ts/parser.svelte.ts
+++ b/src/ts/parser.svelte.ts
@@ -1,4 +1,4 @@
-import DOMPurify from 'isomorphic-dompurify';
+import DOMPurify from 'dompurify';
import markdownit from 'markdown-it'
import { getCurrentCharacter, type Database, type Message, type character, type customscript, type groupChat, type triggerscript } from './storage/database.svelte';
import { DBState } from './stores.svelte';
@@ -1239,6 +1239,11 @@ function basicMatcher (p1:string,matcherArg:matcherArg,vars:{[key:string]:string
case 'screen_height':{
return get(SizeStore).h.toString()
}
+ case 'cbr':
+ case 'cnl':
+ case 'cnewline':{
+ return '\\n'
+ }
}
const arra = p1.split("::")
if(arra.length > 1){
@@ -1677,6 +1682,38 @@ function basicMatcher (p1:string,matcherArg:matcherArg,vars:{[key:string]:string
case 'hash':{
return ((pickHashRand(0, arra[1]) * 10000000) + 1).toFixed(0).padStart(7, '0')
}
+ case 'randint':{
+ const min = Number(arra[1])
+ const max = Number(arra[2])
+ if(isNaN(min) || isNaN(max)){
+ return 'NaN'
+ }
+ return (Math.floor(Math.random() * (max - min + 1)) + min).toString()
+ }
+ case 'cbr':
+ case 'cnl':
+ case 'cnewline':{
+ return '\\n'.repeat(Number(arra[1]))
+ }
+ case 'dice':{
+ const notation = arra[1].split('d')
+ const num = Number(notation[0])
+ const sides = Number(notation[1])
+ if(isNaN(num) || isNaN(sides)){
+ return 'NaN'
+ }
+ let total = 0
+ for(let i = 0; i < num; i++){
+ total += Math.floor(Math.random() * sides) + 1
+ }
+ return total.toString()
+ }
+ case 'fromhex':{
+ return Number.parseInt(arra[1], 16).toString()
+ }
+ case 'tohex':{
+ return Number.parseInt(arra[1]).toString(16)
+ }
}
}
if(p1.startsWith('random')){
@@ -1877,7 +1914,7 @@ const legacyBlockMatcher = (p1:string,matcherArg:matcherArg) => {
return null
}
-type blockMatch = 'ignore'|'parse'|'nothing'|'parse-pure'|'pure'|'each'|'function'|'pure-display'
+type blockMatch = 'ignore'|'parse'|'nothing'|'parse-pure'|'pure'|'each'|'function'|'pure-display'|'normalize'
function parseArray(p1:string):string[]{
try {
@@ -1923,6 +1960,10 @@ function blockStartMatcher(p1:string,matcherArg:matcherArg):{type:blockMatch,typ
if(p1 === '#pure_display' || p1 === '#puredisplay'){
return {type:'pure-display'}
}
+ if(p1 === '#code'){
+ return {type:'normalize'}
+
+ }
if(p1.startsWith('#each')){
let t2 = p1.substring(5).trim()
if(t2.startsWith('as ')){
@@ -1963,6 +2004,34 @@ function blockEndMatcher(p1:string,type:{type:blockMatch,type2?:string},matcherA
case 'parse-pure':{
return p1
}
+ case 'normalize':{
+ return p1Trimed.trim().replaceAll('\n','').replaceAll('\t','')
+ .replaceAll(/\\u([0-9A-Fa-f]{4})/g, (match, p1) => {
+ return String.fromCharCode(parseInt(p1, 16))
+ })
+ .replaceAll(/\\(.)/g, (match, p1) => {
+ switch(p1){
+ case 'n':
+ return '\n'
+ case 'r':
+ return '\r'
+ case 't':
+ return '\t'
+ case 'b':
+ return '\b'
+ case 'f':
+ return '\f'
+ case 'v':
+ return '\v'
+ case 'a':
+ return '\a'
+ case 'x':
+ return '\x00'
+ default:
+ return p1
+ }
+ })
+ }
default:{
return ''
}
@@ -2124,7 +2193,7 @@ export function risuChatParser(da:string, arg:{
break
}
}
- if(dat.startsWith('/')){
+ if(dat.startsWith('/') && !dat.startsWith('//')){
if(stackType[nested.length] === 5){
const blockType = blockNestType.get(nested.length)
if( blockType.type === 'ignore' || blockType.type === 'pure' ||
diff --git a/src/ts/process/lorebook.svelte.ts b/src/ts/process/lorebook.svelte.ts
index 5e578be7..d428c6e1 100644
--- a/src/ts/process/lorebook.svelte.ts
+++ b/src/ts/process/lorebook.svelte.ts
@@ -55,7 +55,8 @@ export async function loadLoreBookV3Prompt(){
const recursiveScanning = char.loreSettings?.recursiveScanning ?? true
let recursivePrompt:{
prompt: string,
- source: string
+ source: string,
+ data: string
}[] = []
let matchLog:{
prompt: string,
@@ -75,23 +76,27 @@ export async function loadLoreBookV3Prompt(){
let mList:{
source:string
prompt:string
+ data:string
}[] = sliced.map((msg, i) => {
if(msg.role === 'user'){
return {
source: `message ${i} by user`,
- prompt: `\x01{{${DBState.db.username}}}:` + msg.data + '\x01'
+ prompt: `\x01{{${DBState.db.username}}}:` + msg.data + '\x01',
+ data: msg.data
}
}
else{
return {
source: `message ${i} by char`,
- prompt: `\x01{{${msg.name ?? (msg.saying ? findCharacterbyId(msg.saying)?.name : null) ?? char.name}}}:` + msg.data + '\x01'
+ prompt: `\x01{{${msg.name ?? (msg.saying ? findCharacterbyId(msg.saying)?.name : null) ?? char.name}}}:` + msg.data + '\x01',
+ data: msg.data
}
}
}).concat(recursivePrompt.map((msg) => {
return {
source: 'lorebook ' + msg.source,
- prompt: msg.prompt
+ prompt: msg.prompt,
+ data: msg.data
}
}))
@@ -106,7 +111,7 @@ export async function loadLoreBookV3Prompt(){
arg.keys[0] = regexString.replace('/'+regexFlag,'')
try {
const regex = new RegExp(arg.keys[0],regexFlag)
- const d = regex.test(mText.prompt)
+ const d = regex.test(mText.data)
if(d){
matchLog.push({
prompt: mText.prompt,
@@ -127,7 +132,8 @@ export async function loadLoreBookV3Prompt(){
mList = mList.map((m) => {
return {
source: m.source,
- prompt: m.prompt.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,'')
+ prompt: m.prompt.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,''),
+ data: m.data.toLocaleLowerCase().replace(/\{\{\/\/(.+?)\}\}/g,'').replace(/\{\{comment:(.+?)\}\}/g,'')
}
})
@@ -135,7 +141,7 @@ export async function loadLoreBookV3Prompt(){
let allModeMatched = true
for(const m of mList){
- let mText = m.prompt
+ let mText = m.data
if(arg.fullWordMatching){
const splited = mText.split(' ')
for(const key of arg.keys){
@@ -510,7 +516,7 @@ export async function importLoreBook(mode:'global'|'local'|'sglobal'){
}
}
-interface CCLorebook{
+export interface CCLorebook{
key:string[]
comment:string
content:string
diff --git a/src/ts/process/lua.ts b/src/ts/process/lua.ts
index 727cfb59..bc737239 100644
--- a/src/ts/process/lua.ts
+++ b/src/ts/process/lua.ts
@@ -13,11 +13,14 @@ import { v4 } from "uuid";
import { getModuleTriggers } from "./modules";
import { Mutex } from "../mutex";
import { tokenize } from "../tokenizer";
+import { fetchNative } from "../globalApi.svelte";
let luaFactory:LuaFactory
let LuaSafeIds = new Set()
let LuaEditDisplayIds = new Set()
let LuaLowLevelIds = new Set()
+let lastRequestResetTime = 0
+let lastRequestsCount = 0
interface LuaEngineState {
code?: string;
@@ -205,6 +208,75 @@ export async function runLua(code:string, arg:{
return await processer.similaritySearch(source)
})
+ luaEngine.global.set('request', async (id:string, url:string) => {
+ if(!LuaLowLevelIds.has(id)){
+ return
+ }
+
+ if(lastRequestResetTime + 60000 < Date.now()){
+ lastRequestsCount = 0
+ lastRequestResetTime = Date.now()
+ }
+
+ if(lastRequestsCount > 5){
+ return {
+ status: 429,
+ data: 'Too many requests. you can request 5 times per minute'
+ }
+ }
+
+ lastRequestsCount++
+
+ try {
+ //for security and other reasons, only get request in 120 char is allowed
+ if(url.length > 120){
+ return {
+ status: 413,
+ data: 'URL to large. max is 120 characters'
+ }
+ }
+
+ if(!url.startsWith('https://')){
+ return {
+ status: 400,
+ data: "Only https requests are allowed"
+ }
+ }
+
+ const bannedURL = [
+ "https://realm.risuai.net",
+ "https://risuai.net",
+ "https://risuai.xyz"
+ ]
+
+ for(const burl of bannedURL){
+
+ if(url.startsWith(burl)){
+ return {
+ status: 400,
+ data: "request to " + url + ' is not allowed'
+ }
+ }
+ }
+
+ //browser fetch
+ const d = await fetchNative(url, {
+ method: "GET"
+ })
+ const text = await d.text()
+ return {
+ status: d.status,
+ data: text
+ }
+
+ } catch (error) {
+ return {
+ status: 400,
+ data: 'internal error'
+ }
+ }
+ })
+
luaEngine.global.set('generateImage', async (id:string, value:string, negValue:string = '') => {
if(!LuaLowLevelIds.has(id)){
return
diff --git a/src/ts/process/scripts.ts b/src/ts/process/scripts.ts
index 3c95ab81..ed4c9ed7 100644
--- a/src/ts/process/scripts.ts
+++ b/src/ts/process/scripts.ts
@@ -74,9 +74,7 @@ function generateScriptCacheKey(scripts: customscript[], data: string, mode: Scr
if(script.type !== mode){
continue
}
- hash += `${script.flag?.includes('') ?
- risuChatParser(script.in, { chatID: chatID, cbsConditions }) :
- script.in}|||${risuChatParser(script.out, { chatID: chatID, cbsConditions})}|||${script.flag ?? ''}|||${script.ableFlag ? 1 : 0}`;
+ hash += `${script.flag?.includes('') ? risuChatParser(script.in, { chatID: chatID, cbsConditions }) : script.in}|||${script.out}${chatID}|||${script.flag ?? ''}|||${script.ableFlag ? 1 : 0}`;
}
return hash;
}
diff --git a/src/ts/process/stableDiff.ts b/src/ts/process/stableDiff.ts
index fb24033a..74e56dd2 100644
--- a/src/ts/process/stableDiff.ts
+++ b/src/ts/process/stableDiff.ts
@@ -132,7 +132,7 @@ export async function generateAIImage(genPrompt:string, currentChar:character, n
"parameters": {
"params_version": 3,
"add_original_image": true,
- "cfg_rescale": 0,
+ "cfg_rescale": db.NAIImgConfig.cfg_rescale,
"controlnet_strength": 1,
"dynamic_thresholding": false,
"n_samples": 1,
@@ -145,7 +145,7 @@ export async function generateAIImage(genPrompt:string, currentChar:character, n
"sm": false,
"sm_dyn": false,
"noise": db.NAIImgConfig.noise,
- "noise_schedule": "native",
+ "noise_schedule": db.NAIImgConfig.noise_schedule,
"strength": db.NAIImgConfig.strength,
"ucPreset": 3,
"uncond_scale": 1,
@@ -435,7 +435,7 @@ export async function generateAIImage(genPrompt:string, currentChar:character, n
}
await new Promise(r => setTimeout(r, 1000))
} // Check history until the generation is complete.
- const genImgInfo = Object.values(item.outputs).flatMap((output: any) => output.images)[0];
+ const genImgInfo = Object.values(item.outputs).flatMap((output: any) => output.images || [])[0];
const imgResponse = await fetchNative(createUrl('/view', {
filename: genImgInfo.filename,
diff --git a/src/ts/storage/database.svelte.ts b/src/ts/storage/database.svelte.ts
index 1c0f6aad..7a5fb478 100644
--- a/src/ts/storage/database.svelte.ts
+++ b/src/ts/storage/database.svelte.ts
@@ -255,8 +255,10 @@ export function setDatabase(data:Database){
width:512,
height:768,
sampler:"k_dpmpp_sde",
+ noise_schedule:"native",
steps:28,
scale:5,
+ cfg_rescale: 0,
sm:true,
sm_dyn:false,
noise:0.0,
@@ -1023,6 +1025,7 @@ export interface Database{
}[]
igpPrompt:string
useTokenizerCaching:boolean
+ showMenuHypaMemoryModal:boolean
}
interface SeparateParameters{
@@ -1408,8 +1411,10 @@ export interface NAIImgConfig{
width:number,
height:number,
sampler:string,
+ noise_schedule:string,
steps:number,
scale:number,
+ cfg_rescale:number,
sm:boolean,
sm_dyn:boolean,
noise:number,
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index e71528a4..25fb442d 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -6,9 +6,27 @@ import { supportsInlayImage } from "./process/files/inlays";
import { risuChatParser } from "./parser.svelte";
import { tokenizeGGUFModel } from "./process/models/local";
import { globalFetch } from "./globalApi.svelte";
-import { getModelInfo, LLMTokenizer } from "./model/modellist";
+import { getModelInfo, LLMTokenizer, type LLMModel } from "./model/modellist";
import { pluginV2 } from "./plugins/plugins";
import type { GemmaTokenizer } from "@huggingface/transformers";
+import { LRUMap } from 'mnemonist';
+
+const MAX_CACHE_SIZE = 1500;
+
+const encodeCache = new LRUMap(MAX_CACHE_SIZE);
+
+function getHash(
+ data: string,
+ aiModel: string,
+ customTokenizer: string,
+ currentPluginProvider: string,
+ googleClaudeTokenizing: boolean,
+ modelInfo: LLMModel,
+ pluginTokenizer: string
+): string {
+ const combined = `${data}::${aiModel}::${customTokenizer}::${currentPluginProvider}::${googleClaudeTokenizing ? '1' : '0'}::${modelInfo.tokenizer}::${pluginTokenizer}`;
+ return combined;
+}
export const tokenizerList = [
@@ -25,100 +43,114 @@ export const tokenizerList = [
] as const
export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
- let db = getDatabase()
+ const db = getDatabase();
+ const modelInfo = getModelInfo(db.aiModel);
+ const pluginTokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer ?? "none";
+
+ let cacheKey = ''
+ if(db.useTokenizerCaching){
+ cacheKey = getHash(
+ data,
+ db.aiModel,
+ db.customTokenizer,
+ db.currentPluginProvider,
+ db.googleClaudeTokenizing,
+ modelInfo,
+ pluginTokenizer
+ );
+ const cachedResult = encodeCache.get(cacheKey);
+ if (cachedResult !== undefined) {
+ return cachedResult;
+ }
+ }
+
+ let result: number[] | Uint32Array | Int32Array;
+
if(db.aiModel === 'openrouter' || db.aiModel === 'reverse_proxy'){
switch(db.customTokenizer){
case 'mistral':
- return await tokenizeWebTokenizers(data, 'mistral')
+ result = await tokenizeWebTokenizers(data, 'mistral'); break;
case 'llama':
- return await tokenizeWebTokenizers(data, 'llama')
+ result = await tokenizeWebTokenizers(data, 'llama'); break;
case 'novelai':
- return await tokenizeWebTokenizers(data, 'novelai')
+ result = await tokenizeWebTokenizers(data, 'novelai'); break;
case 'claude':
- return await tokenizeWebTokenizers(data, 'claude')
+ result = await tokenizeWebTokenizers(data, 'claude'); break;
case 'novellist':
- return await tokenizeWebTokenizers(data, 'novellist')
+ result = await tokenizeWebTokenizers(data, 'novellist'); break;
case 'llama3':
- return await tokenizeWebTokenizers(data, 'llama')
+ result = await tokenizeWebTokenizers(data, 'llama'); break;
case 'gemma':
- return await gemmaTokenize(data)
+ result = await gemmaTokenize(data); break;
case 'cohere':
- return await tokenizeWebTokenizers(data, 'cohere')
+ result = await tokenizeWebTokenizers(data, 'cohere'); break;
case 'deepseek':
- return await tokenizeWebTokenizers(data, 'DeepSeek')
+ result = await tokenizeWebTokenizers(data, 'DeepSeek'); break;
default:
- return await tikJS(data, 'o200k_base')
+ result = await tikJS(data, 'o200k_base'); break;
}
- }
-
- const modelInfo = getModelInfo(db.aiModel)
-
- if(db.aiModel === 'custom' && pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer){
- const tokenizer = pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizer
- switch(tokenizer){
+ } else if (db.aiModel === 'custom' && pluginTokenizer) {
+ switch(pluginTokenizer){
case 'mistral':
- return await tokenizeWebTokenizers(data, 'mistral')
+ result = await tokenizeWebTokenizers(data, 'mistral'); break;
case 'llama':
- return await tokenizeWebTokenizers(data, 'llama')
+ result = await tokenizeWebTokenizers(data, 'llama'); break;
case 'novelai':
- return await tokenizeWebTokenizers(data, 'novelai')
+ result = await tokenizeWebTokenizers(data, 'novelai'); break;
case 'claude':
- return await tokenizeWebTokenizers(data, 'claude')
+ result = await tokenizeWebTokenizers(data, 'claude'); break;
case 'novellist':
- return await tokenizeWebTokenizers(data, 'novellist')
+ result = await tokenizeWebTokenizers(data, 'novellist'); break;
case 'llama3':
- return await tokenizeWebTokenizers(data, 'llama')
+ result = await tokenizeWebTokenizers(data, 'llama'); break;
case 'gemma':
- return await gemmaTokenize(data)
+ result = await gemmaTokenize(data); break;
case 'cohere':
- return await tokenizeWebTokenizers(data, 'cohere')
+ result = await tokenizeWebTokenizers(data, 'cohere'); break;
case 'o200k_base':
- return await tikJS(data, 'o200k_base')
+ result = await tikJS(data, 'o200k_base'); break;
case 'cl100k_base':
- return await tikJS(data, 'cl100k_base')
+ result = await tikJS(data, 'cl100k_base'); break;
case 'custom':
- return await pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizerFunc?.(data) ?? [0]
+ result = await pluginV2.providerOptions.get(db.currentPluginProvider)?.tokenizerFunc?.(data) ?? [0]; break;
default:
- return await tikJS(data, 'o200k_base')
+ result = await tikJS(data, 'o200k_base'); break;
+ }
+ }
+
+ // Fallback
+ if (result === undefined) {
+ if(modelInfo.tokenizer === LLMTokenizer.NovelList){
+ result = await tokenizeWebTokenizers(data, 'novellist');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Claude){
+ result = await tokenizeWebTokenizers(data, 'claude');
+ } else if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
+ result = await tokenizeWebTokenizers(data, 'novelai');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Mistral){
+ result = await tokenizeWebTokenizers(data, 'mistral');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Llama){
+ result = await tokenizeWebTokenizers(data, 'llama');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Local){
+ result = await tokenizeGGUFModel(data);
+ } else if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
+ result = await tikJS(data, 'o200k_base');
+ } else if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
+ result = await tokenizeGoogleCloud(data);
+ } else if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
+ result = await gemmaTokenize(data);
+ } else if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
+ result = await tokenizeWebTokenizers(data, 'DeepSeek');
+ } else if(modelInfo.tokenizer === LLMTokenizer.Cohere){
+ result = await tokenizeWebTokenizers(data, 'cohere');
+ } else {
+ result = await tikJS(data);
}
}
-
- if(modelInfo.tokenizer === LLMTokenizer.NovelList){
- const nv= await tokenizeWebTokenizers(data, 'novellist')
- return nv
- }
- if(modelInfo.tokenizer === LLMTokenizer.Claude){
- return await tokenizeWebTokenizers(data, 'claude')
- }
- if(modelInfo.tokenizer === LLMTokenizer.NovelAI){
- return await tokenizeWebTokenizers(data, 'novelai')
- }
- if(modelInfo.tokenizer === LLMTokenizer.Mistral){
- return await tokenizeWebTokenizers(data, 'mistral')
- }
- if(modelInfo.tokenizer === LLMTokenizer.Llama){
- return await tokenizeWebTokenizers(data, 'llama')
- }
- if(modelInfo.tokenizer === LLMTokenizer.Local){
- return await tokenizeGGUFModel(data)
- }
- if(modelInfo.tokenizer === LLMTokenizer.tiktokenO200Base){
- return await tikJS(data, 'o200k_base')
- }
- if(modelInfo.tokenizer === LLMTokenizer.GoogleCloud && db.googleClaudeTokenizing){
- return await tokenizeGoogleCloud(data)
- }
- if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
- return await gemmaTokenize(data)
- }
- if(modelInfo.tokenizer === LLMTokenizer.DeepSeek){
- return await tokenizeWebTokenizers(data, 'DeepSeek')
- }
- if(modelInfo.tokenizer === LLMTokenizer.Cohere){
- return await tokenizeWebTokenizers(data, 'cohere')
+ if(db.useTokenizerCaching){
+ encodeCache.set(cacheKey, result);
}
- return await tikJS(data)
+ return result;
}
type tokenizerType = 'novellist'|'claude'|'novelai'|'llama'|'mistral'|'llama3'|'gemma'|'cohere'|'googleCloud'|'DeepSeek'
@@ -177,6 +209,7 @@ async function gemmaTokenize(text:string) {
async function tikJS(text:string, model='cl100k_base') {
if(!tikParser || lastTikModel !== model){
+ tikParser?.free()
if(model === 'cl100k_base'){
const {Tiktoken} = await import('@dqbd/tiktoken')
const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");