chore: Update version to 1.102.4

src-tauri/tauri.conf.json

@@ -8,7 +8,7 @@
   },
   "package": {
     "productName": "RisuAI",
-    "version": "1.102.3"
+    "version": "1.102.4"
   },
   "tauri": {
     "allowlist": {

src/ts/storage/database.ts

@@ -14,7 +14,7 @@ import type { OobaChatCompletionRequestParams } from '../model/ooba';
 
 export const DataBase = writable({} as any as Database)
 export const loadedStore = writable(false)
-export let appVer = "1.102.3"
+export let appVer = "1.102.4"
 export let webAppSubVer = ''
 
 export function setDatabase(data:Database){

src/ts/tokenizer.ts

@@ -70,6 +70,9 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
             return await tokenizeWebTokenizers(data, 'llama')
         }
     }
+    if(db.aiModel.startsWith('gpt4o')){
+        return await tikJS(data, 'o200k_base')
+    }
 
     return await tikJS(data)
 }
@@ -79,17 +82,31 @@ type tokenizerType = 'novellist'|'claude'|'novelai'|'llama'|'mistral'|'llama3'
 let tikParser:Tiktoken = null
 let tokenizersTokenizer:Tokenizer = null
 let tokenizersType:tokenizerType = null
+let lastTikModel = 'cl100k_base'
 
-async function tikJS(text:string) {
+async function tikJS(text:string, model='cl100k_base') {
-    if(!tikParser){
+    if(!tikParser || lastTikModel !== model){
-        const {Tiktoken} = await import('@dqbd/tiktoken')
+        if(model === 'cl100k_base'){
-        const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");
+            const {Tiktoken} = await import('@dqbd/tiktoken')
-    
+            const cl100k_base = await import("@dqbd/tiktoken/encoders/cl100k_base.json");
-        tikParser = new Tiktoken(
+            lastTikModel = model   
-            cl100k_base.bpe_ranks,
+        
-            cl100k_base.special_tokens,
+            tikParser = new Tiktoken(
-            cl100k_base.pat_str
+                cl100k_base.bpe_ranks,
-        );
+                cl100k_base.special_tokens,
+                cl100k_base.pat_str
+            );
+        }
+        if(model === 'o200k_base'){
+            const {Tiktoken} = await import('@dqbd/tiktoken')
+            const o200k_base = await import("src/etc/o200k_base.json");
+            lastTikModel = model
+            tikParser = new Tiktoken(
+                o200k_base.bpe_ranks,
+                o200k_base.special_tokens,
+                o200k_base.pat_str
+            );
+        }
     }
     return tikParser.encode(text)
 }

version.json

@@ -1 +1 @@
-{"version":"1.102.3"}
+{"version":"1.102.4"}