[feat] change file saving to msgpack

src/lib/UI/ModelList.svelte

@@ -63,10 +63,11 @@
             <Arcodion name="OpenAI GPT">
                 <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35')}}>GPT-3.5 Turbo</button>
                 <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_16k')}}>GPT-3.5 Turbo 16K</button>
-                <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_16k_0613')}}>GPT-3.5 Turbo 16K 0613</button>
                 <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4')}}>GPT-4</button>
-                <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_0613')}}>GPT-4 0613</button>
                 <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_32k')}}>GPT-4 32K</button>
+                <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_0613')}}>GPT-3.5 Turbo 0613</button>
+                <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_16k_0613')}}>GPT-3.5 Turbo 16K 0613</button>
+                <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_0613')}}>GPT-4 0613</button>
                 <button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_32k_0613')}}>GPT-4 32K 0613</button>
             </Arcodion>
             <Arcodion name="Anthropic Claude">
@@ -80,6 +81,7 @@
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('kobold')}}>Kobold</button>
             {#if isTauri ||isNodeServer}
                 <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('novelai')}}>NovelAI Clio</button>
+                <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('deepai')}}>DeepAI</button>
             {/if}
             <Arcodion name="Horde">
                 {#await getHordeModels()}

src/ts/drive/drive.ts

@@ -2,7 +2,6 @@ import { get } from "svelte/store";
 import { alertError, alertInput, alertNormal, alertSelect, alertStore } from "../alert";
 import { DataBase, setDatabase, type Database } from "../storage/database";
 import { forageStorage, getUnpargeables, isNodeServer, isTauri, openURL } from "../storage/globalApi";
-import pako from "pako";
 import { BaseDirectory, exists, readBinaryFile, readDir, writeBinaryFile } from "@tauri-apps/api/fs";
 import { language } from "../../lang";
 import { relaunch } from '@tauri-apps/api/process';
@@ -10,6 +9,7 @@ import { open } from '@tauri-apps/api/shell';
 import { cloneDeep, isEqual, last } from "lodash";
 import { sleep } from "../util";
 import { hubURL } from "../characterCards";
+import { decodeRisuSave, encodeRisuSave } from "../storage/risuSave";
 
 export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'reftoken'){
     const CLIENT_ID = '580075990041-l26k2d3c0nemmqiu3d3aag01npfrkn76.apps.googleusercontent.com';
@@ -245,10 +245,7 @@ async function backupDrive(ACCESS_TOKEN:string) {
         }
     }
 
-    const dbjson = JSON.stringify(get(DataBase))
+    const dbData = encodeRisuSave(get(DataBase))
-    const dbData = pako.deflate(
-        Buffer.from(dbjson, 'utf-8')
-    )
 
     alertStore.set({
         type: "wait",
@@ -355,7 +352,7 @@ async function loadDrive(ACCESS_TOKEN:string, mode: 'backup'|'sync'):Promise<voi
             }
             const selectedIndex = (await alertSelect([language.loadLatest, language.loadOthers]) === '0') ? 0 : parseInt(await alertSelect(selectables))
             const selectedDb = dbs[selectedIndex][0]
-            const decompressedDb:Database = JSON.parse(Buffer.from(pako.inflate(await getFileData(ACCESS_TOKEN, selectedDb.id))).toString('utf-8'))
+            const decompressedDb:Database = decodeRisuSave(await getFileData(ACCESS_TOKEN, selectedDb.id))
             return decompressedDb
         }
     
@@ -405,10 +402,7 @@ async function loadDrive(ACCESS_TOKEN:string, mode: 'backup'|'sync'):Promise<voi
             }
         }
         db.didFirstSetup = true
-        const dbjson = JSON.stringify(db)
+        const dbData = encodeRisuSave(db)
-        const dbData = pako.deflate(
-            Buffer.from(dbjson, 'utf-8')
-        )
 
         if(isTauri){
             await writeBinaryFile('database/database.bin', dbData, {dir: BaseDirectory.AppData})

src/ts/storage/globalApi.ts

@@ -7,7 +7,6 @@ import { appDataDir, join } from "@tauri-apps/api/path";
 import { get } from "svelte/store";
 import {open} from '@tauri-apps/api/shell'
 import { DataBase, loadedStore, setDatabase, type Database, updateTextTheme, defaultSdDataFunc } from "./database";
-import pako from "pako";
 import { appWindow } from "@tauri-apps/api/window";
 import { checkOldDomain, checkUpdate } from "../update";
 import { selectedCharID } from "../stores";
@@ -21,6 +20,7 @@ import { cloneDeep } from "lodash";
 import { NodeStorage } from "./nodeStorage";
 import { defaultJailbreak, defaultMainPrompt, oldJailbreak, oldMainPrompt } from "./defaultPrompts";
 import { loadRisuAccountData } from "../drive/accounter";
+import { decodeRisuSave, encodeRisuSave } from "./risuSave";
 
 //@ts-ignore
 export const isTauri = !!window.__TAURI__
@@ -196,14 +196,14 @@ let lastSave = ''
 
 export async function saveDb(){
     lastSave =JSON.stringify(get(DataBase))
+    let changed = false
     syncDrive()
+    DataBase.subscribe(() => {
+        changed = true
+    })
     while(true){
-        const dbjson = JSON.stringify(get(DataBase))
+        if(changed){
-        if(dbjson !== lastSave){
+            const dbData = encodeRisuSave(get(DataBase))
-            lastSave = dbjson
-            const dbData = pako.deflate(
-                Buffer.from(dbjson, 'utf-8')
-            )
             if(isTauri){
                 await writeBinaryFile('database/database.bin', dbData, {dir: BaseDirectory.AppData})
                 await writeBinaryFile(`database/dbbackup-${(Date.now()/100).toFixed()}.bin`, dbData, {dir: BaseDirectory.AppData})
@@ -275,12 +275,12 @@ export async function loadData() {
                 }
                 if(!await exists('database/database.bin', {dir: BaseDirectory.AppData})){
                     await writeBinaryFile('database/database.bin',
-                        pako.deflate(Buffer.from(JSON.stringify({}), 'utf-8'))
+                        encodeRisuSave({})
                     ,{dir: BaseDirectory.AppData})
                 }
                 try {
                     setDatabase(
-                        JSON.parse(Buffer.from(pako.inflate(Buffer.from(await readBinaryFile('database/database.bin',{dir: BaseDirectory.AppData})))).toString('utf-8'))
+                        decodeRisuSave(await readBinaryFile('database/database.bin',{dir: BaseDirectory.AppData}))
                     )
                 } catch (error) {
                     const backups = await getDbBackups()
@@ -289,7 +289,7 @@ export async function loadData() {
                         try {
                             const backupData = await readBinaryFile(`database/dbbackup-${backup}.bin`,{dir: BaseDirectory.AppData})
                             setDatabase(
-                                JSON.parse(Buffer.from(pako.inflate(Buffer.from(backupData))).toString('utf-8'))
+                                decodeRisuSave(backupData)
                             )
                             backupLoaded = true
                         } catch (error) {}
@@ -305,12 +305,11 @@ export async function loadData() {
             else{
                 let gotStorage:Uint8Array = await forageStorage.getItem('database/database.bin')
                 if(checkNullish(gotStorage)){
-                    gotStorage = pako.deflate(Buffer.from(JSON.stringify({}), 'utf-8'))
+                    await forageStorage.setItem('database/database.bin', encodeRisuSave({}))
-                    await forageStorage.setItem('database/database.bin', gotStorage)
                 }
                 try {
                     setDatabase(
-                        JSON.parse(Buffer.from(pako.inflate(Buffer.from(gotStorage))).toString('utf-8'))
+                        decodeRisuSave(gotStorage)
                     )
                 } catch (error) {
                     const backups = await getDbBackups()
@@ -319,7 +318,7 @@ export async function loadData() {
                         try {
                             const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`)
                             setDatabase(
-                                JSON.parse(Buffer.from(pako.inflate(Buffer.from(backupData))).toString('utf-8'))
+                                decodeRisuSave(backupData)
                             )
                             backupLoaded = true
                         } catch (error) {}

src/ts/storage/risuSave.ts

@@ -0,0 +1,31 @@
+import { decode, encode } from "@msgpack/msgpack";
+import { isEqual } from "lodash";
+import pako from "pako";
+
+
+export function encodeRisuSave(data:any){
+    const risuSaveHeader = new Uint8Array(Buffer.from("\u0000\u0000RISU",'utf-8'))
+    const encoded = encode(data)
+    const mergedArray = new Uint8Array(risuSaveHeader.length + encoded.length);
+    mergedArray.set(risuSaveHeader);
+    mergedArray.set(encoded, risuSaveHeader.length);
+    return mergedArray
+}
+
+export function decodeRisuSave(data:Uint8Array){
+    const risuSaveHeader = new Uint8Array(Buffer.from("\u0000\u0000RISU",'utf-8'))
+    const sub = data.subarray(0, risuSaveHeader.length)
+    if(isEqual(sub, risuSaveHeader)){
+        try {
+            const realData = data.subarray(risuSaveHeader.length)
+            const dec = decode(realData)
+            return dec
+        } catch (error) {
+            console.error(error)
+            throw error
+        }
+    }
+    else{
+        return JSON.parse(Buffer.from(pako.inflate(Buffer.from(data))).toString('utf-8'))
+    }
+}