mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] change file saving to msgpack

Browse Source
This commit is contained in:
kwaroran
2023-06-22 21:20:56 +09:00
parent 839b288168
commit 4447da7a6b
4 changed files with 53 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/lib/UI/ModelList.svelte
View File

@@ -63,10 +63,11 @@
<Arcodion name="OpenAI GPT">
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35')}}>GPT-3.5 Turbo</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_16k')}}>GPT-3.5 Turbo 16K</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_16k_0613')}}>GPT-3.5 Turbo 16K 0613</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4')}}>GPT-4</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_0613')}}>GPT-4 0613</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_32k')}}>GPT-4 32K</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_0613')}}>GPT-3.5 Turbo 0613</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt35_16k_0613')}}>GPT-3.5 Turbo 16K 0613</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_0613')}}>GPT-4 0613</button>
<button class="p-2 hover:text-green-500" on:click={() => {changeModel('gpt4_32k_0613')}}>GPT-4 32K 0613</button>
</Arcodion>
<Arcodion name="Anthropic Claude">
@@ -80,6 +81,7 @@
<button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('kobold')}}>Kobold</button>
{#if isTauri ||isNodeServer}
<button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('novelai')}}>NovelAI Clio</button>
<button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('deepai')}}>DeepAI</button>
{/if}
<Arcodion name="Horde">
{#await getHordeModels()}

14
src/ts/drive/drive.ts
View File

@@ -2,7 +2,6 @@ import { get } from "svelte/store";
import { alertError, alertInput, alertNormal, alertSelect, alertStore } from "../alert";
import { DataBase, setDatabase, type Database } from "../storage/database";
import { forageStorage, getUnpargeables, isNodeServer, isTauri, openURL } from "../storage/globalApi";
import pako from "pako";
import { BaseDirectory, exists, readBinaryFile, readDir, writeBinaryFile } from "@tauri-apps/api/fs";
import { language } from "../../lang";
import { relaunch } from '@tauri-apps/api/process';
@@ -10,6 +9,7 @@ import { open } from '@tauri-apps/api/shell';
import { cloneDeep, isEqual, last } from "lodash";
import { sleep } from "../util";
import { hubURL } from "../characterCards";
import { decodeRisuSave, encodeRisuSave } from "../storage/risuSave";
export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'reftoken'){
const CLIENT_ID = '580075990041-l26k2d3c0nemmqiu3d3aag01npfrkn76.apps.googleusercontent.com';
@@ -245,10 +245,7 @@ async function backupDrive(ACCESS_TOKEN:string) {
}
}
const dbjson = JSON.stringify(get(DataBase))
const dbData = pako.deflate(
Buffer.from(dbjson, 'utf-8')
)
const dbData = encodeRisuSave(get(DataBase))
alertStore.set({
type: "wait",
@@ -355,7 +352,7 @@ async function loadDrive(ACCESS_TOKEN:string, mode: 'backup'|'sync'):Promise<voi
}
const selectedIndex = (await alertSelect([language.loadLatest, language.loadOthers]) === '0') ? 0 : parseInt(await alertSelect(selectables))
const selectedDb = dbs[selectedIndex][0]
const decompressedDb:Database = JSON.parse(Buffer.from(pako.inflate(await getFileData(ACCESS_TOKEN, selectedDb.id))).toString('utf-8'))
const decompressedDb:Database = decodeRisuSave(await getFileData(ACCESS_TOKEN, selectedDb.id))
return decompressedDb
}
@@ -405,10 +402,7 @@ async function loadDrive(ACCESS_TOKEN:string, mode: 'backup'|'sync'):Promise<voi
}
}
db.didFirstSetup = true
const dbjson = JSON.stringify(db)
const dbData = pako.deflate(
Buffer.from(dbjson, 'utf-8')
)
const dbData = encodeRisuSave(db)
if(isTauri){
await writeBinaryFile('database/database.bin', dbData, {dir: BaseDirectory.AppData})

29
src/ts/storage/globalApi.ts
View File

@@ -7,7 +7,6 @@ import { appDataDir, join } from "@tauri-apps/api/path";
import { get } from "svelte/store";
import {open} from '@tauri-apps/api/shell'
import { DataBase, loadedStore, setDatabase, type Database, updateTextTheme, defaultSdDataFunc } from "./database";
import pako from "pako";
import { appWindow } from "@tauri-apps/api/window";
import { checkOldDomain, checkUpdate } from "../update";
import { selectedCharID } from "../stores";
@@ -21,6 +20,7 @@ import { cloneDeep } from "lodash";
import { NodeStorage } from "./nodeStorage";
import { defaultJailbreak, defaultMainPrompt, oldJailbreak, oldMainPrompt } from "./defaultPrompts";
import { loadRisuAccountData } from "../drive/accounter";
import { decodeRisuSave, encodeRisuSave } from "./risuSave";
//@ts-ignore
export const isTauri = !!window.__TAURI__
@@ -196,14 +196,14 @@ let lastSave = ''
export async function saveDb(){
lastSave =JSON.stringify(get(DataBase))
let changed = false
syncDrive()
DataBase.subscribe(() => {
changed = true
})
while(true){
const dbjson = JSON.stringify(get(DataBase))
if(dbjson !== lastSave){
lastSave = dbjson
const dbData = pako.deflate(
Buffer.from(dbjson, 'utf-8')
)
if(changed){
const dbData = encodeRisuSave(get(DataBase))
if(isTauri){
await writeBinaryFile('database/database.bin', dbData, {dir: BaseDirectory.AppData})
await writeBinaryFile(`database/dbbackup-${(Date.now()/100).toFixed()}.bin`, dbData, {dir: BaseDirectory.AppData})
@@ -275,13 +275,13 @@ export async function loadData() {
}
if(!await exists('database/database.bin', {dir: BaseDirectory.AppData})){
await writeBinaryFile('database/database.bin',
pako.deflate(Buffer.from(JSON.stringify({}), 'utf-8'))
encodeRisuSave({})
,{dir: BaseDirectory.AppData})
}
try {
setDatabase(
JSON.parse(Buffer.from(pako.inflate(Buffer.from(await readBinaryFile('database/database.bin',{dir: BaseDirectory.AppData})))).toString('utf-8'))
)
decodeRisuSave(await readBinaryFile('database/database.bin',{dir: BaseDirectory.AppData}))
)
} catch (error) {
const backups = await getDbBackups()
let backupLoaded = false
@@ -289,7 +289,7 @@ export async function loadData() {
try {
const backupData = await readBinaryFile(`database/dbbackup-${backup}.bin`,{dir: BaseDirectory.AppData})
setDatabase(
JSON.parse(Buffer.from(pako.inflate(Buffer.from(backupData))).toString('utf-8'))
decodeRisuSave(backupData)
)
backupLoaded = true
} catch (error) {}
@@ -305,12 +305,11 @@ export async function loadData() {
else{
let gotStorage:Uint8Array = await forageStorage.getItem('database/database.bin')
if(checkNullish(gotStorage)){
gotStorage = pako.deflate(Buffer.from(JSON.stringify({}), 'utf-8'))
await forageStorage.setItem('database/database.bin', gotStorage)
await forageStorage.setItem('database/database.bin', encodeRisuSave({}))
}
try {
setDatabase(
JSON.parse(Buffer.from(pako.inflate(Buffer.from(gotStorage))).toString('utf-8'))
decodeRisuSave(gotStorage)
)
} catch (error) {
const backups = await getDbBackups()
@@ -319,7 +318,7 @@ export async function loadData() {
try {
const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`)
setDatabase(
JSON.parse(Buffer.from(pako.inflate(Buffer.from(backupData))).toString('utf-8'))
decodeRisuSave(backupData)
)
backupLoaded = true
} catch (error) {}

31
src/ts/storage/risuSave.ts Normal file
View File

@@ -0,0 +1,31 @@
import { decode, encode } from "@msgpack/msgpack";
import { isEqual } from "lodash";
import pako from "pako";
export function encodeRisuSave(data:any){
const risuSaveHeader = new Uint8Array(Buffer.from("\u0000\u0000RISU",'utf-8'))
const encoded = encode(data)
const mergedArray = new Uint8Array(risuSaveHeader.length + encoded.length);
mergedArray.set(risuSaveHeader);
mergedArray.set(encoded, risuSaveHeader.length);
return mergedArray
}
export function decodeRisuSave(data:Uint8Array){
const risuSaveHeader = new Uint8Array(Buffer.from("\u0000\u0000RISU",'utf-8'))
const sub = data.subarray(0, risuSaveHeader.length)
if(isEqual(sub, risuSaveHeader)){
try {
const realData = data.subarray(risuSaveHeader.length)
const dec = decode(realData)
return dec
} catch (error) {
console.error(error)
throw error
}
}
else{
return JSON.parse(Buffer.from(pako.inflate(Buffer.from(data))).toString('utf-8'))
}
}