mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/LightningHyperBlaze45654/RisuAI

Browse Source
This commit is contained in:
LightningHyperBlaze45654
2024-11-29 20:33:11 -08:00
parent a8d52decd0 5ebb934d3b
commit 68f816bda9
220 changed files with 26974 additions and 10497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
src/ts/storage/accountStorage.ts
View File

@@ -1,15 +1,17 @@
import { get, writable } from "svelte/store"
import { DataBase } from "./database"
import { writable } from "svelte/store"
import { getDatabase } from "./database.svelte"
import { hubURL } from "../characterCards"
import localforage from "localforage"
import { alertError, alertLogin, alertStore, alertWait } from "../alert"
import { forageStorage, getUnpargeables, replaceDbResources } from "./globalApi"
import { encodeRisuSave } from "./risuSave"
import { alertLogin, alertNormalWait, alertStore, alertWait } from "../alert"
import { forageStorage, getUnpargeables } from "../globalApi.svelte"
import { encodeRisuSaveLegacy } from "./risuSave"
import { v4 } from "uuid"
import { language } from "src/lang"
import { sleep } from "../util"
export const AccountWarning = writable('')
const risuSession = Date.now().toFixed(0)
const cachedForage = localforage.createInstance({name: "risuaiAccountCached"})
let seenWarnings:string[] = []
@@ -20,7 +22,20 @@ export class AccountStorage{
async setItem(key:string, value:Uint8Array) {
this.checkAuth()
let da:Response
let daText:string|undefined = undefined
const getDaText = async () => {
if(daText === undefined){
daText = await da.text()
}
return daText
}
while((!da) || da.status === 403){
const saveDate = Date.now().toFixed(0)
da = await fetch(hubURL + '/api/account/write', {
method: "POST",
body: value,
@@ -29,11 +44,18 @@ export class AccountStorage{
'x-risu-key': key,
'x-risu-auth': this.auth,
'X-Format': 'nocheck',
'x-risu-session': risuSession
'x-risu-session': risuSession,
'x-risu-save-date': saveDate
}
})
if(key === 'database/database.bin'){
cachedForage.setItem(key, value).then(() => {
cachedForage.setItem(key + '__date', saveDate)
})
}
if(da.headers.get('Content-Type') === 'application/json'){
const json = (await da.json())
const json = JSON.parse(await getDaText())
if(json?.warning){
if(!seenWarnings.includes(json.warning)){
seenWarnings.push(json.warning)
@@ -41,8 +63,10 @@ export class AccountStorage{
}
}
if(json?.reloadSession){
alertWait(language.reloadSession)
location.reload()
alertNormalWait(language.activeTabChange).then(() => {
location.reload()
})
await sleep(100000000) // wait forever
return
}
}
@@ -59,9 +83,9 @@ export class AccountStorage{
}
}
if(da.status < 200 || da.status >= 300){
throw await da.text()
throw await getDaText()
}
return await da.text()
return await getDaText()
}
async getItem(key:string):Promise<Buffer> {
this.checkAuth()
@@ -72,13 +96,16 @@ export class AccountStorage{
}
}
let da:Response
const saveDate = await cachedForage.getItem(key + '__date') as number|undefined
const perf = performance.now()
while((!da) || da.status === 403){
da = await fetch(hubURL + '/api/account/read/' + Buffer.from(key ,'utf-8').toString('hex') +
(key.includes('database') ? ('|' + v4()) : ''), {
method: "GET",
headers: {
'x-risu-key': key,
'x-risu-auth': this.auth
'x-risu-auth': this.auth,
'x-risu-save-date': (saveDate || 0).toString()
}
})
if(da.status === 403){
@@ -86,6 +113,18 @@ export class AccountStorage{
this.checkAuth()
}
}
if(da.status === 303){
console.log(performance.now() - perf)
const data = await da.json()
if(data.match){
const c = Buffer.from(await cachedForage.getItem(key))
return c
}
else{
return null
}
}
if(da.status < 200 || da.status >= 300){
throw await da.text()
}
@@ -99,7 +138,7 @@ export class AccountStorage{
return Buffer.from(ab)
}
async keys():Promise<string[]>{
let db = get(DataBase)
let db = getDatabase()
return getUnpargeables(db, 'pure')
}
async removeItem(key:string){
@@ -107,7 +146,7 @@ export class AccountStorage{
}
private checkAuth(){
const db = get(DataBase)
const db = getDatabase()
this.auth = db?.account?.token
if(!this.auth){
db.account = JSON.parse(localStorage.getItem("fallbackRisuToken"))
@@ -122,7 +161,7 @@ export class AccountStorage{
export async function unMigrationAccount() {
const keys = await forageStorage.keys()
let db = get(DataBase)
let db = getDatabase()
let i = 0;
const MigrationStorage = localforage.createInstance({name: "risuai"})
@@ -136,7 +175,7 @@ export async function unMigrationAccount() {
}
db.account = null
await MigrationStorage.setItem('database/database.bin', encodeRisuSave(db))
await MigrationStorage.setItem('database/database.bin', encodeRisuSaveLegacy(db))
alertStore.set({
type: "none",

13
src/ts/storage/autoStorage.ts
View File

@@ -1,12 +1,11 @@
import localforage from "localforage"
import { isNodeServer, replaceDbResources } from "./globalApi"
import { isNodeServer, replaceDbResources } from "../globalApi.svelte"
import { NodeStorage } from "./nodeStorage"
import { OpfsStorage } from "./opfsStorage"
import { alertInput, alertSelect, alertStore } from "../alert"
import { get } from "svelte/store"
import { DataBase, type Database } from "./database"
import { getDatabase, type Database } from "./database.svelte"
import { AccountStorage } from "./accountStorage"
import { decodeRisuSave, encodeRisuSave } from "./risuSave";
import { decodeRisuSave, encodeRisuSaveLegacy } from "./risuSave";
import { language } from "src/lang"
import { MobileStorage } from "./mobileStorage"
import { Capacitor } from "@capacitor/core"
@@ -40,7 +39,7 @@ export class AutoStorage{
}
async checkAccountSync(){
let db = get(DataBase)
let db = getDatabase()
if(this.isAccount){
return true
}
@@ -89,10 +88,10 @@ export class AutoStorage{
}
const dba = replaceDbResources(db, replaced)
const comp = encodeRisuSave(dba, 'compression')
const comp = encodeRisuSaveLegacy(dba, 'compression')
//try decoding
try {
const z:Database = decodeRisuSave(comp)
const z:Database = await decodeRisuSave(comp)
if(z.formatversion){
await accountStorage.setItem('database/database.bin', comp)
}

250
src/ts/storage/database.ts → src/ts/storage/database.svelte.ts
View File

@@ -1,14 +1,9 @@
export const DataBase = writable({} as any as Database)
export const loadedStore = writable(false)
export let appVer = "134.1.0"
export let webAppSubVer = ''
import { get, writable } from 'svelte/store';
import { checkNullish, decryptBuffer, encryptBuffer, selectSingleFile } from '../util';
import { changeLanguage, language } from '../../lang';
import type { RisuPlugin } from '../plugins/plugins';
import type {triggerscript as triggerscriptMain} from '../process/triggers';
import { downloadFile, saveAsset as saveImageGlobal } from './globalApi';
import { downloadFile, saveAsset as saveImageGlobal } from '../globalApi.svelte';
import { defaultAutoSuggestPrompt, defaultJailbreak, defaultMainPrompt } from './defaultPrompts';
import { alertNormal, alertSelect } from '../alert';
import type { NAISettings } from '../process/models/nai';
@@ -17,6 +12,10 @@ import { defaultColorScheme, type ColorScheme } from '../gui/colorscheme';
import type { PromptItem, PromptSettings } from '../process/prompt';
import type { OobaChatCompletionRequestParams } from '../model/ooba';
export let appVer = "140.1.0"
export let webAppSubVer = ''
export function setDatabase(data:Database){
if(checkNullish(data.characters)){
data.characters = []
@@ -305,6 +304,7 @@ export function setDatabase(data:Database){
data.sendWithEnter ??= true
data.autoSuggestPrompt ??= defaultAutoSuggestPrompt
data.autoSuggestPrefix ??= ""
data.OAIPrediction ??= ''
data.autoSuggestClean ??= true
data.imageCompression ??= true
if(!data.formatingOrder.includes('personaPrompt')){
@@ -319,16 +319,16 @@ export function setDatabase(data:Database){
largePortrait: false
}]
data.classicMaxWidth ??= false
data.ooba ??= structuredClone(defaultOoba)
data.ainconfig ??= structuredClone(defaultAIN)
data.ooba ??= safeStructuredClone(defaultOoba)
data.ainconfig ??= safeStructuredClone(defaultAIN)
data.openrouterKey ??= ''
data.openrouterRequestModel ??= 'openai/gpt-3.5-turbo'
data.toggleConfirmRecommendedPreset ??= true
data.officialplugins ??= {}
data.NAIsettings ??= structuredClone(prebuiltNAIpresets)
data.NAIsettings ??= safeStructuredClone(prebuiltNAIpresets)
data.assetWidth ??= -1
data.animationSpeed ??= 0.4
data.colorScheme ??= structuredClone(defaultColorScheme)
data.colorScheme ??= safeStructuredClone(defaultColorScheme)
data.colorSchemeName ??= 'default'
data.NAIsettings.starter ??= ""
data.hypaModel ??= 'MiniLM'
@@ -354,6 +354,7 @@ export function setDatabase(data:Database){
data.newOAIHandle ??= true
data.gptVisionQuality ??= 'low'
data.huggingfaceKey ??= ''
data.fishSpeechKey ??= ''
data.statistics ??= {}
data.reverseProxyOobaArgs ??= {
mode: 'instruct'
@@ -445,10 +446,86 @@ export function setDatabase(data:Database){
}
data.customQuotes ??= false
data.customQuotesData ??= ['“','”','‘','’']
data.groupOtherBotRole ??= 'user'
data.customGUI ??= ''
data.customAPIFormat ??= LLMFormat.OpenAICompatible
data.systemContentReplacement ??= `system: {{slot}}`
data.systemRoleReplacement ??= 'user'
data.vertexAccessToken ??= ''
data.vertexAccessTokenExpires ??= 0
data.vertexClientEmail ??= ''
data.vertexPrivateKey ??= ''
data.seperateParametersEnabled ??= false
data.seperateParameters = {
memory: {},
emotion: {},
translate: {},
otherAx: {}
}
changeLanguage(data.language)
DataBase.set(data)
setDatabaseLite(data)
}
export function setDatabaseLite(data:Database){
if(import.meta.env.DEV){
console.trace('setDatabaseLite executed')
}
DBState.db = data
}
interface getDatabaseOptions{
snapshot?:boolean
}
export function getDatabase(options:getDatabaseOptions = {}):Database{
if(options.snapshot){
return $state.snapshot(DBState.db) as Database
}
return DBState.db as Database
}
export function getCurrentCharacter(options:getDatabaseOptions = {}):character|groupChat{
const db = getDatabase(options)
if(!db.characters){
db.characters = []
}
const char = db.characters?.[get(selectedCharID)]
return char
}
export function setCurrentCharacter(char:character|groupChat){
if(!DBState.db.characters){
DBState.db.characters = []
}
DBState.db.characters[get(selectedCharID)] = char
}
export function getCharacterByIndex(index:number,options:getDatabaseOptions = {}):character|groupChat{
const db = getDatabase(options)
if(!db.characters){
db.characters = []
}
const char = db.characters?.[index]
return char
}
export function setCharacterByIndex(index:number,char:character|groupChat){
if(!DBState.db.characters){
DBState.db.characters = []
}
DBState.db.characters[index] = char
}
export function getCurrentChat(){
const char = getCurrentCharacter()
return char?.chats[char.chatPage]
}
export function setCurrentChat(chat:Chat){
const char = getCurrentCharacter()
char.chats[char.chatPage] = chat
setCurrentCharacter(char)
}
export interface Database{
characters: (character|groupChat)[],
@@ -654,6 +731,7 @@ export interface Database{
tpo?:boolean
automark?:boolean
huggingfaceKey:string
fishSpeechKey:string
allowAllExtentionFiles?:boolean
translatorPrompt:string
translatorMaxResponse:number
@@ -751,6 +829,40 @@ export interface Database{
}
customQuotes:boolean
customQuotesData?:[string, string, string, string]
groupTemplate?:string
groupOtherBotRole?:string
customGUI:string
guiHTML:string
logShare:boolean
OAIPrediction:string
customAPIFormat:LLMFormat
systemContentReplacement:string
systemRoleReplacement:'user'|'assistant'
vertexPrivateKey: string
vertexClientEmail: string
vertexAccessToken: string
vertexAccessTokenExpires: number
seperateParametersEnabled:boolean
seperateParameters:{
memory: SeparateParameters,
emotion: SeparateParameters,
translate: SeparateParameters,
otherAx: SeparateParameters
}
translateBeforeHTMLFormatting:boolean
autoTranslateCachedOnly:boolean
lightningRealmImport:boolean
}
interface SeparateParameters{
temperature?:number
top_k?:number
repetition_penalty?:number
min_p?:number
top_a?:number
top_p?:number
frequency_penalty?:number
presence_penalty?:number
}
export interface customscript{
@@ -864,6 +976,16 @@ export interface character{
top_k?:number
text_split_method?: "cut0" | "cut1" | "cut2" | "cut3" | "cut4" | "cut5"
}
fishSpeechConfig?:{
model?: {
_id:string
title:string
description:string
},
chunk_length:number,
normalize:boolean,
}
supaMemory?:boolean
additionalAssets?:[string, string, string][]
ttsReadOnlyQuoted?:boolean
@@ -905,6 +1027,7 @@ export interface character{
lowLevelAccess?:boolean
hideChatIcon?:boolean
lastInteraction?:number
translatorNote?:string
}
@@ -916,7 +1039,7 @@ export interface loreSettings{
}
export interface groupChat{
export interface groupChat{
type: 'group'
image?:string
firstMessage:string
@@ -954,6 +1077,30 @@ export interface groupChat{
lowLevelAccess?:boolean
hideChatIcon?:boolean
lastInteraction?:number
//lazy hack for typechecking
voicevoxConfig?:any
ttsSpeech?:string
naittsConfig?:any
oaiVoice?:string
hfTTS?: any
vits?: OnnxModelFiles
gptSoVitsConfig?:any
fishSpeechConfig?:any
ttsReadOnlyQuoted?:boolean
exampleMessage?:string
systemPrompt?:string
replaceGlobalNote?:string
additionalText?:string
personality?:string
scenario?:string
translatorNote?:string
additionalData?: any
depth_prompt?: { depth: number, prompt: string }
additionalAssets?:[string, string, string][]
utilityBot?:boolean
license?:string
realmId:string
}
export interface botPreset{
@@ -1011,6 +1158,19 @@ export interface botPreset{
jsonSchema?:string
strictJsonSchema?:boolean
extractJson?:string
groupTemplate?:string
groupOtherBotRole?:string
seperateParametersEnabled?:boolean
seperateParameters?:{
memory: SeparateParameters,
emotion: SeparateParameters,
translate: SeparateParameters,
otherAx: SeparateParameters
}
customAPIFormat?:LLMFormat
systemContentReplacement?: string
systemRoleReplacement?: 'user'|'assistant'
openAIPrediction?: string
}
@@ -1222,8 +1382,8 @@ export const presetTemplate:botPreset = {
promptPreprocess: false,
proxyKey: '',
bias: [],
ooba: structuredClone(defaultOoba),
ainconfig: structuredClone(defaultAIN),
ooba: safeStructuredClone(defaultOoba),
ainconfig: safeStructuredClone(defaultAIN),
reverseProxyOobaArgs: {
mode: 'instruct'
},
@@ -1242,11 +1402,11 @@ const defaultSdData:[string,string][] = [
]
export const defaultSdDataFunc = () =>{
return structuredClone(defaultSdData)
return safeStructuredClone(defaultSdData)
}
export function saveCurrentPreset(){
let db = get(DataBase)
let db = getDatabase()
let pres = db.botPresets
pres[db.botPresetsId] = {
name: pres[db.botPresetsId].name,
@@ -1272,20 +1432,20 @@ export function saveCurrentPreset(){
bias: db.bias,
koboldURL: db.koboldURL,
proxyKey: db.proxyKey,
ooba: structuredClone(db.ooba),
ainconfig: structuredClone(db.ainconfig),
ooba: safeStructuredClone(db.ooba),
ainconfig: safeStructuredClone(db.ainconfig),
proxyRequestModel: db.proxyRequestModel,
openrouterRequestModel: db.openrouterRequestModel,
NAISettings: structuredClone(db.NAIsettings),
NAISettings: safeStructuredClone(db.NAIsettings),
promptTemplate: db.promptTemplate ?? null,
NAIadventure: db.NAIadventure ?? false,
NAIappendName: db.NAIappendName ?? false,
localStopStrings: db.localStopStrings,
autoSuggestPrompt: db.autoSuggestPrompt,
customProxyRequestModel: db.customProxyRequestModel,
reverseProxyOobaArgs: structuredClone(db.reverseProxyOobaArgs) ?? null,
reverseProxyOobaArgs: safeStructuredClone(db.reverseProxyOobaArgs) ?? null,
top_p: db.top_p ?? 1,
promptSettings: structuredClone(db.promptSettings) ?? null,
promptSettings: safeStructuredClone(db.promptSettings) ?? null,
repetition_penalty: db.repetition_penalty,
min_p: db.min_p,
top_a: db.top_a,
@@ -1301,6 +1461,14 @@ export function saveCurrentPreset(){
jsonSchema:db.jsonSchema ?? '',
strictJsonSchema:db.strictJsonSchema ?? true,
extractJson:db.extractJson ?? '',
groupOtherBotRole: db.groupOtherBotRole ?? 'user',
groupTemplate: db.groupTemplate ?? '',
seperateParametersEnabled: db.seperateParametersEnabled ?? false,
seperateParameters: safeStructuredClone(db.seperateParameters),
openAIPrediction: db.OAIPrediction,
customAPIFormat: safeStructuredClone(db.customAPIFormat),
systemContentReplacement: db.systemContentReplacement,
systemRoleReplacement: db.systemRoleReplacement,
}
db.botPresets = pres
setDatabase(db)
@@ -1308,9 +1476,9 @@ export function saveCurrentPreset(){
export function copyPreset(id:number){
saveCurrentPreset()
let db = get(DataBase)
let db = getDatabase()
let pres = db.botPresets
const newPres = structuredClone(pres[id])
const newPres = safeStructuredClone(pres[id])
newPres.name += " Copy"
db.botPresets.push(newPres)
setDatabase(db)
@@ -1320,7 +1488,7 @@ export function changeToPreset(id =0, savecurrent = true){
if(savecurrent){
saveCurrentPreset()
}
let db = get(DataBase)
let db = getDatabase()
let pres = db.botPresets
const newPres = pres[id]
db.botPresetsId = id
@@ -1350,8 +1518,8 @@ export function setPreset(db:Database, newPres: botPreset){
db.bias = newPres.bias ?? db.bias
db.koboldURL = newPres.koboldURL ?? db.koboldURL
db.proxyKey = newPres.proxyKey ?? db.proxyKey
db.ooba = structuredClone(newPres.ooba ?? db.ooba)
db.ainconfig = structuredClone(newPres.ainconfig ?? db.ainconfig)
db.ooba = safeStructuredClone(newPres.ooba ?? db.ooba)
db.ainconfig = safeStructuredClone(newPres.ainconfig ?? db.ainconfig)
db.openrouterRequestModel = newPres.openrouterRequestModel ?? db.openrouterRequestModel
db.proxyRequestModel = newPres.proxyRequestModel ?? db.proxyRequestModel
db.NAIsettings = newPres.NAISettings ?? db.NAIsettings
@@ -1366,12 +1534,12 @@ export function setPreset(db:Database, newPres: botPreset){
db.NAIsettings.mirostat_lr ??= 1
db.localStopStrings = newPres.localStopStrings
db.customProxyRequestModel = newPres.customProxyRequestModel ?? ''
db.reverseProxyOobaArgs = structuredClone(newPres.reverseProxyOobaArgs) ?? {
db.reverseProxyOobaArgs = safeStructuredClone(newPres.reverseProxyOobaArgs) ?? {
mode: 'instruct'
}
db.top_p = newPres.top_p ?? 1
//@ts-ignore //for legacy mistpings
db.promptSettings = structuredClone(newPres.promptSettings) ?? {
db.promptSettings = safeStructuredClone(newPres.promptSettings) ?? {
assistantPrefill: '',
postEndInnerFormat: '',
sendChatAsSystem: false,
@@ -1393,6 +1561,19 @@ export function setPreset(db:Database, newPres: botPreset){
db.jsonSchema = newPres.jsonSchema ?? ''
db.strictJsonSchema = newPres.strictJsonSchema ?? true
db.extractJson = newPres.extractJson ?? ''
db.groupOtherBotRole = newPres.groupOtherBotRole ?? 'user'
db.groupTemplate = newPres.groupTemplate ?? ''
db.seperateParametersEnabled = newPres.seperateParametersEnabled ?? false
db.seperateParameters = newPres.seperateParameters ? safeStructuredClone(newPres.seperateParameters) : {
memory: {},
emotion: {},
translate: {},
otherAx: {}
}
db.OAIPrediction = newPres.openAIPrediction ?? ''
db.customAPIFormat = safeStructuredClone(newPres.customAPIFormat) ?? LLMFormat.OpenAICompatible
db.systemContentReplacement = newPres.systemContentReplacement ?? ''
db.systemRoleReplacement = newPres.systemRoleReplacement ?? 'user'
return db
}
@@ -1402,11 +1583,14 @@ import type { OnnxModelFiles } from '../process/transformers';
import type { RisuModule } from '../process/modules';
import type { HypaV2Data } from '../process/memory/hypav2';
import { decodeRPack, encodeRPack } from '../rpack/rpack_bg';
import { DBState, selectedCharID } from '../stores.svelte';
import { LLMFormat } from '../model/modellist';
import type { Parameter } from '../process/request';
export async function downloadPreset(id:number, type:'json'|'risupreset'|'return' = 'json'){
saveCurrentPreset()
let db = get(DataBase)
let pres = structuredClone(db.botPresets[id])
let db = getDatabase()
let pres = safeStructuredClone(db.botPresets[id])
console.log(pres)
pres.openAIKey = ''
pres.forceReplaceUrl = ''
@@ -1476,10 +1660,10 @@ export async function importPreset(f:{
pre = {...presetTemplate,...(JSON.parse(Buffer.from(f.data).toString('utf-8')))}
console.log(pre)
}
let db = get(DataBase)
let db = getDatabase()
if(pre.presetVersion && pre.presetVersion >= 3){
//NAI preset
const pr = structuredClone(prebuiltPresets.NAI2)
const pr = safeStructuredClone(prebuiltPresets.NAI2)
pr.temperature = pre.parameters.temperature * 100
pr.maxResponse = pre.parameters.max_length
pr.NAISettings.topK = pre.parameters.top_k
@@ -1504,7 +1688,7 @@ export async function importPreset(f:{
if(Array.isArray(pre?.prompt_order?.[0]?.order) && Array.isArray(pre?.prompts)){
//ST preset
const pr = structuredClone(presetTemplate)
const pr = safeStructuredClone(presetTemplate)
pr.promptTemplate = []
function findPrompt(identifier:number){

6
src/ts/storage/exportAsDataset.ts
View File

@@ -1,11 +1,11 @@
import { get } from "svelte/store";
import { DataBase } from "./database";
import { downloadFile } from "./globalApi";
import { getDatabase } from "./database.svelte";
import { downloadFile } from "../globalApi.svelte";
import { alertNormal } from "../alert";
import { language } from "src/lang";
export async function exportAsDataset(){
const db = get(DataBase)
const db = getDatabase()
let dataset = []
for(const char of db.characters){

1691
src/ts/storage/globalApi.ts
View File

File diff suppressed because it is too large Load Diff

6
src/ts/storage/persistant.ts
View File

@@ -1,8 +1,8 @@
import { get } from "svelte/store";
import { DataBase } from "./database";
import { getDatabase } from "./database.svelte";
import { alertNormal } from "../alert";
import { language } from "src/lang";
import { isNodeServer, isTauri } from "./globalApi";
import { isNodeServer, isTauri } from "../globalApi.svelte";
async function requestPersistantStorageMain() {
@@ -38,7 +38,7 @@ async function requestPersistantStorageMain() {
}
export async function persistantStorageRecommended() {
const db = get(DataBase)
const db = getDatabase()
if(navigator.storage && navigator.storage.persist && (!isTauri) && (!isNodeServer)) {
if(await navigator.storage.persisted()) {
return false;

60
src/ts/storage/risuSave.ts
View File

@@ -1,6 +1,6 @@
import { Packr, Unpackr, decode } from "msgpackr";
import * as fflate from "fflate";
import { isTauri } from "./globalApi";
import { AppendableBuffer, isTauri } from "../globalApi.svelte";
const packr = new Packr({
useRecords:false
@@ -12,9 +12,22 @@ const unpackr = new Unpackr({
})
const magicHeader = new Uint8Array([0, 82, 73, 83, 85, 83, 65, 86, 69, 0, 7]);
const magicCompressedHeader = new Uint8Array([0, 82, 73, 83, 85, 83, 65, 86, 69, 0, 8]);
const magicCompressedHeader = new Uint8Array([0, 82, 73, 83, 85, 83, 65, 86, 69, 0, 8]);
const magicStreamCompressedHeader = new Uint8Array([0, 82, 73, 83, 85, 83, 65, 86, 69, 0, 9]);
export function encodeRisuSave(data:any, compression:'noCompression'|'compression' = 'noCompression'){
async function checkCompressionStreams(){
if(!CompressionStream){
const {makeCompressionStream} = await import('compression-streams-polyfill/ponyfill');
globalThis.CompressionStream = makeCompressionStream(TransformStream);
}
if(!DecompressionStream){
const {makeDecompressionStream} = await import('compression-streams-polyfill/ponyfill');
globalThis.DecompressionStream = makeDecompressionStream(TransformStream);
}
}
export function encodeRisuSaveLegacy(data:any, compression:'noCompression'|'compression' = 'noCompression'){
let encoded:Uint8Array = packr.encode(data)
if(compression === 'compression'){
encoded = fflate.compressSync(encoded)
@@ -31,7 +44,21 @@ export function encodeRisuSave(data:any, compression:'noCompression'|'compressio
}
}
export function decodeRisuSave(data:Uint8Array){
export async function encodeRisuSave(data:any) {
await checkCompressionStreams()
let encoded:Uint8Array = packr.encode(data)
const cs = new CompressionStream('gzip');
const writer = cs.writable.getWriter();
writer.write(encoded);
writer.close();
const buf = await new Response(cs.readable).arrayBuffer()
const result = new Uint8Array(new Uint8Array(buf).length + magicStreamCompressedHeader.length);
result.set(magicStreamCompressedHeader, 0)
result.set(new Uint8Array(buf), magicStreamCompressedHeader.length)
return result
}
export async function decodeRisuSave(data:Uint8Array){
try {
switch(checkHeader(data)){
case "compressed":
@@ -40,6 +67,16 @@ export function decodeRisuSave(data:Uint8Array){
case "raw":
data = data.slice(magicHeader.length)
return unpackr.decode(data)
case "stream":{
await checkCompressionStreams()
data = data.slice(magicStreamCompressedHeader.length)
const cs = new DecompressionStream('gzip');
const writer = cs.writable.getWriter();
writer.write(data);
writer.close();
const buf = await new Response(cs.readable).arrayBuffer()
return unpackr.decode(new Uint8Array(buf))
}
}
return unpackr.decode(data)
}
@@ -63,7 +100,7 @@ export function decodeRisuSave(data:Uint8Array){
function checkHeader(data: Uint8Array) {
let header:'none'|'compressed'|'raw' = 'raw'
let header:'none'|'compressed'|'raw'|'stream' = 'raw'
if (data.length < magicHeader.length) {
return false;
@@ -84,7 +121,18 @@ function checkHeader(data: Uint8Array) {
break
}
}
}
}
if(header === 'none'){
header = 'stream'
for (let i = 0; i < magicStreamCompressedHeader.length; i++) {
if (data[i] !== magicStreamCompressedHeader[i]) {
header = 'none'
break
}
}
}
// All bytes matched
return header;
}