[feat] opfs storage

package.json

@@ -73,8 +73,8 @@
     "tailwindcss": "^3.3.1",
     "tslib": "^2.4.1",
     "typescript": "^4.9.5",
-    "vite": "^4.2.1",
-    "vite-plugin-top-level-await": "^1.3.0",
+    "vite": "^4.3.9",
+    "vite-plugin-top-level-await": "^1.3.1",
     "vite-plugin-wasm": "^3.2.2"
   }
 }

pnpm-lock.yaml

@@ -107,7 +107,7 @@ dependencies:
 devDependencies:
   '@sveltejs/vite-plugin-svelte':
     specifier: ^2.0.0
-    version: 2.0.4(svelte@3.58.0)(vite@4.2.1)
+    version: 2.0.4(svelte@3.58.0)(vite@4.3.9)
   '@tailwindcss/typography':
     specifier: ^0.5.9
     version: 0.5.9(tailwindcss@3.3.1)
@@ -175,14 +175,14 @@ devDependencies:
     specifier: ^4.9.5
     version: 4.9.5
   vite:
-    specifier: ^4.2.1
-    version: 4.2.1(@types/node@18.15.11)
+    specifier: ^4.3.9
+    version: 4.3.9(@types/node@18.15.11)
   vite-plugin-top-level-await:
-    specifier: ^1.3.0
-    version: 1.3.0(rollup@3.21.3)(vite@4.2.1)
+    specifier: ^1.3.1
+    version: 1.3.1(rollup@3.21.3)(vite@4.3.9)
   vite-plugin-wasm:
     specifier: ^3.2.2
-    version: 3.2.2(vite@4.2.1)
+    version: 3.2.2(vite@4.3.9)
 
 packages:
 
@@ -536,7 +536,7 @@ packages:
       rollup: 3.21.3
     dev: true
 
-  /@sveltejs/vite-plugin-svelte@2.0.4(svelte@3.58.0)(vite@4.2.1):
+  /@sveltejs/vite-plugin-svelte@2.0.4(svelte@3.58.0)(vite@4.3.9):
     resolution: {integrity: sha512-pjqhW00KwK2uzDGEr+yJBwut+D+4XfJO/+bHHdHzPRXn9+1Jeq5JcFHyrUiYaXgHtyhX0RsllCTm4ssAx4ZY7Q==}
     engines: {node: ^14.18.0 || >= 16}
     peerDependencies:
@@ -549,8 +549,8 @@ packages:
       magic-string: 0.30.0
       svelte: 3.58.0
       svelte-hmr: 0.15.1(svelte@3.58.0)
-      vite: 4.2.1(@types/node@18.15.11)
-      vitefu: 0.2.4(vite@4.2.1)
+      vite: 4.3.9(@types/node@18.15.11)
+      vitefu: 0.2.4(vite@4.3.9)
     transitivePeerDependencies:
       - supports-color
     dev: true
@@ -3265,30 +3265,30 @@ packages:
     engines: {node: '>= 0.8'}
     dev: false
 
-  /vite-plugin-top-level-await@1.3.0(rollup@3.21.3)(vite@4.2.1):
-    resolution: {integrity: sha512-owIfsgWudMlQODWJSwp0sQB3AZZu3qsMygeBjZy8CyjEk6OB9AGd8lHqmgwrcEqgvy9N58lYxSBLVk3/4ejEiA==}
+  /vite-plugin-top-level-await@1.3.1(rollup@3.21.3)(vite@4.3.9):
+    resolution: {integrity: sha512-55M1h4NAwkrpxPNOJIBzKZFihqLUzIgnElLSmPNPMR2Fn9+JHKaNg3sVX1Fq+VgvuBksQYxiD3OnwQAUu7kaPQ==}
     peerDependencies:
       vite: '>=2.8'
     dependencies:
       '@rollup/plugin-virtual': 3.0.1(rollup@3.21.3)
       '@swc/core': 1.3.49
       uuid: 9.0.0
-      vite: 4.2.1(@types/node@18.15.11)
+      vite: 4.3.9(@types/node@18.15.11)
     transitivePeerDependencies:
       - '@swc/helpers'
       - rollup
     dev: true
 
-  /vite-plugin-wasm@3.2.2(vite@4.2.1):
+  /vite-plugin-wasm@3.2.2(vite@4.3.9):
     resolution: {integrity: sha512-cdbBUNR850AEoMd5nvLmnyeq63CSfoP1ctD/L2vLk/5+wsgAPlAVAzUK5nGKWO/jtehNlrSSHLteN+gFQw7VOA==}
     peerDependencies:
       vite: ^2 || ^3 || ^4
     dependencies:
-      vite: 4.2.1(@types/node@18.15.11)
+      vite: 4.3.9(@types/node@18.15.11)
     dev: true
 
-  /vite@4.2.1(@types/node@18.15.11):
-    resolution: {integrity: sha512-7MKhqdy0ISo4wnvwtqZkjke6XN4taqQ2TBaTccLIpOKv7Vp2h4Y+NpmWCnGDeSvvn45KxvWgGyb0MkHvY1vgbg==}
+  /vite@4.3.9(@types/node@18.15.11):
+    resolution: {integrity: sha512-qsTNZjO9NoJNW7KnOrgYwczm0WctJ8m/yqYAMAK9Lxt4SoySUfS5S8ia9K7JHpa3KEeMfyF8LoJ3c5NeBJy6pg==}
     engines: {node: ^14.18.0 || >=16.0.0}
     hasBin: true
     peerDependencies:
@@ -3315,13 +3315,12 @@ packages:
       '@types/node': 18.15.11
       esbuild: 0.17.15
       postcss: 8.4.23
-      resolve: 1.22.1
       rollup: 3.21.3
     optionalDependencies:
       fsevents: 2.3.2
     dev: true
 
-  /vitefu@0.2.4(vite@4.2.1):
+  /vitefu@0.2.4(vite@4.3.9):
     resolution: {integrity: sha512-fanAXjSaf9xXtOOeno8wZXIhgia+CZury481LsDaV++lSvcU2R9Ch2bPh3PYFyoHW+w9LqAeYRISVQjUIew14g==}
     peerDependencies:
       vite: ^3.0.0 || ^4.0.0
@@ -3329,7 +3328,7 @@ packages:
       vite:
         optional: true
     dependencies:
-      vite: 4.2.1(@types/node@18.15.11)
+      vite: 4.3.9(@types/node@18.15.11)
     dev: true
 
   /w3c-xmlserializer@4.0.0:

src/ts/storage/autoStorage.ts

@@ -0,0 +1,77 @@
+import localforage from "localforage"
+import { isNodeServer } from "./globalApi"
+import { NodeStorage } from "./nodeStorage"
+import { OpfsStorage } from "./opfsStorage"
+import { alertConfirm, alertStore } from "../alert"
+
+export class AutoStorage{
+
+    realStorage:LocalForage|NodeStorage|OpfsStorage
+
+    async setItem(key:string, value:Uint8Array) {
+        await this.Init()
+        return await this.realStorage.setItem(key, value)
+    }
+    async getItem(key:string):Promise<Buffer> {
+        await this.Init()
+        return await this.realStorage.getItem(key)
+
+    }
+    async keys():Promise<string[]>{
+        await this.Init()
+        return await this.realStorage.keys()
+
+    }
+    async removeItem(key:string){
+        await this.Init()
+        return await this.realStorage.removeItem(key)
+    }
+
+    private async Init(){
+        if(!this.realStorage){
+            if(isNodeServer){
+                console.log("using node storage")
+                this.realStorage = new NodeStorage()
+                return
+            }
+            else if(window.navigator?.storage?.getDirectory && localStorage.getItem('flag_opfs')){
+                console.log("using opfs storage")
+
+                const forage = localforage.createInstance({
+                    name: "risuai"
+                })
+
+                const i = await forage.getItem("database/database.bin")
+
+                if((!i) || (await forage.getItem("migrated"))){
+                    this.realStorage = new OpfsStorage()
+                    return
+                }
+                else if(!(await forage.getItem("denied_opfs"))){
+                    console.log("migrating")
+                    const keys = await forage.keys()
+                    let i = 0;
+                    const opfs = new OpfsStorage()
+                    for(const key of keys){
+                        console.log(i)
+                        alertStore.set({
+                            type: "none",
+                            msg: `Migrating your data...(${i}/${keys.length})`
+                        })
+                        await opfs.setItem(key,await forage.getItem(key))
+                        i += 1
+                    }
+                    this.realStorage = opfs
+                    await forage.setItem("migrated", true)
+                    return
+                }
+            }
+            console.log("using forage storage")
+            this.realStorage = localforage.createInstance({
+                name: "risuai"
+            })
+        }
+    }
+
+    listItem = this.keys
+}

src/ts/storage/globalApi.ts

@@ -1,6 +1,5 @@
 import { writeBinaryFile,BaseDirectory, readBinaryFile, exists, createDir, readDir, removeFile } from "@tauri-apps/api/fs"
 import { changeFullscreen, checkNullish, findCharacterbyId, sleep } from "../util"
-import localforage from 'localforage'
 import { convertFileSrc, invoke } from "@tauri-apps/api/tauri"
 import { v4 as uuidv4 } from 'uuid';
 import { appDataDir, join } from "@tauri-apps/api/path";
@@ -12,23 +11,21 @@ import { checkOldDomain, checkUpdate } from "../update";
 import { selectedCharID } from "../stores";
 import { Body, ResponseType, fetch as TauriFetch } from "@tauri-apps/api/http";
 import { loadPlugins } from "../plugins/plugins";
-import { alertError, alertStore } from "../alert";
+import { alertError } from "../alert";
 import { checkDriverInit, syncDrive } from "../drive/drive";
 import { hasher } from "../parser";
 import { characterHubImport } from "../characterCards";
 import { cloneDeep } from "lodash";
-import { NodeStorage } from "./nodeStorage";
 import { defaultJailbreak, defaultMainPrompt, oldJailbreak, oldMainPrompt } from "./defaultPrompts";
 import { loadRisuAccountData } from "../drive/accounter";
 import { decodeRisuSave, encodeRisuSave } from "./risuSave";
+import { AutoStorage } from "./autoStorage";
 
 //@ts-ignore
 export const isTauri = !!window.__TAURI__
 //@ts-ignore
 export const isNodeServer = !!globalThis.__NODE__
-export const forageStorage = isNodeServer ? new NodeStorage() : localforage.createInstance({
-    name: "risuai"
-})
+export const forageStorage = new AutoStorage()
 
 interface fetchLog{
     body:string

src/ts/storage/opfsStorage.ts

@@ -0,0 +1,61 @@
+export class OpfsStorage{
+
+    opfs:FileSystemDirectoryHandle
+
+    async setItem(key:string, value:Uint8Array) {
+        await this.Init()
+        const handle = await this.opfs.getFileHandle(Buffer.from(key, 'utf-8').toString('hex'), {
+            create: true
+        })
+        const stream = await handle.createWritable()
+        await stream.write(value)
+        stream.close()
+    }
+    async getItem(key:string):Promise<Buffer> {
+        try {
+            await this.Init()
+            const handle = await this.opfs.getFileHandle(Buffer.from(key, 'utf-8').toString('hex'), {
+                create: false
+            })
+            const stream = await handle.getFile();
+    
+            return Buffer.from(await stream.arrayBuffer())   
+        } catch (error) {
+            if(error instanceof DOMException){
+                if(error.name === "NotFoundError"){
+                    return null
+                }
+            }
+            throw error
+        }
+    }
+    async keys():Promise<string[]>{
+        await this.Init()
+        let entries:string[] = []
+        for await (const entry of this.opfs.values()) {
+            entries.push(Buffer.from(entry.name, 'hex').toString('utf-8'))
+        }
+        return entries
+    }
+    async removeItem(key:string){
+        try {
+            await this.Init()
+            await this.opfs.removeEntry(Buffer.from(key, 'utf-8').toString('hex'))
+        } catch (error) {
+            if(error instanceof DOMException){
+                if(error.name === "NotFoundError"){
+                    return null
+                }
+            }
+            throw error
+        }
+    }
+
+    private async Init(){
+        if(!this.opfs){
+            this.opfs = await window.navigator.storage.getDirectory()
+        }
+    }
+
+    listItem = this.keys
+}

vite.config.ts

@@ -55,6 +55,12 @@ export default defineConfig(async () => {
       sourcemap: !!process.env.TAURI_DEBUG,
       chunkSizeWarningLimit: 2000
     },
+    
+    optimizeDeps:{
+      needsInterop:[
+        "@mlc-ai/web-tokenizers"
+      ]
+    },
 
     resolve:{
       alias:{