[feat] data sync

src/lang/en.ts

@@ -312,6 +312,10 @@ export const languageEnglish = {
     recent: 'Recent',
     downloads: 'Downloads',
     trending: "Trending",
-    imageCompression: "Image Compression"
+    imageCompression: "Image Compression",
+    notLoggedIn: "Not Logged in to Risu Account",
+    googleDriveInfo: "Connect to google drive to sync your data.",
+    googleDriveConnection: "Google Drive Connection",
+    googleDriveConnected: "Google Drive Connected"
 
 }

src/lib/Setting/Pages/UserSettings.svelte

@@ -1,10 +1,40 @@
-<script>
+<script lang="ts">
     import { language } from "src/lang";
+  import Help from "src/lib/Others/Help.svelte";
+    import { hubURL } from "src/ts/characterCards";
     import { getCharImage, selectUserImg } from "src/ts/characters";
+    import { loadRisuAccountData, saveRisuAccountData } from "src/ts/drive/accounter";
+    import { checkDriver } from "src/ts/drive/drive";
     import { DataBase } from "src/ts/storage/database";
+    let openIframe = false
+    let openIframeURL = ''
+    let popup:Window = null
 </script>
 
+<svelte:window on:message={async (e) => {
+    if(e.origin.startsWith("https://sv.risuai.xyz") || e.origin.startsWith("http://127.0.0.1")){
+        if(e.data.msg.type === 'drive'){
+            console.log(e.data.msg)
+            await loadRisuAccountData()
+            $DataBase.account.data.refresh_token = e.data.msg.data.refresh_token
+            $DataBase.account.data.access_token = e.data.msg.data.access_token
+            $DataBase.account.data.expires_in = (e.data.msg.data.expires_in * 700) + Date.now()
+            await saveRisuAccountData()
+            popup.close()
+        }
+        else if(e.data.msg.data.vaild){
+            openIframe = false
+            $DataBase.account = {
+                id: e.data.msg.id,
+                token: e.data.msg.token,
+                data: e.data.msg.data
+            }
+        }
+    }
+}}></svelte:window>
+
 <h2 class="mb-2 text-2xl font-bold mt-2">{language.user}</h2>
+
 <span class="text-neutral-200 mt-2 mb-2">{language.userIcon}</span>
 <button on:click={() => {selectUserImg()}}>
     {#if $DataBase.userIcon === ''}
@@ -19,3 +49,44 @@
 </button>
 <span class="text-neutral-200 mt-4">{language.username}</span>
 <input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected" placeholder="User" bind:value={$DataBase.username}>
+{#if $DataBase.useExperimental}
+    <div class="bg-darkbg p-3 rounded-md mb-2 flex flex-col items-start">
+        <div class="w-full">
+            <h1 class="text-3xl font-black min-w-0">Risu Account{#if $DataBase.account} <Help key="experimental"/>
+                <button class="bg-selected p-1 text-sm font-light rounded-md hover:bg-green-500 transition-colors float-right" on:click={async () => {
+                    $DataBase.account = undefined
+                }}>Logout</button>
+            {/if}</h1>
+        </div>
+        {#if $DataBase.account}
+            <span class="mb-4 text-gray-400">ID: {$DataBase.account.id}</span>
+            <h1 class="text-xl font-bold mt-2">{language.googleDriveConnection}</h1>
+            {#if !$DataBase.account.data.refresh_token}
+                <span class="text-sm font-light mb-2 text-gray-400">{language.googleDriveInfo}</span>
+                <button class="bg-selected p-2 rounded-md hover:bg-green-500 transition-colors" on:click={async () => {
+                    if((!popup) || popup.closed){
+                        popup = window.open(await checkDriver('reftoken'))                    
+                    }
+                }}>
+                    Connect to Google Drive
+                </button>
+            {:else}
+                <span class="text-sm font-light mb-2 text-gray-400">{language.googleDriveConnected}</span>
+            {/if}
+        {:else}
+            <span>{language.notLoggedIn}</span>
+            <button class="bg-selected p-2 rounded-md mt-2 hover:bg-green-500 transition-colors" on:click={() => {
+                openIframeURL = hubURL + '/hub/login'
+                openIframe = true
+            }}>
+                Login
+            </button>
+        {/if}
+    </div>
+{/if}
+{#if openIframe}
+    <div class="fixed top-0 left-0 bg-black bg-opacity-50 w-full h-full flex justify-center items-center">
+        <iframe src={openIframeURL} title="login" class="w-full h-full">
+        </iframe>
+    </div>
+{/if}

src/ts/drive/accounter.ts

@@ -1,3 +1,52 @@
-async function loginWithGoogle() {
+import { get } from "svelte/store"
-    
+import { hubURL } from "../characterCards"
+import { DataBase } from "../storage/database"
+import { alertError } from "../alert"
+
+export async function risuLogin() {
+    const win = window.open(hubURL + '/hub/login')
+    window.addEventListener("message", (ev) => {
+        console.log(ev)
+        const data = JSON.parse(ev.data)
+        console.log(data)
+        win.close()
+    })
+}
+
+export async function saveRisuAccountData() {
+    const db = get(DataBase)
+    if(!db.account){
+        alertError("Not logged in error")
+        return
+    }
+    const s = await fetch(hubURL + '/hub/account/save', {
+        method: "POST",
+        body: JSON.stringify({
+            token: db.account.token,
+            save: db.account.data
+        })
+    })
+    if(s.status !== 200){
+        alertError(await s.text())
+        return
+    }
+}
+
+export async function loadRisuAccountData() {
+    const db = get(DataBase)
+    if(!db.account){
+        alertError("Not logged in error")
+        return
+    }
+    const s = await fetch(hubURL + '/hub/account/load', {
+        method: "POST",
+        body: JSON.stringify({
+            token: db.account.token
+        })
+    })
+    if(s.status !== 200){
+        alertError(await s.text())
+        return
+    }
+    db.account.data = await s.json()
 }

src/ts/drive/drive.ts

@@ -7,10 +7,13 @@ import { BaseDirectory, exists, readBinaryFile, readDir, writeBinaryFile } from
 import { language } from "../../lang";
 import { relaunch } from '@tauri-apps/api/process';
 import { open } from '@tauri-apps/api/shell';
+import { cloneDeep, isEqual, last } from "lodash";
+import { sleep } from "../util";
+import { hubURL } from "../characterCards";
 
 export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'reftoken'){
     const CLIENT_ID = '580075990041-l26k2d3c0nemmqiu3d3aag01npfrkn76.apps.googleusercontent.com';
-    const REDIRECT_URI = (isTauri || isNodeServer) ? "https://risuai.xyz/" : `https://${location.host}/`
+    const REDIRECT_URI = 'reftoken' ? 'https://sv.risuai.xyz/drive' : ((isTauri || isNodeServer) ? "https://risuai.xyz/" : `https://${location.host}/`)
     const SCOPE = 'https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/drive.appdata';
     const encodedRedirectUri = encodeURIComponent(REDIRECT_URI);
     const authorizationUrl = `https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${encodedRedirectUri}&scope=${SCOPE}&response_type=code&state=${type}`;
@@ -18,8 +21,7 @@ export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'re
 
     if(type === 'reftoken'){
         const authorizationUrl = `https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${encodedRedirectUri}&scope=${SCOPE}&response_type=code&state=${"accesstauri"}&access_type=offline&prompt=consent`;
-        openURL(authorizationUrl)
+        return authorizationUrl
-        return
     }
 
     if(type === 'save' || type === 'load'){
@@ -39,7 +41,7 @@ export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'re
                 code = code.substring(code.lastIndexOf(' ')).trim()
             }
             if(type === 'loadtauri'){
-                await loadDrive(code)
+                await loadDrive(code, 'backup')
             }
             else{
                 await backupDrive(code)
@@ -69,7 +71,7 @@ export async function checkDriverInit() {
                     await backupDrive(json.access_token)
                 }
                 else if(da === 'load'){
-                    await loadDrive(json.access_token)
+                    await loadDrive(json.access_token, 'backup')
                 }
                 else if(da === 'savetauri' || da === 'loadtauri'){
                     alertStore.set({
@@ -101,8 +103,79 @@ export async function checkDriverInit() {
     }
 }
 
+let lastSaved:number = parseInt(localStorage.getItem('risu_lastsaved') ?? '-1')
+let BackupDb:Database = null
 
 
+export async function syncDrive() {
+    BackupDb = cloneDeep(get(DataBase))
+    while(true){
+        const maindb = get(DataBase)
+        if(maindb?.account?.data?.access_token && maindb?.account?.data?.refresh_token && maindb?.account?.data?.expires_in){
+            if(maindb.account.data.expires_in < Date.now()){
+                if(!maindb.account){
+                    alertError("Not logged in error")
+                    return
+                }
+                const s = await fetch(hubURL + '/drive/refresh', {
+                    method: "POST",
+                    body: JSON.stringify({
+                        token: maindb.account.token
+                    })
+                })
+                if(s.status !== 200){
+                    alertError(await s.text())
+                    return
+                }
+                maindb.account.data = await s.json()
+            }
+            const ACCESS_TOKEN = maindb.account.data.access_token
+            await loadDrive(ACCESS_TOKEN, 'sync')
+            if(!isEqual(maindb, BackupDb)){
+                BackupDb = cloneDeep(maindb)
+                const files:DriveFile[] = await getFilesInFolder(ACCESS_TOKEN)
+                const fileNames = files.map((d) => {
+                    return d.name
+                })
+                if(isTauri){
+                    const assets = await readDir('assets', {dir: BaseDirectory.AppData})
+                    let i = 0;
+                    for(let asset of assets){
+                        i += 1;
+                        const key = asset.name
+                        if(!key || !key.endsWith('.png')){
+                            continue
+                        }
+                        const formatedKey = formatKeys(key)
+                        if(!fileNames.includes(formatedKey)){
+                            await createFileInFolder(ACCESS_TOKEN, formatedKey, await readBinaryFile(asset.path))
+                        }
+                    }
+                }
+                else{
+                    const keys = await forageStorage.keys()
+            
+                    for(let i=0;i<keys.length;i++){
+                        const key = keys[i]
+                        if(!key.endsWith('.png')){
+                            continue
+                        }
+                        const formatedKey = formatKeys(key)
+                        if(!fileNames.includes(formatedKey)){
+                            await createFileInFolder(ACCESS_TOKEN, formatedKey, await forageStorage.getItem(key))
+                        }
+                    }
+                }
+                const dbjson = JSON.stringify(get(DataBase))
+                lastSaved = Math.floor(Date.now() / 1000)
+                localStorage.setItem('risu_lastsaved', `${lastSaved}`)
+                await createFileInFolder(ACCESS_TOKEN, `${lastSaved}-database.risudat2`, Buffer.from(dbjson, 'utf-8'))
+            }
+        }
+        await sleep(3000)
+    }
+}
+
 
 async function backupDrive(ACCESS_TOKEN:string) {
     alertStore.set({
@@ -176,11 +249,13 @@ type DriveFile = {
     id: string
 }
 
-async function loadDrive(ACCESS_TOKEN:string) {
+async function loadDrive(ACCESS_TOKEN:string, mode: 'backup'|'sync') {
-    alertStore.set({
+    if(mode === 'backup'){
-        type: "wait",
+        alertStore.set({
-        msg: "Loading Backup..."
+            type: "wait",
-    })
+            msg: "Loading Backup..."
+        })
+    }
     const files:DriveFile[] = await getFilesInFolder(ACCESS_TOKEN)
     let foragekeys:string[] = []
     let loadedForageKeys = false
@@ -204,42 +279,80 @@ async function loadDrive(ACCESS_TOKEN:string) {
 
     let dbs:[DriveFile,number][] = []
 
-    for(const f of files){
+    if(mode === 'backup'){
-        if(f.name.endsWith("-database.risudat")){
+        for(const f of files){
-            const tm = parseInt(f.name.split('-')[0])
+            if(f.name.endsWith("-database.risudat")){
-            if(isNaN(tm)){
+                const tm = parseInt(f.name.split('-')[0])
-                continue
+                if(isNaN(tm)){
-            }
+                    continue
-            else{
+                }
-                dbs.push([f,tm])
+                else{
+                    dbs.push([f,tm])
+                }
             }
         }
+        dbs.sort((a,b) => {
+            return b[1] - a[1]
+        })
+    }
+    else if(mode === 'sync'){
+        for(const f of files){
+            if(f.name.endsWith("-database.risudat2")){
+                const tm = parseInt(f.name.split('-')[0])
+                if(isNaN(tm)){
+                    continue
+                }
+                else if(tm > lastSaved){
+                    dbs.push([f,tm])
+                }
+            }
+        }
+        dbs.sort((a,b) => {
+            return b[1] - a[1]
+        })
     }
-    dbs.sort((a,b) => {
-        return b[1] - a[1]
-    })
 
     if(dbs.length !== 0){
-        let selectables:string[] = []
+        if(mode === 'sync'){
-        for(let i=0;i<dbs.length;i++){
+            alertStore.set({
-            selectables.push(`Backup saved in ${(new Date(dbs[i][1] * 1000)).toLocaleString()}`)
+                type: "wait",
-            if(selectables.length > 7){
+                msg: "Sync Data..."
-                break
+            })
-            }
         }
-        const selectedIndex = (await alertSelect([language.loadLatest, language.loadOthers]) === '0') ? 0 : parseInt(await alertSelect(selectables))
+        async function getDbFromList(){
-        const selectedDb = dbs[selectedIndex][0]
+            let selectables:string[] = []
-        
+            for(let i=0;i<dbs.length;i++){
-        const db:Database = JSON.parse(Buffer.from(pako.inflate(await getFileData(ACCESS_TOKEN, selectedDb.id))).toString('utf-8'))
+                selectables.push(`Backup saved in ${(new Date(dbs[i][1] * 1000)).toLocaleString()}`)
+                if(selectables.length > 7){
+                    break
+                }
+            }
+            const selectedIndex = (await alertSelect([language.loadLatest, language.loadOthers]) === '0') ? 0 : parseInt(await alertSelect(selectables))
+            const selectedDb = dbs[selectedIndex][0]
+            const decompressedDb:Database = JSON.parse(Buffer.from(pako.inflate(await getFileData(ACCESS_TOKEN, selectedDb.id))).toString('utf-8'))
+            return decompressedDb
+        }
+    
+        const db:Database = mode === 'backup' ? await getDbFromList() : JSON.parse(Buffer.from(await getFileData(ACCESS_TOKEN, dbs[0][0].id)).toString('utf-8'))
+        lastSaved = Date.now()
+        localStorage.setItem('risu_lastsaved', `${lastSaved}`)
         const requiredImages = (getUnpargeables(db))
         let ind = 0;
         for(const images of requiredImages){
             ind += 1
             const formatedImage = formatKeys(images)
-            alertStore.set({
+            if(mode === 'sync'){
-                type: "wait",
+                alertStore.set({
-                msg: `Loading Backup... (${ind} / ${requiredImages.length})`
+                    type: "wait",
-            })
+                    msg: `Sync Files... (${ind} / ${requiredImages.length})`
+                })
+            }
+            else{
+                alertStore.set({
+                    type: "wait",
+                    msg: `Loading Backup... (${ind} / ${requiredImages.length})`
+                })
+            }
             if(await checkImageExists(images)){
                 //skip process
             }
@@ -276,7 +389,7 @@ async function loadDrive(ACCESS_TOKEN:string) {
             relaunch()
             alertStore.set({
                 type: "wait",
-                msg: "Success, Refresh your app."
+                msg: "Success, Refreshing your app."
             })
         }
         else{
@@ -284,11 +397,11 @@ async function loadDrive(ACCESS_TOKEN:string) {
             location.search = ''
             alertStore.set({
                 type: "wait",
-                msg: "Success, Refresh your app."
+                msg: "Success, Refreshing your app."
             })
         }
     }
-    else{
+    else if(mode === 'backup'){
         location.search = ''
     }
 }

src/ts/storage/database.ts

@@ -509,6 +509,15 @@ export interface Database{
     novellistAPI:string,
     useAutoTranslateInput:boolean
     imageCompression:boolean
+    account?:{
+        token:string
+        id:string,
+        data: {
+            refresh_token?:string,
+            access_token?:string
+            expires_in?: number
+        }
+    }
 }
 
 interface hordeConfig{

src/ts/storage/globalApi.ts

@@ -14,12 +14,13 @@ import { selectedCharID } from "../stores";
 import { Body, ResponseType, fetch as TauriFetch } from "@tauri-apps/api/http";
 import { loadPlugins } from "../process/plugins";
 import { alertError, alertStore } from "../alert";
-import { checkDriverInit } from "../drive/drive";
+import { checkDriverInit, syncDrive } from "../drive/drive";
 import { hasher } from "../parser";
 import { characterHubImport } from "../characterCards";
 import { cloneDeep } from "lodash";
 import { NodeStorage } from "./nodeStorage";
 import { defaultJailbreak, defaultMainPrompt, oldJailbreak, oldMainPrompt } from "./defaultPrompts";
+import { loadRisuAccountData } from "../drive/accounter";
 
 //@ts-ignore
 export const isTauri = !!window.__TAURI__
@@ -195,6 +196,7 @@ let lastSave = ''
 
 export async function saveDb(){
     lastSave =JSON.stringify(get(DataBase))
+    syncDrive()
     while(true){
         const dbjson = JSON.stringify(get(DataBase))
         if(dbjson !== lastSave){
@@ -358,6 +360,11 @@ export async function loadData() {
             } catch (error) {}
             await checkNewFormat()
             updateTextTheme()
+            if(get(DataBase).account){
+                try {
+                    await loadRisuAccountData()                    
+                } catch (error) {}
+            }
             loadedStore.set(true)
             selectedCharID.set(-1)
             saveDb()   

src/ts/tokenizer.ts

@@ -49,7 +49,6 @@ export class ChatTokenizer {
         this.useName = useName
     }
     async tokenizeChat(data:OpenAIChat) {
-        console.log(data.content)
         let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
         if(data.name && this.useName ==='name'){
             encoded += (await encode(data.name)).length