mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] data sync

Browse Source
This commit is contained in:
kwaroran
2023-06-12 12:27:01 +09:00
parent 49a01a0950
commit 2de284f4e9
7 changed files with 296 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/lang/en.ts
View File

@@ -312,6 +312,10 @@ export const languageEnglish = {
recent: 'Recent',
downloads: 'Downloads',
trending: "Trending",
imageCompression: "Image Compression"
imageCompression: "Image Compression",
notLoggedIn: "Not Logged in to Risu Account",
googleDriveInfo: "Connect to google drive to sync your data.",
googleDriveConnection: "Google Drive Connection",
googleDriveConnected: "Google Drive Connected"
}

73
src/lib/Setting/Pages/UserSettings.svelte
View File

@@ -1,10 +1,40 @@
<script>
<script lang="ts">
import { language } from "src/lang";
import Help from "src/lib/Others/Help.svelte";
import { hubURL } from "src/ts/characterCards";
import { getCharImage, selectUserImg } from "src/ts/characters";
import { loadRisuAccountData, saveRisuAccountData } from "src/ts/drive/accounter";
import { checkDriver } from "src/ts/drive/drive";
import { DataBase } from "src/ts/storage/database";
let openIframe = false
let openIframeURL = ''
let popup:Window = null
</script>
<svelte:window on:message={async (e) => {
if(e.origin.startsWith("https://sv.risuai.xyz") || e.origin.startsWith("http://127.0.0.1")){
if(e.data.msg.type === 'drive'){
console.log(e.data.msg)
await loadRisuAccountData()
$DataBase.account.data.refresh_token = e.data.msg.data.refresh_token
$DataBase.account.data.access_token = e.data.msg.data.access_token
$DataBase.account.data.expires_in = (e.data.msg.data.expires_in * 700) + Date.now()
await saveRisuAccountData()
popup.close()
}
else if(e.data.msg.data.vaild){
openIframe = false
$DataBase.account = {
id: e.data.msg.id,
token: e.data.msg.token,
data: e.data.msg.data
}
}
}
}}></svelte:window>
<h2 class="mb-2 text-2xl font-bold mt-2">{language.user}</h2>
<span class="text-neutral-200 mt-2 mb-2">{language.userIcon}</span>
<button on:click={() => {selectUserImg()}}>
{#if $DataBase.userIcon === ''}
@@ -19,3 +49,44 @@
</button>
<span class="text-neutral-200 mt-4">{language.username}</span>
<input class="text-neutral-200 mt-2 mb-4 p-2 bg-transparent input-text focus:bg-selected" placeholder="User" bind:value={$DataBase.username}>
{#if $DataBase.useExperimental}
<div class="bg-darkbg p-3 rounded-md mb-2 flex flex-col items-start">
<div class="w-full">
<h1 class="text-3xl font-black min-w-0">Risu Account{#if $DataBase.account} <Help key="experimental"/>
<button class="bg-selected p-1 text-sm font-light rounded-md hover:bg-green-500 transition-colors float-right" on:click={async () => {
$DataBase.account = undefined
}}>Logout</button>
{/if}</h1>
</div>
{#if $DataBase.account}
<span class="mb-4 text-gray-400">ID: {$DataBase.account.id}</span>
<h1 class="text-xl font-bold mt-2">{language.googleDriveConnection}</h1>
{#if !$DataBase.account.data.refresh_token}
<span class="text-sm font-light mb-2 text-gray-400">{language.googleDriveInfo}</span>
<button class="bg-selected p-2 rounded-md hover:bg-green-500 transition-colors" on:click={async () => {
if((!popup) || popup.closed){
popup = window.open(await checkDriver('reftoken'))
}
}}>
Connect to Google Drive
</button>
{:else}
<span class="text-sm font-light mb-2 text-gray-400">{language.googleDriveConnected}</span>
{/if}
{:else}
<span>{language.notLoggedIn}</span>
<button class="bg-selected p-2 rounded-md mt-2 hover:bg-green-500 transition-colors" on:click={() => {
openIframeURL = hubURL + '/hub/login'
openIframe = true
}}>
Login
</button>
{/if}
</div>
{/if}
{#if openIframe}
<div class="fixed top-0 left-0 bg-black bg-opacity-50 w-full h-full flex justify-center items-center">
<iframe src={openIframeURL} title="login" class="w-full h-full">
</iframe>
</div>
{/if}

53
src/ts/drive/accounter.ts
View File

@@ -1,3 +1,52 @@
async function loginWithGoogle() {
import { get } from "svelte/store"
import { hubURL } from "../characterCards"
import { DataBase } from "../storage/database"
import { alertError } from "../alert"
export async function risuLogin() {
const win = window.open(hubURL + '/hub/login')
window.addEventListener("message", (ev) => {
console.log(ev)
const data = JSON.parse(ev.data)
console.log(data)
win.close()
})
}
export async function saveRisuAccountData() {
const db = get(DataBase)
if(!db.account){
alertError("Not logged in error")
return
}
const s = await fetch(hubURL + '/hub/account/save', {
method: "POST",
body: JSON.stringify({
token: db.account.token,
save: db.account.data
})
})
if(s.status !== 200){
alertError(await s.text())
return
}
}
export async function loadRisuAccountData() {
const db = get(DataBase)
if(!db.account){
alertError("Not logged in error")
return
}
const s = await fetch(hubURL + '/hub/account/load', {
method: "POST",
body: JSON.stringify({
token: db.account.token
})
})
if(s.status !== 200){
alertError(await s.text())
return
}
db.account.data = await s.json()
}

189
src/ts/drive/drive.ts
View File

@@ -7,10 +7,13 @@ import { BaseDirectory, exists, readBinaryFile, readDir, writeBinaryFile } from
import { language } from "../../lang";
import { relaunch } from '@tauri-apps/api/process';
import { open } from '@tauri-apps/api/shell';
import { cloneDeep, isEqual, last } from "lodash";
import { sleep } from "../util";
import { hubURL } from "../characterCards";
export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'reftoken'){
const CLIENT_ID = '580075990041-l26k2d3c0nemmqiu3d3aag01npfrkn76.apps.googleusercontent.com';
const REDIRECT_URI = (isTauri || isNodeServer) ? "https://risuai.xyz/" : `https://${location.host}/`
const REDIRECT_URI = 'reftoken' ? 'https://sv.risuai.xyz/drive' : ((isTauri || isNodeServer) ? "https://risuai.xyz/" : `https://${location.host}/`)
const SCOPE = 'https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/drive.appdata';
const encodedRedirectUri = encodeURIComponent(REDIRECT_URI);
const authorizationUrl = `https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${encodedRedirectUri}&scope=${SCOPE}&response_type=code&state=${type}`;
@@ -18,8 +21,7 @@ export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'re
if(type === 'reftoken'){
const authorizationUrl = `https://accounts.google.com/o/oauth2/auth?client_id=${CLIENT_ID}&redirect_uri=${encodedRedirectUri}&scope=${SCOPE}&response_type=code&state=${"accesstauri"}&access_type=offline&prompt=consent`;
openURL(authorizationUrl)
return
return authorizationUrl
}
if(type === 'save' || type === 'load'){
@@ -39,7 +41,7 @@ export async function checkDriver(type:'save'|'load'|'loadtauri'|'savetauri'|'re
code = code.substring(code.lastIndexOf(' ')).trim()
}
if(type === 'loadtauri'){
await loadDrive(code)
await loadDrive(code, 'backup')
}
else{
await backupDrive(code)
@@ -69,7 +71,7 @@ export async function checkDriverInit() {
await backupDrive(json.access_token)
}
else if(da === 'load'){
await loadDrive(json.access_token)
await loadDrive(json.access_token, 'backup')
}
else if(da === 'savetauri' || da === 'loadtauri'){
alertStore.set({
@@ -101,8 +103,79 @@ export async function checkDriverInit() {
}
}
let lastSaved:number = parseInt(localStorage.getItem('risu_lastsaved') ?? '-1')
let BackupDb:Database = null
export async function syncDrive() {
BackupDb = cloneDeep(get(DataBase))
while(true){
const maindb = get(DataBase)
if(maindb?.account?.data?.access_token && maindb?.account?.data?.refresh_token && maindb?.account?.data?.expires_in){
if(maindb.account.data.expires_in < Date.now()){
if(!maindb.account){
alertError("Not logged in error")
return
}
const s = await fetch(hubURL + '/drive/refresh', {
method: "POST",
body: JSON.stringify({
token: maindb.account.token
})
})
if(s.status !== 200){
alertError(await s.text())
return
}
maindb.account.data = await s.json()
}
const ACCESS_TOKEN = maindb.account.data.access_token
await loadDrive(ACCESS_TOKEN, 'sync')
if(!isEqual(maindb, BackupDb)){
BackupDb = cloneDeep(maindb)
const files:DriveFile[] = await getFilesInFolder(ACCESS_TOKEN)
const fileNames = files.map((d) => {
return d.name
})
if(isTauri){
const assets = await readDir('assets', {dir: BaseDirectory.AppData})
let i = 0;
for(let asset of assets){
i += 1;
const key = asset.name
if(!key || !key.endsWith('.png')){
continue
}
const formatedKey = formatKeys(key)
if(!fileNames.includes(formatedKey)){
await createFileInFolder(ACCESS_TOKEN, formatedKey, await readBinaryFile(asset.path))
}
}
}
else{
const keys = await forageStorage.keys()
for(let i=0;i<keys.length;i++){
const key = keys[i]
if(!key.endsWith('.png')){
continue
}
const formatedKey = formatKeys(key)
if(!fileNames.includes(formatedKey)){
await createFileInFolder(ACCESS_TOKEN, formatedKey, await forageStorage.getItem(key))
}
}
}
const dbjson = JSON.stringify(get(DataBase))
lastSaved = Math.floor(Date.now() / 1000)
localStorage.setItem('risu_lastsaved', `${lastSaved}`)
await createFileInFolder(ACCESS_TOKEN, `${lastSaved}-database.risudat2`, Buffer.from(dbjson, 'utf-8'))
}
}
await sleep(3000)
}
}
async function backupDrive(ACCESS_TOKEN:string) {
alertStore.set({
@@ -176,11 +249,13 @@ type DriveFile = {
id: string
}
async function loadDrive(ACCESS_TOKEN:string) {
alertStore.set({
type: "wait",
msg: "Loading Backup..."
})
async function loadDrive(ACCESS_TOKEN:string, mode: 'backup'|'sync') {
if(mode === 'backup'){
alertStore.set({
type: "wait",
msg: "Loading Backup..."
})
}
const files:DriveFile[] = await getFilesInFolder(ACCESS_TOKEN)
let foragekeys:string[] = []
let loadedForageKeys = false
@@ -204,42 +279,80 @@ async function loadDrive(ACCESS_TOKEN:string) {
let dbs:[DriveFile,number][] = []
for(const f of files){
if(f.name.endsWith("-database.risudat")){
const tm = parseInt(f.name.split('-')[0])
if(isNaN(tm)){
continue
}
else{
dbs.push([f,tm])
if(mode === 'backup'){
for(const f of files){
if(f.name.endsWith("-database.risudat")){
const tm = parseInt(f.name.split('-')[0])
if(isNaN(tm)){
continue
}
else{
dbs.push([f,tm])
}
}
}
dbs.sort((a,b) => {
return b[1] - a[1]
})
}
else if(mode === 'sync'){
for(const f of files){
if(f.name.endsWith("-database.risudat2")){
const tm = parseInt(f.name.split('-')[0])
if(isNaN(tm)){
continue
}
else if(tm > lastSaved){
dbs.push([f,tm])
}
}
}
dbs.sort((a,b) => {
return b[1] - a[1]
})
}
dbs.sort((a,b) => {
return b[1] - a[1]
})
if(dbs.length !== 0){
let selectables:string[] = []
for(let i=0;i<dbs.length;i++){
selectables.push(`Backup saved in ${(new Date(dbs[i][1] * 1000)).toLocaleString()}`)
if(selectables.length > 7){
break
}
if(mode === 'sync'){
alertStore.set({
type: "wait",
msg: "Sync Data..."
})
}
const selectedIndex = (await alertSelect([language.loadLatest, language.loadOthers]) === '0') ? 0 : parseInt(await alertSelect(selectables))
const selectedDb = dbs[selectedIndex][0]
const db:Database = JSON.parse(Buffer.from(pako.inflate(await getFileData(ACCESS_TOKEN, selectedDb.id))).toString('utf-8'))
async function getDbFromList(){
let selectables:string[] = []
for(let i=0;i<dbs.length;i++){
selectables.push(`Backup saved in ${(new Date(dbs[i][1] * 1000)).toLocaleString()}`)
if(selectables.length > 7){
break
}
}
const selectedIndex = (await alertSelect([language.loadLatest, language.loadOthers]) === '0') ? 0 : parseInt(await alertSelect(selectables))
const selectedDb = dbs[selectedIndex][0]
const decompressedDb:Database = JSON.parse(Buffer.from(pako.inflate(await getFileData(ACCESS_TOKEN, selectedDb.id))).toString('utf-8'))
return decompressedDb
}
const db:Database = mode === 'backup' ? await getDbFromList() : JSON.parse(Buffer.from(await getFileData(ACCESS_TOKEN, dbs[0][0].id)).toString('utf-8'))
lastSaved = Date.now()
localStorage.setItem('risu_lastsaved', `${lastSaved}`)
const requiredImages = (getUnpargeables(db))
let ind = 0;
for(const images of requiredImages){
ind += 1
const formatedImage = formatKeys(images)
alertStore.set({
type: "wait",
msg: `Loading Backup... (${ind} / ${requiredImages.length})`
})
if(mode === 'sync'){
alertStore.set({
type: "wait",
msg: `Sync Files... (${ind} / ${requiredImages.length})`
})
}
else{
alertStore.set({
type: "wait",
msg: `Loading Backup... (${ind} / ${requiredImages.length})`
})
}
if(await checkImageExists(images)){
//skip process
}
@@ -276,7 +389,7 @@ async function loadDrive(ACCESS_TOKEN:string) {
relaunch()
alertStore.set({
type: "wait",
msg: "Success, Refresh your app."
msg: "Success, Refreshing your app."
})
}
else{
@@ -284,11 +397,11 @@ async function loadDrive(ACCESS_TOKEN:string) {
location.search = ''
alertStore.set({
type: "wait",
msg: "Success, Refresh your app."
msg: "Success, Refreshing your app."
})
}
}
else{
else if(mode === 'backup'){
location.search = ''
}
}

9
src/ts/storage/database.ts
View File

@@ -509,6 +509,15 @@ export interface Database{
novellistAPI:string,
useAutoTranslateInput:boolean
imageCompression:boolean
account?:{
token:string
id:string,
data: {
refresh_token?:string,
access_token?:string
expires_in?: number
}
}
}
interface hordeConfig{

9
src/ts/storage/globalApi.ts
View File

@@ -14,12 +14,13 @@ import { selectedCharID } from "../stores";
import { Body, ResponseType, fetch as TauriFetch } from "@tauri-apps/api/http";
import { loadPlugins } from "../process/plugins";
import { alertError, alertStore } from "../alert";
import { checkDriverInit } from "../drive/drive";
import { checkDriverInit, syncDrive } from "../drive/drive";
import { hasher } from "../parser";
import { characterHubImport } from "../characterCards";
import { cloneDeep } from "lodash";
import { NodeStorage } from "./nodeStorage";
import { defaultJailbreak, defaultMainPrompt, oldJailbreak, oldMainPrompt } from "./defaultPrompts";
import { loadRisuAccountData } from "../drive/accounter";
//@ts-ignore
export const isTauri = !!window.__TAURI__
@@ -195,6 +196,7 @@ let lastSave = ''
export async function saveDb(){
lastSave =JSON.stringify(get(DataBase))
syncDrive()
while(true){
const dbjson = JSON.stringify(get(DataBase))
if(dbjson !== lastSave){
@@ -358,6 +360,11 @@ export async function loadData() {
} catch (error) {}
await checkNewFormat()
updateTextTheme()
if(get(DataBase).account){
try {
await loadRisuAccountData()
} catch (error) {}
}
loadedStore.set(true)
selectedCharID.set(-1)
saveDb()

1
src/ts/tokenizer.ts
View File

@@ -49,7 +49,6 @@ export class ChatTokenizer {
this.useName = useName
}
async tokenizeChat(data:OpenAIChat) {
console.log(data.content)
let encoded = (await encode(data.content)).length + this.chatAdditonalTokens
if(data.name && this.useName ==='name'){
encoded += (await encode(data.name)).length