mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add LoadingStatusState and improve tokenizer functionality

Browse Source
This commit is contained in:
Kwaroran
2024-12-26 06:00:07 +09:00
parent 575eb0505e
commit fe47f58c61
7 changed files with 836867 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

836758
public/token/gemma/tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
src/App.svelte
View File

@@ -1,5 +1,5 @@
<script lang="ts">
import { DynamicGUI, settingsOpen, sideBarStore, ShowRealmFrameStore, openPresetList, openPersonaList, MobileGUI, CustomGUISettingMenuStore, loadedStore, alertStore } from './ts/stores.svelte';
import { DynamicGUI, settingsOpen, sideBarStore, ShowRealmFrameStore, openPresetList, openPersonaList, MobileGUI, CustomGUISettingMenuStore, loadedStore, alertStore, LoadingStatusState } from './ts/stores.svelte';
import Sidebar from './lib/SideBars/Sidebar.svelte';
import { DBState } from './ts/stores.svelte';
import ChatScreen from './lib/ChatScreens/ChatScreen.svelte';
@@ -46,6 +46,7 @@
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"></path>
</svg>
<span>Loading...</span>
<span>{LoadingStatusState.text}</span>
</div>
{:else if $CustomGUISettingMenuStore}
<CustomGUISettingMenu />

83
src/ts/globalApi.svelte.ts
View File

@@ -16,7 +16,7 @@ import {open} from '@tauri-apps/plugin-shell'
import { setDatabase, type Database, defaultSdDataFunc, getDatabase, type character } from "./storage/database.svelte";
import { getCurrentWebviewWindow } from "@tauri-apps/api/webviewWindow";
import { checkRisuUpdate } from "./update";
import { MobileGUI, botMakerMode, selectedCharID, loadedStore, DBState } from "./stores.svelte";
import { MobileGUI, botMakerMode, selectedCharID, loadedStore, DBState, LoadingStatusState } from "./stores.svelte";
import { loadPlugins } from "./plugins/plugins";
import { alertConfirm, alertError, alertNormal, alertNormalWait, alertSelect, alertTOS, alertWait } from "./alert";
import { checkDriverInit, syncDrive } from "./drive/drive";
@@ -42,6 +42,7 @@ import { updateLorebooks } from "./characters";
import { initMobileGesture } from "./hotkey";
import { fetch as TauriHTTPFetch } from '@tauri-apps/plugin-http';
import { moduleUpdate } from "./process/modules";
import type { AccountStorage } from "./storage/accountStorage";
//@ts-ignore
export const isTauri = !!window.__TAURI_INTERNALS__
@@ -498,6 +499,7 @@ export async function loadData() {
if(!loaded){
try {
if(isTauri){
LoadingStatusState.text = "Checking Files..."
appWindow.maximize()
if(!await exists('', {baseDir: BaseDirectory.AppData})){
await mkdir('', {baseDir: BaseDirectory.AppData})
@@ -512,13 +514,18 @@ export async function loadData() {
await writeFile('database/database.bin', encodeRisuSaveLegacy({}), {baseDir: BaseDirectory.AppData});
}
try {
const decoded = await decodeRisuSave(await readFile('database/database.bin',{baseDir: BaseDirectory.AppData}))
LoadingStatusState.text = "Reading Save File..."
const readed = await readFile('database/database.bin',{baseDir: BaseDirectory.AppData})
LoadingStatusState.text = "Decoding Save File..."
const decoded = await decodeRisuSave(readed)
setDatabase(decoded)
} catch (error) {
LoadingStatusState.text = "Reading Backup Files..."
const backups = await getDbBackups()
let backupLoaded = false
for(const backup of backups){
try {
LoadingStatusState.text = `Reading Backup File ${backup}...`
const backupData = await readFile(`database/dbbackup-${backup}.bin`,{baseDir: BaseDirectory.AppData})
setDatabase(
await decodeRisuSave(backupData)
@@ -532,12 +539,49 @@ export async function loadData() {
throw "Your save file is corrupted"
}
}
LoadingStatusState.text = "Checking Update..."
await checkRisuUpdate()
await changeFullscreen()
}
else{
await forageStorage.Init()
if(await forageStorage.checkAccountSync()){
LoadingStatusState.text = "Checking Account Sync..."
let gotStorage:Uint8Array = await (forageStorage.realStorage as AccountStorage).getItem('database/database.bin', (v) => {
LoadingStatusState.text = `Loading Save File ${(v*100).toFixed(2)}%`
})
if(checkNullish(gotStorage)){
gotStorage = encodeRisuSaveLegacy({})
await forageStorage.setItem('database/database.bin', gotStorage)
}
try {
setDatabase(
await decodeRisuSave(gotStorage)
)
} catch (error) {
const backups = await getDbBackups()
let backupLoaded = false
for(const backup of backups){
try {
LoadingStatusState.text = `Reading Backup File ${backup}...`
const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`) as unknown as Uint8Array
setDatabase(
await decodeRisuSave(backupData)
)
backupLoaded = true
} catch (error) {}
}
if(!backupLoaded){
throw "Your save file is corrupted"
}
}
}
else{
LoadingStatusState.text = "Loading Save File..."
let gotStorage:Uint8Array = await forageStorage.getItem('database/database.bin') as unknown as Uint8Array
LoadingStatusState.text = "Decoding Save File..."
if(checkNullish(gotStorage)){
gotStorage = encodeRisuSaveLegacy({})
await forageStorage.setItem('database/database.bin', gotStorage)
@@ -552,32 +596,7 @@ export async function loadData() {
let backupLoaded = false
for(const backup of backups){
try {
const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`) as unknown as Uint8Array
setDatabase(
await decodeRisuSave(backupData)
)
backupLoaded = true
} catch (error) {}
}
if(!backupLoaded){
throw "Your save file is corrupted"
}
}
if(await forageStorage.checkAccountSync()){
let gotStorage:Uint8Array = await forageStorage.getItem('database/database.bin') as unknown as Uint8Array
if(checkNullish(gotStorage)){
gotStorage = encodeRisuSaveLegacy({})
await forageStorage.setItem('database/database.bin', gotStorage)
}
try {
setDatabase(
await decodeRisuSave(gotStorage)
)
} catch (error) {
const backups = await getDbBackups()
let backupLoaded = false
for(const backup of backups){
try {
LoadingStatusState.text = `Reading Backup File ${backup}...`
const backupData:Uint8Array = await forageStorage.getItem(`database/dbbackup-${backup}.bin`) as unknown as Uint8Array
setDatabase(
await decodeRisuSave(backupData)
@@ -590,10 +609,12 @@ export async function loadData() {
}
}
}
LoadingStatusState.text = "Checking Drive Sync..."
const isDriverMode = await checkDriverInit()
if(isDriverMode){
return
}
LoadingStatusState.text = "Checking Service Worker..."
if(navigator.serviceWorker && (!Capacitor.isNativePlatform())){
usingSw = true
await registerSw()
@@ -605,13 +626,16 @@ export async function loadData() {
characterURLImport()
}
}
LoadingStatusState.text = "Checking Unnecessary Files..."
try {
await pargeChunks()
} catch (error) {}
LoadingStatusState.text = "Loading Plugins..."
try {
await loadPlugins()
} catch (error) {}
if(getDatabase().account){
LoadingStatusState.text = "Checking Account Data..."
try {
await loadRisuAccountData()
} catch (error) {}
@@ -625,8 +649,11 @@ export async function loadData() {
} catch (error) {
}
LoadingStatusState.text = "Checking For Format Update..."
await checkNewFormat()
const db = getDatabase();
LoadingStatusState.text = "Updating States..."
updateColorScheme()
updateTextThemeAndCSS()
updateAnimationSpeed()

35
src/ts/storage/accountStorage.ts
View File

@@ -3,7 +3,7 @@ import { getDatabase } from "./database.svelte"
import { hubURL } from "../characterCards"
import localforage from "localforage"
import { alertLogin, alertNormalWait, alertStore, alertWait } from "../alert"
import { forageStorage, getUnpargeables } from "../globalApi.svelte"
import { AppendableBuffer, forageStorage, getUnpargeables } from "../globalApi.svelte"
import { encodeRisuSaveLegacy } from "./risuSave"
import { v4 } from "uuid"
import { language } from "src/lang"
@@ -87,7 +87,7 @@ export class AccountStorage{
}
return await getDaText()
}
async getItem(key:string):Promise<Buffer> {
async getItem(key:string, callback?:(status:number) => void):Promise<Buffer> {
this.checkAuth()
if(key.startsWith('assets/')){
const k:ArrayBuffer = await localforage.getItem(key)
@@ -131,12 +131,39 @@ export class AccountStorage{
if(da.status === 204){
return null
}
const ab = await da.arrayBuffer()
if(key.startsWith('assets/')){
const ab = await da.arrayBuffer()
await localforage.setItem(key, ab)
}
return Buffer.from(ab)
}
if(!callback){
const ab = await da.arrayBuffer()
return Buffer.from(ab)
}
const size = parseInt(da.headers.get('x-body-size'))
const appendable = new Uint8Array(size)
const reader = da.body.getReader()
//log all headers
console.log('logging headers')
for(const [key, value] of da.headers.entries()){
console.log(key, value)
}
let i = 0
while(true){
const {done, value} = await reader.read()
if(done){
break
}
console.log(value, size)
appendable.set(value, i)
i += value.length
callback(i/size)
}
return Buffer.from(appendable)
}
async keys():Promise<string[]>{
let db = getDatabase()
return getUnpargeables(db, 'pure')

2
src/ts/storage/autoStorage.ts
View File

@@ -117,7 +117,7 @@ export class AutoStorage{
return false
}
private async Init(){
async Init(){
if(!this.realStorage){
if(localStorage.getItem('accountst') === 'able'){
this.realStorage = new AccountStorage()

4
src/ts/stores.svelte.ts
View File

@@ -94,6 +94,10 @@ export const DBState = $state({
db: {} as any as Database
});
export const LoadingStatusState = $state({
text: '',
})
export const disableHighlight = writable(true)
ReloadGUIPointer.subscribe(() => {

18
src/ts/tokenizer.ts
View File

@@ -8,6 +8,7 @@ import { tokenizeGGUFModel } from "./process/models/local";
import { globalFetch } from "./globalApi.svelte";
import { getModelInfo, LLMTokenizer } from "./model/modellist";
import { pluginV2 } from "./plugins/plugins";
import type { GemmaTokenizer } from "@huggingface/transformers";
export const tokenizerList = [
@@ -39,7 +40,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
case 'llama3':
return await tokenizeWebTokenizers(data, 'llama')
case 'gemma':
return await tokenizeWebTokenizers(data, 'gemma')
return await gemmaTokenize(data)
case 'cohere':
return await tokenizeWebTokenizers(data, 'cohere')
default:
@@ -65,7 +66,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
case 'llama3':
return await tokenizeWebTokenizers(data, 'llama')
case 'gemma':
return await tokenizeWebTokenizers(data, 'gemma')
return await gemmaTokenize(data)
case 'cohere':
return await tokenizeWebTokenizers(data, 'cohere')
case 'o200k_base':
@@ -105,7 +106,7 @@ export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Arr
return await tokenizeGoogleCloud(data)
}
if(modelInfo.tokenizer === LLMTokenizer.Gemma || modelInfo.tokenizer === LLMTokenizer.GoogleCloud){
return await tokenizeWebTokenizers(data, 'gemma')
return await gemmaTokenize(data)
}
if(modelInfo.tokenizer === LLMTokenizer.Cohere){
return await tokenizeWebTokenizers(data, 'cohere')
@@ -157,6 +158,17 @@ async function tokenizeGoogleCloud(text:string) {
return new Uint32Array(count)
}
let gemmaTokenizer:GemmaTokenizer = null
async function gemmaTokenize(text:string) {
if(!gemmaTokenizer){
const {GemmaTokenizer} = await import('@huggingface/transformers')
gemmaTokenizer = new GemmaTokenizer(
await (await fetch("/token/llama/llama3.json")
).json(), {})
}
return gemmaTokenizer.encode(text)
}
async function tikJS(text:string, model='cl100k_base') {
if(!tikParser || lastTikModel !== model){
if(model === 'cl100k_base'){