mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework importCharacterProcess and selectFileByDom functions

Browse Source
This commit is contained in:
kwaroran
2024-03-26 04:01:07 +09:00
parent 92c2cd1a1a
commit 5b4f24019b
4 changed files with 86 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/ts/characterCards.ts
View File

@@ -1,11 +1,11 @@
import { get, writable, type Writable } from "svelte/store"
import { alertCardExport, alertConfirm, alertError, alertInput, alertMd, alertNormal, alertSelect, alertStore, alertTOS, alertWait } from "./alert"
import { DataBase, defaultSdDataFunc, type character, setDatabase, type customscript, type loreSettings, type loreBook, type triggerscript } from "./storage/database"
import { checkNullish, decryptBuffer, encryptBuffer, selectMultipleFile, sleep } from "./util"
import { checkNullish, decryptBuffer, encryptBuffer, selectFileByDom, selectMultipleFile, sleep } from "./util"
import { language } from "src/lang"
import { v4 as uuidv4 } from 'uuid';
import { characterFormatUpdate } from "./characters"
import { checkCharOrder, downloadFile, loadAsset, LocalWriter, readImage, saveAsset } from "./storage/globalApi"
import { AppendableBuffer, checkCharOrder, downloadFile, loadAsset, LocalWriter, readImage, saveAsset } from "./storage/globalApi"
import { cloneDeep } from "lodash"
import { selectedCharID } from "./stores"
import { convertImage, hasher } from "./parser"
@@ -18,13 +18,17 @@ export const hubURL = "https://sv.risuai.xyz"
export async function importCharacter() {
try {
const files = await selectMultipleFile(['png', 'json'])
const files = await selectFileByDom(['png', 'json'])
if(!files){
return
}
for(const f of files){
await importCharacterProcess(f)
console.log(f)
await importCharacterProcess({
name: f.name,
data: f
})
checkCharOrder()
}
} catch (error) {
@@ -35,10 +39,11 @@ export async function importCharacter() {
async function importCharacterProcess(f:{
name: string;
data: Uint8Array;
data: Uint8Array|File
}) {
if(f.name.endsWith('json')){
const da = JSON.parse(Buffer.from(f.data).toString('utf-8'))
const data = f.data instanceof Uint8Array ? f.data : new Uint8Array(await f.data.arrayBuffer())
const da = JSON.parse(Buffer.from(data).toString('utf-8'))
if(await importSpecv2(da)){
let db = get(DataBase)
return db.characters.length - 1
@@ -60,16 +65,23 @@ async function importCharacterProcess(f:{
msg: 'Loading... (Reading)'
})
await sleep(10)
const img = f.data
// const readed = PngChunk.read(img, ['chara'])?.['chara']
let readedChara = ''
const readGenerator = PngChunk.readGenerator(img)
let img:Uint8Array
const readGenerator = PngChunk.readGenerator(f.data, {
returnTrimed: true
})
const assets:{[key:string]:string} = {}
for await(const chunk of readGenerator){
for await (const chunk of readGenerator){
console.log(chunk)
if(!chunk){
break
}
if(chunk instanceof AppendableBuffer){
img = chunk.buffer
break
}
if(chunk.key === 'chara'){
//For memory reason, limit to 2MB
if(readedChara.length < 2 * 1024 * 1024){

55
src/ts/pngChunk.ts
View File

@@ -1,6 +1,7 @@
import { Buffer } from 'buffer';
import crc32 from 'crc/crc32';
import type { LocalWriter } from './storage/globalApi';
import { AppendableBuffer, type LocalWriter } from './storage/globalApi';
import { blobToUint8Array } from './util';
class StreamChunkWriter{
constructor(private data:Uint8Array, private writer:LocalWriter){
@@ -113,24 +114,51 @@ export const PngChunk = {
return chunks
},
readGenerator: function*(data:Uint8Array, arg:{checkCrc?:boolean} = {}):Generator<{key:string,value:string},null>{
readGenerator: async function*(data:File|Uint8Array, arg:{checkCrc?:boolean,returnTrimed?:boolean} = {}):AsyncGenerator<
{key:string,value:string}|AppendableBuffer,null
>{
const trimedData = new AppendableBuffer()
async function appendTrimed(data:Uint8Array){
if(arg.returnTrimed){
trimedData.append(data)
}
}
async function slice(start:number,end?:number):Promise<Uint8Array> {
if(data instanceof File){
return await blobToUint8Array (data.slice(start,end))
}
else{
return data.slice(start,end)
}
}
await appendTrimed(await slice(0,8))
let pos = 8
while(pos < data.length){
const len = data[pos] * 0x1000000 + data[pos+1] * 0x10000 + data[pos+2] * 0x100 + data[pos+3]
const type = data.slice(pos+4,pos+8)
const size = data instanceof File ? data.size : data.length
while(pos < size){
const dataPart = await slice(pos,pos+4)
const len = dataPart[0] * 0x1000000 + dataPart[1] * 0x10000 + dataPart[2] * 0x100 + dataPart[3]
const type = await slice(pos+4,pos+8)
const typeString = new TextDecoder().decode(type)
if(arg.checkCrc){
const crc = data[pos+8+len] * 0x1000000 + data[pos+9+len] * 0x10000 + data[pos+10+len] * 0x100 + data[pos+11+len]
const crcCheck = crc32(data.slice(pos+4,pos+8+len))
const dataPart = await slice(pos+8+len,pos+12+len)
const crc = dataPart[0] * 0x1000000 + dataPart[1] * 0x10000 + dataPart[2] * 0x100 + dataPart[3]
const crcCheck = crc32(await slice(pos+4,pos+8+len))
if(crc !== crcCheck){
throw new Error('crc check failed')
}
}
if(typeString === 'IEND'){
await appendTrimed(await slice(pos,pos+12+len))
break
}
if(typeString === 'tEXt'){
const chunkData = data.slice(pos+8,pos+8+len)
else if(typeString === 'tEXt'){
const chunkData = await slice(pos+8,pos+8+len)
let key=''
let value=''
for(let i=0;i<70;i++){
@@ -142,11 +170,17 @@ export const PngChunk = {
}
yield {key,value}
}
else{
await appendTrimed(await slice(pos,pos+12+len))
}
pos += 12 + len
}
if(arg.returnTrimed){
yield trimedData
}
return null
},
trim: (data:Uint8Array) => {
let pos = 8
let newData:Uint8Array[] = []
@@ -170,6 +204,7 @@ export const PngChunk = {
return Buffer.concat(newData)
},
write: async (data:Uint8Array, chunks:{[key:string]:string}, options:{writer?:LocalWriter} = {}):Promise<void | Buffer> => {
let pos = 8
let newData:Uint8Array[] = []

2
src/ts/storage/globalApi.ts
View File

@@ -1234,7 +1234,7 @@ if(Capacitor.isNativePlatform()){
streamedFetchListening = true
}
class AppendableBuffer{
export class AppendableBuffer{
buffer:Uint8Array
constructor(){
this.buffer = new Uint8Array(0)

20
src/ts/util.ts
View File

@@ -108,7 +108,7 @@ export const replacePlaceholders = (msg:string, name:string) => {
export function checkIsIos(){
return /(iPad|iPhone|iPod)/g.test(navigator.userAgent)
}
function selectFileByDom(allowedExtensions:string[], multiple:'multiple'|'single' = 'single') {
export function selectFileByDom(allowedExtensions:string[], multiple:'multiple'|'single' = 'single') {
return new Promise<null|File[]>((resolve) => {
const fileInput = document.createElement('input');
fileInput.type = 'file';
@@ -424,4 +424,22 @@ export const capitalize = (s:string) => {
return s.charAt(0).toUpperCase() + s.slice(1)
}
export function blobToUint8Array(data:Blob){
return new Promise<Uint8Array>((resolve,reject) => {
const reader = new FileReader()
reader.onload = () => {
if(reader.result instanceof ArrayBuffer){
resolve(new Uint8Array(reader.result))
}
else{
reject(new Error('reader.result is not ArrayBuffer'))
}
}
reader.onerror = () => {
reject(reader.error)
}
reader.readAsArrayBuffer(data)
})
}
export const languageCodes = ["af","ak","am","an","ar","as","ay","az","be","bg","bh","bm","bn","br","bs","ca","co","cs","cy","da","de","dv","ee","el","en","eo","es","et","eu","fa","fi","fo","fr","fy","ga","gd","gl","gn","gu","ha","he","hi","hr","ht","hu","hy","ia","id","ig","is","it","iu","ja","jv","ka","kk","km","kn","ko","ku","ky","la","lb","lg","ln","lo","lt","lv","mg","mi","mk","ml","mn","mr","ms","mt","my","nb","ne","nl","nn","no","ny","oc","om","or","pa","pl","ps","pt","qu","rm","ro","ru","rw","sa","sd","si","sk","sl","sm","sn","so","sq","sr","st","su","sv","sw","ta","te","tg","th","ti","tk","tl","tn","to","tr","ts","tt","tw","ug","uk","ur","uz","vi","wa","wo","xh","yi","yo","zh","zu"]