mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] database

Browse Source
This commit is contained in:
kwaroran
2023-06-28 06:24:01 +09:00
parent 3081aeacf9
commit 1026945996
7 changed files with 97 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@@ -17,7 +17,7 @@
"@adobe/css-tools": "4.3.0-beta.2", "@adobe/css-tools": "4.3.0-beta.2",
"@dqbd/tiktoken": "^1.0.4", "@dqbd/tiktoken": "^1.0.4",
"@mlc-ai/web-tokenizers": "^0.1.0", "@mlc-ai/web-tokenizers": "^0.1.0",
"@tauri-apps/api": "1.3.0", "@tauri-apps/api": "1.4.0",
"blueimp-md5": "^2.19.0", "blueimp-md5": "^2.19.0",
"body-parser": "^1.20.2", "body-parser": "^1.20.2",
"buffer": "^6.0.3", "buffer": "^6.0.3",
@@ -50,7 +50,7 @@
"devDependencies": { "devDependencies": {
"@sveltejs/vite-plugin-svelte": "^2.0.0", "@sveltejs/vite-plugin-svelte": "^2.0.0",
"@tailwindcss/typography": "^0.5.9", "@tailwindcss/typography": "^0.5.9",
"@tauri-apps/cli": "1.3.1", "@tauri-apps/cli": "1.4.0",
"@tsconfig/svelte": "^3.0.0", "@tsconfig/svelte": "^3.0.0",
"@types/blueimp-md5": "^2.18.0", "@types/blueimp-md5": "^2.18.0",
"@types/dompurify": "^3.0.1", "@types/dompurify": "^3.0.1",

80
pnpm-lock.yaml generated
View File

@@ -11,8 +11,8 @@ dependencies:
specifier: ^0.1.0 specifier: ^0.1.0
version: 0.1.0 version: 0.1.0
'@tauri-apps/api': '@tauri-apps/api':
specifier: 1.3.0 specifier: 1.4.0
version: 1.3.0 version: 1.4.0
blueimp-md5: blueimp-md5:
specifier: ^2.19.0 specifier: ^2.19.0
version: 2.19.0 version: 2.19.0
@@ -106,8 +106,8 @@ devDependencies:
specifier: ^0.5.9 specifier: ^0.5.9
version: 0.5.9(tailwindcss@3.3.1) version: 0.5.9(tailwindcss@3.3.1)
'@tauri-apps/cli': '@tauri-apps/cli':
specifier: 1.3.1 specifier: 1.4.0
version: 1.3.1 version: 1.4.0
'@tsconfig/svelte': '@tsconfig/svelte':
specifier: ^3.0.0 specifier: ^3.0.0
version: 3.0.0 version: 3.0.0
@@ -630,13 +630,13 @@ packages:
tailwindcss: 3.3.1(postcss@8.4.23) tailwindcss: 3.3.1(postcss@8.4.23)
dev: true dev: true
/@tauri-apps/api@1.3.0: /@tauri-apps/api@1.4.0:
resolution: {integrity: sha512-AH+3FonkKZNtfRtGrObY38PrzEj4d+1emCbwNGu0V2ENbXjlLHMZQlUh+Bhu/CRmjaIwZMGJ3yFvWaZZgTHoog==} resolution: {integrity: sha512-Jd6HPoTM1PZSFIzq7FB8VmMu3qSSyo/3lSwLpoapW+lQ41CL5Dow2KryLg+gyazA/58DRWI9vu/XpEeHK4uMdw==}
engines: {node: '>= 14.6.0', npm: '>= 6.6.0', yarn: '>= 1.19.1'} engines: {node: '>= 14.6.0', npm: '>= 6.6.0', yarn: '>= 1.19.1'}
dev: false dev: false
/@tauri-apps/cli-darwin-arm64@1.3.1: /@tauri-apps/cli-darwin-arm64@1.4.0:
resolution: {integrity: sha512-QlepYVPgOgspcwA/u4kGG4ZUijlXfdRtno00zEy+LxinN/IRXtk+6ErVtsmoLi1ZC9WbuMwzAcsRvqsD+RtNAg==} resolution: {integrity: sha512-nA/ml0SfUt6/CYLVbHmT500Y+ijqsuv5+s9EBnVXYSLVg9kbPUZJJHluEYK+xKuOj6xzyuT/+rZFMRapmJD3jQ==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [arm64] cpu: [arm64]
os: [darwin] os: [darwin]
@@ -644,8 +644,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-darwin-x64@1.3.1: /@tauri-apps/cli-darwin-x64@1.4.0:
resolution: {integrity: sha512-fKcAUPVFO3jfDKXCSDGY0MhZFF/wDtx3rgFnogWYu4knk38o9RaqRkvMvqJhLYPuWaEM5h6/z1dRrr9KKCbrVg==} resolution: {integrity: sha512-ov/F6Zr+dg9B0PtRu65stFo2G0ow2TUlneqYYrkj+vA3n+moWDHfVty0raDjMLQbQt3rv3uayFMXGPMgble9OA==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [darwin] os: [darwin]
@@ -653,8 +653,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-linux-arm-gnueabihf@1.3.1: /@tauri-apps/cli-linux-arm-gnueabihf@1.4.0:
resolution: {integrity: sha512-+4H0dv8ltJHYu/Ma1h9ixUPUWka9EjaYa8nJfiMsdCI4LJLNE6cPveE7RmhZ59v9GW1XB108/k083JUC/OtGvA==} resolution: {integrity: sha512-zwjbiMncycXDV7doovymyKD7sCg53ouAmfgpUqEBOTY3vgBi9TwijyPhJOqoG5vUVWhouNBC08akGmE4dja15g==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [arm] cpu: [arm]
os: [linux] os: [linux]
@@ -662,8 +662,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-linux-arm64-gnu@1.3.1: /@tauri-apps/cli-linux-arm64-gnu@1.4.0:
resolution: {integrity: sha512-Pj3odVO1JAxLjYmoXKxcrpj/tPxcA8UP8N06finhNtBtBaxAjrjjxKjO4968KB0BUH7AASIss9EL4Tr0FGnDuw==} resolution: {integrity: sha512-5MCBcziqXC72mMXnkZU68mutXIR6zavDxopArE2gQtK841IlE06bIgtLi0kUUhlFJk2nhPRgiDgdLbrPlyt7fw==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [arm64] cpu: [arm64]
os: [linux] os: [linux]
@@ -671,8 +671,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-linux-arm64-musl@1.3.1: /@tauri-apps/cli-linux-arm64-musl@1.4.0:
resolution: {integrity: sha512-tA0JdDLPFaj42UDIVcF2t8V0tSha40rppcmAR/MfQpTCxih6399iMjwihz9kZE1n4b5O4KTq9GliYo50a8zYlQ==} resolution: {integrity: sha512-7J3pRB6n6uNYgIfCeKt2Oz8J7oSaz2s8GGFRRH2HPxuTHrBNCinzVYm68UhVpJrL3bnGkU0ziVZLsW/iaOGfUg==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [arm64] cpu: [arm64]
os: [linux] os: [linux]
@@ -680,8 +680,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-linux-x64-gnu@1.3.1: /@tauri-apps/cli-linux-x64-gnu@1.4.0:
resolution: {integrity: sha512-FDU+Mnvk6NLkqQimcNojdKpMN4Y3W51+SQl+NqG9AFCWprCcSg62yRb84751ujZuf2MGT8HQOfmd0i77F4Q3tQ==} resolution: {integrity: sha512-Zh5gfAJxOv5AVWxcwuueaQ2vIAhlg0d6nZui6nMyfIJ8dbf3aZQ5ZzP38sYow5h/fbvgL+3GSQxZRBIa3c2E1w==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [linux] os: [linux]
@@ -689,8 +689,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-linux-x64-musl@1.3.1: /@tauri-apps/cli-linux-x64-musl@1.4.0:
resolution: {integrity: sha512-MpO3akXFmK8lZYEbyQRDfhdxz1JkTBhonVuz5rRqxwA7gnGWHa1aF1+/2zsy7ahjB2tQ9x8DDFDMdVE20o9HrA==} resolution: {integrity: sha512-OLAYoICU3FaYiTdBsI+lQTKnDHeMmFMXIApN0M+xGiOkoIOQcV9CConMPjgmJQ867+NHRNgUGlvBEAh9CiJodQ==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [linux] os: [linux]
@@ -698,8 +698,17 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-win32-ia32-msvc@1.3.1: /@tauri-apps/cli-win32-arm64-msvc@1.4.0:
resolution: {integrity: sha512-9Boeo3K5sOrSBAZBuYyGkpV2RfnGQz3ZhGJt4hE6P+HxRd62lS6+qDKAiw1GmkZ0l1drc2INWrNeT50gwOKwIQ==} resolution: {integrity: sha512-gZ05GENFbI6CB5MlOUsLlU0kZ9UtHn9riYtSXKT6MYs8HSPRffPHaHSL0WxsJweWh9nR5Hgh/TUU8uW3sYCzCg==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [win32]
requiresBuild: true
dev: true
optional: true
/@tauri-apps/cli-win32-ia32-msvc@1.4.0:
resolution: {integrity: sha512-JsetT/lTx/Zq98eo8T5CiRyF1nKeX04RO8JlJrI3ZOYsZpp/A5RJvMd/szQ17iOzwiHdge+tx7k2jHysR6oBlQ==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [ia32] cpu: [ia32]
os: [win32] os: [win32]
@@ -707,8 +716,8 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli-win32-x64-msvc@1.3.1: /@tauri-apps/cli-win32-x64-msvc@1.4.0:
resolution: {integrity: sha512-wMrTo91hUu5CdpbElrOmcZEoJR4aooTG+fbtcc87SMyPGQy1Ux62b+ZdwLvL1sVTxnIm//7v6QLRIWGiUjCPwA==} resolution: {integrity: sha512-z8Olcnwp5aYhzqUAarFjqF+oELCjuYWnB2HAJHlfsYNfDCAORY5kct3Fklz8PSsubC3U2EugWn8n42DwnThurg==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
cpu: [x64] cpu: [x64]
os: [win32] os: [win32]
@@ -716,20 +725,21 @@ packages:
dev: true dev: true
optional: true optional: true
/@tauri-apps/cli@1.3.1: /@tauri-apps/cli@1.4.0:
resolution: {integrity: sha512-o4I0JujdITsVRm3/0spfJX7FcKYrYV1DXJqzlWIn6IY25/RltjU6qbC1TPgVww3RsRX63jyVUTcWpj5wwFl+EQ==} resolution: {integrity: sha512-VXYr2i2iVFl98etQSQsqLzXgX96bnWiNZd1YADgatqwy/qecbd6Kl5ZAPB5R4ynsgE8A1gU7Fbzh7dCEQYFfmA==}
engines: {node: '>= 10'} engines: {node: '>= 10'}
hasBin: true hasBin: true
optionalDependencies: optionalDependencies:
'@tauri-apps/cli-darwin-arm64': 1.3.1 '@tauri-apps/cli-darwin-arm64': 1.4.0
'@tauri-apps/cli-darwin-x64': 1.3.1 '@tauri-apps/cli-darwin-x64': 1.4.0
'@tauri-apps/cli-linux-arm-gnueabihf': 1.3.1 '@tauri-apps/cli-linux-arm-gnueabihf': 1.4.0
'@tauri-apps/cli-linux-arm64-gnu': 1.3.1 '@tauri-apps/cli-linux-arm64-gnu': 1.4.0
'@tauri-apps/cli-linux-arm64-musl': 1.3.1 '@tauri-apps/cli-linux-arm64-musl': 1.4.0
'@tauri-apps/cli-linux-x64-gnu': 1.3.1 '@tauri-apps/cli-linux-x64-gnu': 1.4.0
'@tauri-apps/cli-linux-x64-musl': 1.3.1 '@tauri-apps/cli-linux-x64-musl': 1.4.0
'@tauri-apps/cli-win32-ia32-msvc': 1.3.1 '@tauri-apps/cli-win32-arm64-msvc': 1.4.0
'@tauri-apps/cli-win32-x64-msvc': 1.3.1 '@tauri-apps/cli-win32-ia32-msvc': 1.4.0
'@tauri-apps/cli-win32-x64-msvc': 1.4.0
dev: true dev: true
/@tootallnate/once@2.0.0: /@tootallnate/once@2.0.0:

27
src/ts/process/request.ts
View File

@@ -573,22 +573,33 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}; };
// const send_body = {
// text: stringlizeAINChat(formated, currentChar?.name ?? ''),
// length: maxTokens,
// temperature: temperature,
// top_p: 0.7,
// tailfree: 1.0,
// rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
// model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
// userbadwords: ["【質問】"].join("<<|>>"),
// };
const send_body = { const send_body = {
text: stringlizeAINChat(formated, currentChar?.name ?? ''), text: stringlizeAINChat(formated, currentChar?.name ?? ''),
length: maxTokens, length: maxTokens,
temperature: temperature, temperature: 0.925,
top_p: 0.7, top_p: 0.7,
tailfree: 1.0, rep_pen: 1.0625,
rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1, top_a: 0.08,
model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin', rep_pen_slope: 1.7,
userbadwords: ["【質問】"].join("<<|>>")
};
model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
stoptokens: ["「"].join("<<|>>"),
};
const response = await globalFetch(api_server_url + '/api', { const response = await globalFetch(api_server_url + '/api', {
method: 'POST', method: 'POST',
headers: headers, headers: headers,
body: send_body, body: send_body
plainFetchForce: true
}); });
if(!response.ok){ if(!response.ok){

31
src/ts/process/stringlize.ts
View File

@@ -26,7 +26,7 @@ export function stringlizeChat(formated:OpenAIChat[], char:string = ''){
export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){
let minIndex = -1 let minIndex = -1
const chunks = getUnstringlizerChunks(formated, char) const chunks = getUnstringlizerChunks(formated, char).chunks
for(const chunk of chunks){ for(const chunk of chunks){
@@ -48,7 +48,10 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:'ain'|'normal' = 'normal'){ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:'ain'|'normal' = 'normal'){
let chunks:string[] = ["system note:", "system:","system note：", "system："] let chunks:string[] = ["system note:", "system:","system note：", "system："]
let charNames:string[] = []
const db = get(DataBase)
if(char){ if(char){
charNames.push(char)
if(mode === 'ain'){ if(mode === 'ain'){
chunks.push(`${char} `) chunks.push(`${char} `)
chunks.push(`${char}　`) chunks.push(`${char}　`)
@@ -60,9 +63,23 @@ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:
chunks.push(`${char}： `) chunks.push(`${char}： `)
} }
} }
if(db.username){
charNames.push(db.username)
if(mode === 'ain'){
chunks.push(`${db.username} `)
chunks.push(`${db.username}　`)
}
else{
chunks.push(`${db.username}:`)
chunks.push(`${db.username}：`)
chunks.push(`${db.username}: `)
chunks.push(`${db.username}： `)
}
}
for(const form of formated){ for(const form of formated){
if(form.name){ if(form.name){
charNames.push(form.name)
if(mode === 'ain'){ if(mode === 'ain'){
if(!chunks.includes(`${form.name} `)){ if(!chunks.includes(`${form.name} `)){
chunks.push(`${form.name} `) chunks.push(`${form.name} `)
@@ -79,7 +96,7 @@ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:
} }
} }
} }
return chunks return {chunks,extChunk:charNames.concat(chunks)}
} }
export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){ export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){
@@ -159,10 +176,18 @@ function extractAINOutputStrings(inputString:string, characters:string[]) {
export function unstringlizeAIN(data:string,formated:OpenAIChat[], char:string = ''){ export function unstringlizeAIN(data:string,formated:OpenAIChat[], char:string = ''){
const db = get(DataBase) const db = get(DataBase)
const chunks = getUnstringlizerChunks(formated, char ,'ain') const chunksResult = getUnstringlizerChunks(formated, char ,'ain')
const chunks = chunksResult.chunks
let result:['char'|'user',string][] = [] let result:['char'|'user',string][] = []
data = `${char} 「` + data data = `${char} 「` + data
for(const n of chunksResult.extChunk){
if(data.endsWith(n)){
data = data.substring(0, data.length - n.length)
console.log('trimed')
}
}
const contents = extractAINOutputStrings(data, chunks) const contents = extractAINOutputStrings(data, chunks)
for(const cont of contents){ for(const cont of contents){
if(cont.character === '[narrator]'){ if(cont.character === '[narrator]'){

2
src/ts/storage/database.ts
View File

@@ -9,7 +9,7 @@ import { alertNormal } from '../alert';
export const DataBase = writable({} as any as Database) export const DataBase = writable({} as any as Database)
export const loadedStore = writable(false) export const loadedStore = writable(false)
export let appVer = '1.28.1' export let appVer = '1.28.2'
export let webAppSubVer = '' export let webAppSubVer = ''
export function setDatabase(data:Database){ export function setDatabase(data:Database){

2
src/ts/storage/globalApi.ts
View File

@@ -458,7 +458,7 @@ export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body
} }
} }
if(isTauri){ if(isTauri){
if(db.requester === 'new'){ if(db.requester === 'pure_reqwest'){
try { try {
let preHeader = arg.headers ?? {} let preHeader = arg.headers ?? {}
let body:any let body:any

2
version.json
View File

@@ -1 +1 @@
{"version":"1.28.1"} {"version":"1.28.2"}