mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] database

Browse Source
This commit is contained in:
kwaroran
2023-06-28 06:24:01 +09:00
parent 3081aeacf9
commit 1026945996
7 changed files with 97 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
package.json
View File

@@ -17,7 +17,7 @@
"@adobe/css-tools": "4.3.0-beta.2",
"@dqbd/tiktoken": "^1.0.4",
"@mlc-ai/web-tokenizers": "^0.1.0",
"@tauri-apps/api": "1.3.0",
"@tauri-apps/api": "1.4.0",
"blueimp-md5": "^2.19.0",
"body-parser": "^1.20.2",
"buffer": "^6.0.3",
@@ -50,7 +50,7 @@
"devDependencies": {
"@sveltejs/vite-plugin-svelte": "^2.0.0",
"@tailwindcss/typography": "^0.5.9",
"@tauri-apps/cli": "1.3.1",
"@tauri-apps/cli": "1.4.0",
"@tsconfig/svelte": "^3.0.0",
"@types/blueimp-md5": "^2.18.0",
"@types/dompurify": "^3.0.1",

80
pnpm-lock.yaml generated
View File

@@ -11,8 +11,8 @@ dependencies:
specifier: ^0.1.0
version: 0.1.0
'@tauri-apps/api':
specifier: 1.3.0
version: 1.3.0
specifier: 1.4.0
version: 1.4.0
blueimp-md5:
specifier: ^2.19.0
version: 2.19.0
@@ -106,8 +106,8 @@ devDependencies:
specifier: ^0.5.9
version: 0.5.9(tailwindcss@3.3.1)
'@tauri-apps/cli':
specifier: 1.3.1
version: 1.3.1
specifier: 1.4.0
version: 1.4.0
'@tsconfig/svelte':
specifier: ^3.0.0
version: 3.0.0
@@ -630,13 +630,13 @@ packages:
tailwindcss: 3.3.1(postcss@8.4.23)
dev: true
/@tauri-apps/api@1.3.0:
resolution: {integrity: sha512-AH+3FonkKZNtfRtGrObY38PrzEj4d+1emCbwNGu0V2ENbXjlLHMZQlUh+Bhu/CRmjaIwZMGJ3yFvWaZZgTHoog==}
/@tauri-apps/api@1.4.0:
resolution: {integrity: sha512-Jd6HPoTM1PZSFIzq7FB8VmMu3qSSyo/3lSwLpoapW+lQ41CL5Dow2KryLg+gyazA/58DRWI9vu/XpEeHK4uMdw==}
engines: {node: '>= 14.6.0', npm: '>= 6.6.0', yarn: '>= 1.19.1'}
dev: false
/@tauri-apps/cli-darwin-arm64@1.3.1:
resolution: {integrity: sha512-QlepYVPgOgspcwA/u4kGG4ZUijlXfdRtno00zEy+LxinN/IRXtk+6ErVtsmoLi1ZC9WbuMwzAcsRvqsD+RtNAg==}
/@tauri-apps/cli-darwin-arm64@1.4.0:
resolution: {integrity: sha512-nA/ml0SfUt6/CYLVbHmT500Y+ijqsuv5+s9EBnVXYSLVg9kbPUZJJHluEYK+xKuOj6xzyuT/+rZFMRapmJD3jQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [darwin]
@@ -644,8 +644,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-darwin-x64@1.3.1:
resolution: {integrity: sha512-fKcAUPVFO3jfDKXCSDGY0MhZFF/wDtx3rgFnogWYu4knk38o9RaqRkvMvqJhLYPuWaEM5h6/z1dRrr9KKCbrVg==}
/@tauri-apps/cli-darwin-x64@1.4.0:
resolution: {integrity: sha512-ov/F6Zr+dg9B0PtRu65stFo2G0ow2TUlneqYYrkj+vA3n+moWDHfVty0raDjMLQbQt3rv3uayFMXGPMgble9OA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
@@ -653,8 +653,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-linux-arm-gnueabihf@1.3.1:
resolution: {integrity: sha512-+4H0dv8ltJHYu/Ma1h9ixUPUWka9EjaYa8nJfiMsdCI4LJLNE6cPveE7RmhZ59v9GW1XB108/k083JUC/OtGvA==}
/@tauri-apps/cli-linux-arm-gnueabihf@1.4.0:
resolution: {integrity: sha512-zwjbiMncycXDV7doovymyKD7sCg53ouAmfgpUqEBOTY3vgBi9TwijyPhJOqoG5vUVWhouNBC08akGmE4dja15g==}
engines: {node: '>= 10'}
cpu: [arm]
os: [linux]
@@ -662,8 +662,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-linux-arm64-gnu@1.3.1:
resolution: {integrity: sha512-Pj3odVO1JAxLjYmoXKxcrpj/tPxcA8UP8N06finhNtBtBaxAjrjjxKjO4968KB0BUH7AASIss9EL4Tr0FGnDuw==}
/@tauri-apps/cli-linux-arm64-gnu@1.4.0:
resolution: {integrity: sha512-5MCBcziqXC72mMXnkZU68mutXIR6zavDxopArE2gQtK841IlE06bIgtLi0kUUhlFJk2nhPRgiDgdLbrPlyt7fw==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
@@ -671,8 +671,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-linux-arm64-musl@1.3.1:
resolution: {integrity: sha512-tA0JdDLPFaj42UDIVcF2t8V0tSha40rppcmAR/MfQpTCxih6399iMjwihz9kZE1n4b5O4KTq9GliYo50a8zYlQ==}
/@tauri-apps/cli-linux-arm64-musl@1.4.0:
resolution: {integrity: sha512-7J3pRB6n6uNYgIfCeKt2Oz8J7oSaz2s8GGFRRH2HPxuTHrBNCinzVYm68UhVpJrL3bnGkU0ziVZLsW/iaOGfUg==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
@@ -680,8 +680,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-linux-x64-gnu@1.3.1:
resolution: {integrity: sha512-FDU+Mnvk6NLkqQimcNojdKpMN4Y3W51+SQl+NqG9AFCWprCcSg62yRb84751ujZuf2MGT8HQOfmd0i77F4Q3tQ==}
/@tauri-apps/cli-linux-x64-gnu@1.4.0:
resolution: {integrity: sha512-Zh5gfAJxOv5AVWxcwuueaQ2vIAhlg0d6nZui6nMyfIJ8dbf3aZQ5ZzP38sYow5h/fbvgL+3GSQxZRBIa3c2E1w==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
@@ -689,8 +689,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-linux-x64-musl@1.3.1:
resolution: {integrity: sha512-MpO3akXFmK8lZYEbyQRDfhdxz1JkTBhonVuz5rRqxwA7gnGWHa1aF1+/2zsy7ahjB2tQ9x8DDFDMdVE20o9HrA==}
/@tauri-apps/cli-linux-x64-musl@1.4.0:
resolution: {integrity: sha512-OLAYoICU3FaYiTdBsI+lQTKnDHeMmFMXIApN0M+xGiOkoIOQcV9CConMPjgmJQ867+NHRNgUGlvBEAh9CiJodQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
@@ -698,8 +698,17 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-win32-ia32-msvc@1.3.1:
resolution: {integrity: sha512-9Boeo3K5sOrSBAZBuYyGkpV2RfnGQz3ZhGJt4hE6P+HxRd62lS6+qDKAiw1GmkZ0l1drc2INWrNeT50gwOKwIQ==}
/@tauri-apps/cli-win32-arm64-msvc@1.4.0:
resolution: {integrity: sha512-gZ05GENFbI6CB5MlOUsLlU0kZ9UtHn9riYtSXKT6MYs8HSPRffPHaHSL0WxsJweWh9nR5Hgh/TUU8uW3sYCzCg==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [win32]
requiresBuild: true
dev: true
optional: true
/@tauri-apps/cli-win32-ia32-msvc@1.4.0:
resolution: {integrity: sha512-JsetT/lTx/Zq98eo8T5CiRyF1nKeX04RO8JlJrI3ZOYsZpp/A5RJvMd/szQ17iOzwiHdge+tx7k2jHysR6oBlQ==}
engines: {node: '>= 10'}
cpu: [ia32]
os: [win32]
@@ -707,8 +716,8 @@ packages:
dev: true
optional: true
/@tauri-apps/cli-win32-x64-msvc@1.3.1:
resolution: {integrity: sha512-wMrTo91hUu5CdpbElrOmcZEoJR4aooTG+fbtcc87SMyPGQy1Ux62b+ZdwLvL1sVTxnIm//7v6QLRIWGiUjCPwA==}
/@tauri-apps/cli-win32-x64-msvc@1.4.0:
resolution: {integrity: sha512-z8Olcnwp5aYhzqUAarFjqF+oELCjuYWnB2HAJHlfsYNfDCAORY5kct3Fklz8PSsubC3U2EugWn8n42DwnThurg==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
@@ -716,20 +725,21 @@ packages:
dev: true
optional: true
/@tauri-apps/cli@1.3.1:
resolution: {integrity: sha512-o4I0JujdITsVRm3/0spfJX7FcKYrYV1DXJqzlWIn6IY25/RltjU6qbC1TPgVww3RsRX63jyVUTcWpj5wwFl+EQ==}
/@tauri-apps/cli@1.4.0:
resolution: {integrity: sha512-VXYr2i2iVFl98etQSQsqLzXgX96bnWiNZd1YADgatqwy/qecbd6Kl5ZAPB5R4ynsgE8A1gU7Fbzh7dCEQYFfmA==}
engines: {node: '>= 10'}
hasBin: true
optionalDependencies:
'@tauri-apps/cli-darwin-arm64': 1.3.1
'@tauri-apps/cli-darwin-x64': 1.3.1
'@tauri-apps/cli-linux-arm-gnueabihf': 1.3.1
'@tauri-apps/cli-linux-arm64-gnu': 1.3.1
'@tauri-apps/cli-linux-arm64-musl': 1.3.1
'@tauri-apps/cli-linux-x64-gnu': 1.3.1
'@tauri-apps/cli-linux-x64-musl': 1.3.1
'@tauri-apps/cli-win32-ia32-msvc': 1.3.1
'@tauri-apps/cli-win32-x64-msvc': 1.3.1
'@tauri-apps/cli-darwin-arm64': 1.4.0
'@tauri-apps/cli-darwin-x64': 1.4.0
'@tauri-apps/cli-linux-arm-gnueabihf': 1.4.0
'@tauri-apps/cli-linux-arm64-gnu': 1.4.0
'@tauri-apps/cli-linux-arm64-musl': 1.4.0
'@tauri-apps/cli-linux-x64-gnu': 1.4.0
'@tauri-apps/cli-linux-x64-musl': 1.4.0
'@tauri-apps/cli-win32-arm64-msvc': 1.4.0
'@tauri-apps/cli-win32-ia32-msvc': 1.4.0
'@tauri-apps/cli-win32-x64-msvc': 1.4.0
dev: true
/@tootallnate/once@2.0.0:

27
src/ts/process/request.ts
View File

@@ -573,22 +573,33 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
'Content-Type': 'application/json'
};
// const send_body = {
// text: stringlizeAINChat(formated, currentChar?.name ?? ''),
// length: maxTokens,
// temperature: temperature,
// top_p: 0.7,
// tailfree: 1.0,
// rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
// model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
// userbadwords: ["【質問】"].join("<<|>>"),
// };
const send_body = {
text: stringlizeAINChat(formated, currentChar?.name ?? ''),
length: maxTokens,
temperature: temperature,
temperature: 0.925,
top_p: 0.7,
tailfree: 1.0,
rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100) + 1,
model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
userbadwords: ["【質問】"].join("<<|>>")
};
rep_pen: 1.0625,
top_a: 0.08,
rep_pen_slope: 1.7,
model: aiModel === 'novellist_damsel' ? 'damsel' : 'supertrin',
stoptokens: ["「"].join("<<|>>"),
};
const response = await globalFetch(api_server_url + '/api', {
method: 'POST',
headers: headers,
body: send_body,
plainFetchForce: true
body: send_body
});
if(!response.ok){

31
src/ts/process/stringlize.ts
View File

@@ -26,7 +26,7 @@ export function stringlizeChat(formated:OpenAIChat[], char:string = ''){
export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string = ''){
let minIndex = -1
const chunks = getUnstringlizerChunks(formated, char)
const chunks = getUnstringlizerChunks(formated, char).chunks
for(const chunk of chunks){
@@ -48,7 +48,10 @@ export function unstringlizeChat(text:string, formated:OpenAIChat[], char:string
export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:'ain'|'normal' = 'normal'){
let chunks:string[] = ["system note:", "system:","system note：", "system："]
let charNames:string[] = []
const db = get(DataBase)
if(char){
charNames.push(char)
if(mode === 'ain'){
chunks.push(`${char} `)
chunks.push(`${char}　`)
@@ -60,9 +63,23 @@ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:
chunks.push(`${char}： `)
}
}
if(db.username){
charNames.push(db.username)
if(mode === 'ain'){
chunks.push(`${db.username} `)
chunks.push(`${db.username}　`)
}
else{
chunks.push(`${db.username}:`)
chunks.push(`${db.username}：`)
chunks.push(`${db.username}: `)
chunks.push(`${db.username}： `)
}
}
for(const form of formated){
if(form.name){
charNames.push(form.name)
if(mode === 'ain'){
if(!chunks.includes(`${form.name} `)){
chunks.push(`${form.name} `)
@@ -79,7 +96,7 @@ export function getUnstringlizerChunks(formated:OpenAIChat[], char:string, mode:
}
}
}
return chunks
return {chunks,extChunk:charNames.concat(chunks)}
}
export function stringlizeAINChat(formated:OpenAIChat[], char:string = ''){
@@ -159,10 +176,18 @@ function extractAINOutputStrings(inputString:string, characters:string[]) {
export function unstringlizeAIN(data:string,formated:OpenAIChat[], char:string = ''){
const db = get(DataBase)
const chunks = getUnstringlizerChunks(formated, char ,'ain')
const chunksResult = getUnstringlizerChunks(formated, char ,'ain')
const chunks = chunksResult.chunks
let result:['char'|'user',string][] = []
data = `${char} 「` + data
for(const n of chunksResult.extChunk){
if(data.endsWith(n)){
data = data.substring(0, data.length - n.length)
console.log('trimed')
}
}
const contents = extractAINOutputStrings(data, chunks)
for(const cont of contents){
if(cont.character === '[narrator]'){

2
src/ts/storage/database.ts
View File

@@ -9,7 +9,7 @@ import { alertNormal } from '../alert';
export const DataBase = writable({} as any as Database)
export const loadedStore = writable(false)
export let appVer = '1.28.1'
export let appVer = '1.28.2'
export let webAppSubVer = ''
export function setDatabase(data:Database){

2
src/ts/storage/globalApi.ts
View File

@@ -458,7 +458,7 @@ export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body
}
}
if(isTauri){
if(db.requester === 'new'){
if(db.requester === 'pure_reqwest'){
try {
let preHeader = arg.headers ?? {}
let body:any

2
version.json
View File

@@ -1 +1 @@
{"version":"1.28.1"}
{"version":"1.28.2"}