Merge branch 'kwaroran:main' into main

This commit is contained in:
HyperBlaze
2024-06-18 11:14:22 -07:00
committed by GitHub
22 changed files with 225 additions and 191 deletions

View File

@@ -8,7 +8,7 @@ android {
minSdkVersion rootProject.ext.minSdkVersion
targetSdkVersion rootProject.ext.targetSdkVersion
versionCode 2
versionName "114.4.0"
versionName "114.5.0"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
aaptOptions {
// Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps.

View File

@@ -12,7 +12,7 @@
"filters": [],
"attributes": [],
"versionCode": 2,
"versionName": "114.4.0",
"versionName": "114.5.0",
"outputFile": "app-release.apk"
}
],

View File

@@ -8,7 +8,7 @@
},
"package": {
"productName": "RisuAI",
"version": "114.4.0"
"version": "114.5.0"
},
"tauri": {
"allowlist": {

View File

@@ -131,6 +131,7 @@ export const languageEnglish = {
defaultVariables: "Here you can define your own default variables. use `<variable name>=<variable value>` format, seperated by newline. for example, `name=RisuAI`, which then can be used with trigger scripts and variables CBS like `{{getvar::A}}`, `{{setvar::A::B}}` or `{{? $A + 1}}`. if prompt template's default variable and character's default variable has same name, character's default variable will be used.",
lowLevelAccess: "If enabled, it will enable access to features that requires high computing powers and executing AI model via triggers in the character. do not enable this unless you really need these features.",
triggerLLMPrompt: "A prompt that would be sent to the model. you can use multi turns and roles by using `@@role user`, `@@role system`, `@@role assistant`. for example, \n\`\`\`\n@@role system\nrespond as hello\n@@role assistant\nhello\n@@role user\nhi\n\`\`\`",
legacyTranslation: "If enabled, it will use the old translation method, which preprocess markdown and quotes before translations instead of postprocessing after translations."
},
setup: {
chooseProvider: "Choose AI Provider",
@@ -655,4 +656,11 @@ export const languageEnglish = {
lineHeight: "Line Height",
loadAutoServerBackup: "Load Auto Server Backup",
notCharxWarn: "This character uses multiple assets. it is recommended to export this character as a CharX format for better compatibility.",
noPlugins: "No Plugins Installed",
legacyTranslation: "Legacy Translation",
clipboardSuccess: "Copied to Clipboard",
translateContent: 'Translate Content',
doNotTranslate: "Do Not Translate",
includePersonaName: "Include Persona Name",
hidePersonaName: "Hide Persona Name",
}

View File

@@ -1,6 +1,6 @@
<script lang="ts">
import { ArrowLeft, ArrowRight, PencilIcon, LanguagesIcon, RefreshCcwIcon, TrashIcon, CopyIcon, Volume2Icon, BotIcon, ArrowLeftRightIcon, UserIcon } from "lucide-svelte";
import { ParseMarkdown, type simpleCharacterArgument } from "../../ts/parser";
import { ParseMarkdown, postTranslationParse, type simpleCharacterArgument } from "../../ts/parser";
import AutoresizeArea from "../UI/GUI/TextAreaResizable.svelte";
import { alertConfirm, alertError, alertRequestData } from "../../ts/alert";
import { language } from "../../lang";
@@ -102,13 +102,25 @@
} catch (error) {}
}
if(translateText){
const marked = await ParseMarkdown(data, charArg, mode, chatID)
translating = true
const translated = await translateHTML(marked, false, charArg, chatID)
translating = false
lastParsed = translated
lastCharArg = charArg
return translated
if(!$DataBase.legacyTranslation){
const marked = await ParseMarkdown(data, charArg, 'pretranslate', chatID)
translating = true
console.log(marked)
const translated = postTranslationParse(await translateHTML(marked, false, charArg, chatID))
translating = false
lastParsed = translated
lastCharArg = charArg
return translated
}
else{
const marked = await ParseMarkdown(data, charArg, mode, chatID)
translating = true
const translated = await translateHTML(marked, false, charArg, chatID)
translating = false
lastParsed = translated
lastCharArg = charArg
return translated
}
}
else{
const marked = await ParseMarkdown(data, charArg, mode, chatID)

View File

@@ -338,11 +338,14 @@
for(const chat of chats){
const cnv = await html2canvas.toCanvas(chat as HTMLElement)
alertWait("Taking screenShot... "+canvases.length+"/"+chats.length)
canvases.push(cnv)
}
canvases.reverse()
alertWait("Merging images...")
let mergedCanvas = document.createElement('canvas');
mergedCanvas.width = 0;
mergedCanvas.height = 0;

View File

@@ -127,4 +127,10 @@
<Help key="combineTranslation"/>
</Check>
</div>
<div class="flex items-center mt-4">
<Check bind:check={$DataBase.legacyTranslation} name={language.legacyTranslation}>
<Help key="legacyTranslation"/>
</Check>
</div>
{/if}

View File

@@ -17,29 +17,12 @@
<span class="text-draculared text-xs mb-4">{language.pluginWarn}</span>
<div class="border-solid border-borderc p-2 flex flex-col border-1">
<div class="flex">
<span class="font-bold flex-grow">Metric Systemizer <Help key="metrica" /> <span class="text-green-500 italic">(Official Plugin)</span></span>
</div>
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.officialplugins.metrica} name={language.able}/>
</div>
<div class="flex">
<span class="font-bold flex-grow">OpenAI Fixer <Help key="openAIFixer" /> <span class="text-green-500 italic">(Official Plugin)</span></span>
</div>
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.officialplugins.oaiFix} name={language.able}/>
</div>
{#if $DataBase.officialplugins.oaiFix}
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.officialplugins.oaiFixEmdash} name={"Remove Emdash"}/>
</div>
<div class="flex items-center mt-2">
<Check bind:check={$DataBase.officialplugins.oaiFixLetters} name={"Fix Letters"}/>
</div>
<div class="border-solid border-darkborderc p-2 flex flex-col border-1">
{#if !$DataBase.plugins || $DataBase.plugins?.length === 0}
<span class="text-textcolor2">{language.noPlugins}</span>
{/if}
{#each $DataBase.plugins as plugin, i}
<div class="border-borderc mt-2 mb-2 w-full border-solid border-b-1 seperator"></div>
<div class="border-darkborderc mt-2 mb-2 w-full border-solid border-b-1 seperator"></div>
<div class="flex">
<span class="font-bold flex-grow">{plugin.displayName ?? plugin.name}</span>
<button class="textcolor2 hover:gray-200 cursor-pointer" on:click={async () => {
@@ -48,7 +31,7 @@
if($DataBase.currentPluginProvider === plugin.name){
$DataBase.currentPluginProvider = ''
}
let plugins = $DataBase.plugins
let plugins = $DataBase.plugins ?? []
plugins.splice(i, 1)
$DataBase.plugins = plugins
}

View File

@@ -95,7 +95,7 @@
{/if}
<button class="text-textcolor2 hover:text-green-500" on:click|stopPropagation={async () => {
await navigator.clipboard.writeText(`https://realm.risuai.net/character/${openedData.id}`)
alertNormal("Copied to clipboard")
alertNormal(language.clipboardSuccess)
}}>
<PaperclipIcon />
</button>

View File

@@ -222,16 +222,6 @@ async function importCharacterProcess(f:{
return db.characters.length - 1
}
}
if(checkedVersion === 'v1'){
const converted = CCardLib.character.convert(parsed, {
from: 'v1',
to: 'v2'
})
if(await importCharacterCardSpec(converted, img, "normal", assets)){
let db = get(DataBase)
return db.characters.length - 1
}
}
}
const charaData:OldTavernChar = JSON.parse(Buffer.from(readedChara, 'base64').toString('utf-8'))
console.log(charaData)

View File

@@ -1,6 +1,6 @@
import { get, writable } from "svelte/store";
import { DataBase, saveImage, setDatabase, type character, type Chat, defaultSdDataFunc, type loreBook } from "./storage/database";
import { alertConfirm, alertError, alertNormal, alertSelect, alertStore } from "./alert";
import { alertConfirm, alertError, alertNormal, alertSelect, alertStore, alertWait } from "./alert";
import { language } from "../lang";
import { decode as decodeMsgpack } from "msgpackr";
import { checkNullish, findCharacterbyId, selectMultipleFile, selectSingleFile, sleep } from "./util";
@@ -10,6 +10,8 @@ import { checkCharOrder, downloadFile, getFileSrc } from "./storage/globalApi";
import { reencodeImage } from "./process/files/image";
import { updateInlayScreen } from "./process/inlayScreen";
import { PngChunk } from "./pngChunk";
import { parseMarkdownSafe } from "./parser";
import { translateHTML } from "./translator/translator";
export function createNewCharacter() {
let db = get(DataBase)
@@ -153,12 +155,31 @@ export async function rmCharEmotion(charId:number, emotionId:number) {
export async function exportChat(page:number){
try {
const mode = await alertSelect(['Export as JSON', "Export as TXT"])
const mode = await alertSelect(['Export as JSON', "Export as TXT", "Export as HTML File", "Export as HTML Embed"])
const doTranslate = (mode === '2' || mode === '3') ? (await alertSelect([language.translateContent, language.doNotTranslate])) === '0' : false
const anonymous = (mode === '2' || mode === '3') ? ((await alertSelect([language.includePersonaName, language.hidePersonaName])) === '1') : false
const selectedID = get(selectedCharID)
const db = get(DataBase)
const chat = db.characters[selectedID].chats[page]
const char = db.characters[selectedID]
const date = new Date().toJSON();
const htmlChatParse = async (v:string) => {
v = parseMarkdownSafe(v)
if(doTranslate){
v = await translateHTML(v, false, '', -1)
}
if(anonymous){
//case insensitive match, replace all
const excapedName = char.name.replace(/[-\/\\^$*+\?\.()|[\]{}]/g, '\\$&')
v = v.replace(new RegExp(`${excapedName}`, 'gi'), '×××')
}
return v
}
if(mode === '0'){
const stringl = Buffer.from(JSON.stringify({
type: 'risuChat',
@@ -168,20 +189,132 @@ export async function exportChat(page:number){
await downloadFile(`${char.name}_${date}_chat`.replace(/[<>:"/\\|?*\.\,]/g, "") + '.json', stringl)
}
else if(mode === '2'){
let chatContentHTML = ''
let i = 0
for(const v of chat.message){
alertWait(`Translating... ${i++}/${chat.message.length}`)
const name = v.saying ? findCharacterbyId(v.saying).name : v.role === 'char' ? char.name : anonymous ? '×××' : db.username
chatContentHTML += `<div class="chat">
<h2>${name}</h2>
<div>${await htmlChatParse(v.data)}</div>
</div>`
}
const doc = `
<!DOCTYPE html>
<html>
<head>
<title>${char.name} Chat</title>
<style>
body{
font-family: Arial, sans-serif;
display: flex;
justify-content: center;
}
.container{
max-width: 800px;
padding: 1rem;
border-radius: 10px;
display: flex;
flex-direction: column;
gap: 1rem;
}
.chat{
background: #f0f0f0;
padding: 1rem;
border-radius: 10px;
display: flex;
flex-direction: column;
}
.idat{
display: none;
}
h2{
margin: 0;
}
.chat div{
margin-top: 0.5rem;
break-word: break-all;
}
</style>
</head>
<body>
<div class="container">
<div class="chat">
<h2>${char.name}</h2>
<div>${await htmlChatParse(
char.firstMsgIndex === -1 ? char.firstMessage : char.alternateGreetings?.[char.firstMsgIndex ?? 0]
)}</div>
</div>
${chatContentHTML}
</div>
<div class="idat">${
JSON.stringify(chat).replace(/</g, '&lt;').replace(/>/g, '&gt;')
}</div>
</body>
`
await downloadFile(`${char.name}_${date}_chat`.replace(/[<>:"/\\|?*\.\,]/g, "") + '.html', Buffer.from(doc, 'utf-8'))
}
else if(mode === '3'){
//create a html table
let chatContentHTML = ''
let i = 0
for(const v of chat.message){
alertWait(`Translating... ${i++}/${chat.message.length}`)
const name = v.saying ? findCharacterbyId(v.saying).name : v.role === 'char' ? char.name : anonymous ? '×××' : db.username
chatContentHTML += `<tr>
<td>${name}</td>
<td>${await htmlChatParse(v.data)}</td>
</tr>`
}
const template = `
<table>
<tr>
<th>Character</th>
<th>Message</th>
</tr>
<tr>
<td>${char.name}</td>
<td>${await htmlChatParse(char.firstMessage)}</td>
</tr>
${chatContentHTML}
</table>
<p>Chat from RisuAI</p>
`
//copy to clipboard
const item = new ClipboardItem({
'text/html': new Blob([template], { type: 'text/html' }),
'text/plain': new Blob([template], { type: 'text/plain' })
})
await navigator.clipboard.write([item])
alertNormal(language.clipboardSuccess)
return
}
else{
let stringl = chat.message.map((v) => {
if(v.saying){
return `${findCharacterbyId(v.saying).name}\n${v.data}`
return `--${findCharacterbyId(v.saying).name}\n${v.data}`
}
else{
return `${v.role === 'char' ? char.name : db.username}\n${v.data}`
return `--${v.role === 'char' ? char.name : db.username}\n${v.data}`
}
}).join('\n\n')
if(char.type !== 'group'){
stringl = `${char.name}\n${char.firstMessage}\n\n` + stringl
stringl = `--${char.name}\n${char.firstMessage}\n\n` + stringl
}
await downloadFile(`${char.name}_${date}_chat`.replace(/[<>:"/\\|?*\.\,]/g, "") + '.txt', Buffer.from(stringl, 'utf-8'))
@@ -194,7 +327,7 @@ export async function exportChat(page:number){
}
export async function importChat(){
const dat =await selectSingleFile(['json','jsonl'])
const dat =await selectSingleFile(['json','jsonl','txt','html'])
if(!dat){
return
}
@@ -236,7 +369,7 @@ export async function importChat(){
setDatabase(db)
alertNormal(language.successImport)
}
else{
else if(dat.name.endsWith('json')){
const json = JSON.parse(Buffer.from(dat.data).toString('utf-8'))
if(json.type === 'risuChat' && json.ver === 1){
const das:Chat = json.data
@@ -256,7 +389,19 @@ export async function importChat(){
return
}
}
else if(dat.name.endsWith('html')){
const doc = new DOMParser().parseFromString(Buffer.from(dat.data).toString('utf-8'), 'text/html')
const chat = doc.querySelector('.idat').textContent
const json = JSON.parse(chat)
if(json.message && json.note && json.name && json.localLore){
db.characters[selectedID].chats.unshift(json)
setDatabase(db)
alertNormal(language.successImport)
}
else{
alertError(language.errors.noData)
}
}
} catch (error) {
alertError(`${error}`)
}

View File

@@ -97,7 +97,7 @@ const normalCBSwithParams = [
'arraypop', 'array_pop', 'arraypush', 'array_push', 'arraysplice', 'array_splice',
'makearray', 'array', 'a', 'make_array', 'history', 'messages', 'range', 'date', 'time', 'datetimeformat', 'date_time_format',
'random', 'pick', 'roll', 'datetimeformat', 'hidden_key', 'reverse', 'getglobalvar', 'position', 'slot', 'rollp',
'and', 'or', 'not', 'message_time_array', 'filter'
'and', 'or', 'not', 'message_time_array', 'filter', 'greater', 'less', 'greater_equal', 'less_equal'
]
const displayRelatedCBS = [
@@ -113,7 +113,7 @@ const deprecatedCBS = [
]
const deprecatedCBSwithParams = [
'greater', 'less', 'greater_equal', 'less_equal', 'remaind', 'pow'
'remaind', 'pow'
]
const decorators = [

View File

@@ -169,25 +169,24 @@ export interface simpleCharacterArgument{
}
export async function ParseMarkdown(data:string, charArg:(character|simpleCharacterArgument | groupChat | string) = null, mode:'normal'|'back' = 'normal', chatID=-1) {
export async function ParseMarkdown(data:string, charArg:(character|simpleCharacterArgument | groupChat | string) = null, mode:'normal'|'back'|'pretranslate' = 'normal', chatID=-1) {
let firstParsed = ''
const orgDat = data
const db = get(DataBase)
const additionalAssetMode = (mode === 'back') ? 'back' : 'normal'
let char = (typeof(charArg) === 'string') ? (findCharacterbyId(charArg)) : (charArg)
if(char && char.type !== 'group'){
data = await parseAdditionalAssets(data, char, mode, 'pre')
data = await parseAdditionalAssets(data, char, additionalAssetMode, 'pre')
firstParsed = data
}
if(char){
data = (await processScriptFull(char, data, 'editdisplay', chatID)).data
}
if(firstParsed !== data && char && char.type !== 'group'){
data = await parseAdditionalAssets(data, char, mode, 'post')
data = await parseAdditionalAssets(data, char, additionalAssetMode, 'post')
}
data = await parseInlayImages(data)
data = encodeStyle(data)
if(mode !== 'back'){
if(mode === 'normal'){
data = risuFormater(data)
data = mconverted.parse(data)
}
@@ -197,6 +196,20 @@ export async function ParseMarkdown(data:string, charArg:(character|simpleCharac
}))
}
export function postTranslationParse(data:string){
let lines = risuFormater(data).split('\n')
for(let i=0;i<lines.length;i++){
const trimed = lines[i].trim()
if(trimed.startsWith('<')){
lines[i] = trimed
}
}
data = mconverted.parse(lines.join('\n'))
return data
}
export function parseMarkdownSafe(data:string) {
return DOMPurify.sanitize(mconverted.parse(data), {
FORBID_TAGS: ["a", "style"],

View File

@@ -94,8 +94,6 @@ export function risuFormater(dat:string){
}
}
console.log(lines)
let result = ''
for(let i=0;i<lines.length;i++){
if(lines[i][0] !== ''){
@@ -196,11 +194,8 @@ export function risuFormater(dat:string){
lineResult = lineResult.substring(0,lineResult.length-1)
}
console.log(lineResult)
result += lineResult
}
console.log(result)
return result.trim()
}

View File

@@ -284,8 +284,6 @@ export async function runCharacterJS(arg:{
const runCode = codes[arg.mode]
console.log(compCode[code])
if(!compCode[code].includes(runCode)){
continue
}

View File

@@ -1,54 +0,0 @@
import { get } from "svelte/store"
import { DataBase } from "../storage/database"
export function OaifixBias(bias:{[key:number]:number}){
const db = get(DataBase)
const emdashes = [
2001, 2345, 8713, 16620, 17223,
22416, 29096, 29472, 30697, 35192,
38542, 41128, 44603, 49525, 50004,
50617, 51749, 51757, 55434, 60654,
61311, 63750, 63938, 63977, 66101,
68850, 71201, 71480, 72318, 76070,
76929, 80078, 81902, 83872, 84941,
85366, 86319, 87247, 87671, 88958,
90863, 93830, 96197, 99563
]
const biases = []
if(db.officialplugins.oaiFixEmdash){
biases.push(...emdashes)
}
for (const key of biases) {
bias[key] = -100
}
return bias
}
export function OaiFixKorean(text:string){
//tokenizer problem fixes
const replacer = {
//commonly wrong english
'피츠': '피스',
'스커츠': '스커트',
'스파츠': '스커트',
'스마트폰': '스파트폰',
'스위츠': '스위치',
'해도 되': '해도 돼',
'해도 됩니다': '해도 돼요',
'에레베이터': '엘리베이터',
'에리베이터': '엘리베이터',
'에레바토르': '엘리베이터',
}
for (const key in replacer) {
text = text.replace(key, replacer[key])
}
return text
}

View File

@@ -1,49 +0,0 @@
// [from, to, ratio, isMetrica]
const convertion:[string,string,number, boolean][] = [
['kg', 'lbs', 2.20462, true],
['m', 'ft', 3.28084, true],
['cm', 'inch', 0.393701, false],
['mm', 'inch', 0.0393701, false],
['km', 'mi', 0.621371, false],
['killogram', 'pound', 2.20462, true],
['meter', 'foot', 3.28084, true],
['centimeter', 'inch', 0.393701, true],
['millimeter', 'inch', 0.0393701, true],
['kilometer', 'mile', 0.621371, true],
]
export function metricaPlugin(data:string, toSystem:'metrics'|'imperial'){
const c = convertion.sort((a,b) => b[0].length - a[0].length);
for(let i = 0; i < c.length; i++){
let [from, to, ratio] = c[i];
if(toSystem !== 'imperial'){
[from, to] = [to, from];
ratio = 1 / ratio;
}
if(!c[i][3]){
from = from + ' '
to = to + ' '
}
const reg = new RegExp(`(\\d+(?:\\.\\d+)?)\\s*${from}`, 'g');
data = data.replace(reg, (_, value) => {
// if value is integer, parse it as integer
if(value.indexOf('.') === -1){
const result = parseInt(value) * ratio
return `${result.toFixed(0)} ${to}`;
}
const result = parseFloat(value) * ratio;
return `${result.toFixed(2)} ${to}`;
});
}
//convert height like 5' 11'' to 180 cm
if(toSystem === 'metrics'){
const reg = /(\d+)'\s*(\d+)"/g;
data = data.replace(reg, (_, feet, inch) => {
const result = parseFloat(feet) * 30.48 + parseFloat(inch) * 2.54;
return `${result.toFixed(2)} cm`;
});
}
return data
}

View File

@@ -93,6 +93,7 @@ export async function importPlugin(){
displayName: displayName
}
db.plugins ??= []
db.plugins.push(pluginData)
DataBase.set(db)

View File

@@ -15,7 +15,6 @@ import { HttpRequest } from "@smithy/protocol-http";
import { Sha256 } from "@aws-crypto/sha256-js";
import { v4 } from "uuid";
import { supportsInlayImage } from "./files/image";
import { OaifixBias } from "../plugins/fixer";
import { Capacitor } from "@capacitor/core";
import { getFreeOpenRouterModel } from "../model/openrouter";
import { runTransformers } from "./transformers";
@@ -254,13 +253,6 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
}
}
if(raiModel.startsWith('gpt')){
if(db.officialplugins.oaiFix){
bias = OaifixBias(bias)
}
}
let oaiFunctions:OaiFunctions[] = []

View File

@@ -7,8 +7,6 @@ import { language } from "src/lang";
import { selectSingleFile } from "../util";
import { assetRegex, risuChatParser as risuChatParserOrg, type simpleCharacterArgument } from "../parser";
import { runCharacterJS } from "../plugins/embedscript";
import { metricaPlugin } from "../plugins/metrica";
import { OaiFixKorean } from "../plugins/fixer";
import { getModuleRegexScripts } from "./modules";
import { HypaProcesser } from "./memory/hypamemory";
@@ -62,15 +60,6 @@ export async function processScriptFull(char:character|groupChat|simpleCharacter
let db = get(DataBase)
let emoChanged = false
const scripts = (db.globalscript ?? []).concat(char.customscript).concat(getModuleRegexScripts())
if(db.officialplugins.metrica && mode === 'editdisplay'){
data = metricaPlugin(data, 'metrics')
}
if(db.officialplugins.metrica && (mode === 'editinput' || mode === 'editoutput' || mode === 'editprocess')){
data = metricaPlugin(data, 'imperial')
}
if(db.officialplugins.oaiFixLetters && db.officialplugins.oaiFix && (mode === 'editoutput' || mode === 'editdisplay')){
data = OaiFixKorean(data)
}
data = await runCharacterJS({
code: char.virtualscript ?? null,
mode,

View File

@@ -14,7 +14,7 @@ import type { OobaChatCompletionRequestParams } from '../model/ooba';
export const DataBase = writable({} as any as Database)
export const loadedStore = writable(false)
export let appVer = "114.4.0"
export let appVer = "114.5.0"
export let webAppSubVer = ''
export function setDatabase(data:Database){
@@ -418,6 +418,7 @@ export function setDatabase(data:Database){
data.lineHeight ??= 1.25
data.stabilityModel ??= 'sd3-large'
data.stabllityStyle ??= ''
data.legacyTranslation ??= false
changeLanguage(data.language)
DataBase.set(data)
}
@@ -693,6 +694,7 @@ export interface Database{
stabilityModel: string
stabilityKey: string
stabllityStyle: string
legacyTranslation: boolean
}
export interface customscript{

View File

@@ -1 +1 @@
{"version":"114.4.0"}
{"version":"114.5.0"}