From b1521384d37c1e21e91ec32bb53d9bf54e75f0ba Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 6 Dec 2023 06:07:26 +0900 Subject: [PATCH] [feat] oaifixer --- src/lib/Setting/Pages/PluginSettings.svelte | 20 +++++++--- src/ts/plugins/fixer.ts | 43 +++++++++++++++++++++ src/ts/process/index.ts | 23 ----------- src/ts/process/request.ts | 8 +++- src/ts/process/scripts.ts | 4 ++ src/ts/storage/database.ts | 3 ++ 6 files changed, 71 insertions(+), 30 deletions(-) create mode 100644 src/ts/plugins/fixer.ts diff --git a/src/lib/Setting/Pages/PluginSettings.svelte b/src/lib/Setting/Pages/PluginSettings.svelte index de0a486f..d9cce85d 100644 --- a/src/lib/Setting/Pages/PluginSettings.svelte +++ b/src/lib/Setting/Pages/PluginSettings.svelte @@ -24,18 +24,26 @@
-
- Romanizer (Official Plugin) -
-
- -
Metric Systemizer (Official Plugin)
+
+ OpenAI Fixer (Official Plugin) +
+
+ +
+ {#if $DataBase.officialplugins.oaiFix} +
+ +
+
+ +
+ {/if} {#each $DataBase.plugins as plugin, i}
diff --git a/src/ts/plugins/fixer.ts b/src/ts/plugins/fixer.ts new file mode 100644 index 00000000..e6edba2e --- /dev/null +++ b/src/ts/plugins/fixer.ts @@ -0,0 +1,43 @@ +export function OaifixEmdash(bias:{[key:number]:number}){ + const emdashes = [ + 2001, 2345, 8713, 16620, 17223, + 22416, 29096, 29472, 30697, 35192, + 38542, 41128, 44603, 49525, 50004, + 50617, 51749, 51757, 55434, 60654, + 61311, 63750, 63938, 63977, 66101, + 68850, 71201, 71480, 72318, 76070, + 76929, 80078, 81902, 83872, 84941, + 85366, 86319, 87247, 87671, 88958, + 90863, 93830, 96197, 99563 + ] + + for (const emdash of emdashes) { + bias[emdash] = -100 + } + + return bias + +} + +export function OaiFixKorean(text:string){ + + //tokenizer problem fixes + const replacer = { + //commonly wrong english + '피츠': '피스', + '스커츠': '스커트', + '스파츠': '스커트', + '스마트폰': '스파트폰', + '스위츠': '스위치', + '해도 되': '해도 돼', + '해도 됩니다': '해도 돼요', + '에레베이터': '엘리베이터', + '에리베이터': '엘리베이터', + '에레바토르': '엘리베이터', + } + + for (const key in replacer) { + text = text.replace(key, replacer[key]) + } + return text +} \ No newline at end of file diff --git a/src/ts/process/index.ts b/src/ts/process/index.ts index dc46f36d..5106015d 100644 --- a/src/ts/process/index.ts +++ b/src/ts/process/index.ts @@ -557,29 +557,6 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n currentTokens += await tokenizer.tokenizeChat(chat) } - if(db.officialplugins.romanizer){ - const romanizer = await import('../plugins/romanizer') - const r = romanizer.romanizer(chats.map((v) => { - return v.content - })) - - for(let i=0;i await tokenizer.tokenizeChat(pchat)){ - chats[i] = pchat - } - } - - if(r.mostUsed !== 'roman'){ - - unformated.postEverything.push({ - role: 'system', - content: `user and assistant are chatting with romanized ${r.mostUsed}, but always respond with ${r.mostUsed} with ${r.mostUsed} letters.` - }) - } - } - if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){ const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, { asHyper: db.supaMemoryType !== 'subModel' && db.hypaMemory diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 6c0df107..f04809e7 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -18,6 +18,7 @@ import { Sha256 } from "@aws-crypto/sha256-js"; import { v4 } from "uuid"; import { cloneDeep } from "lodash"; import { supportsInlayImage } from "../image"; +import { OaifixEmdash } from "../plugins/fixer"; @@ -236,6 +237,12 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' } } + if(db.officialplugins.oaiFix && db.officialplugins.oaiFixEmdash){ + if(raiModel.startsWith('gpt35') || raiModel.startsWith('gpt4')){ + bias = OaifixEmdash(bias) + } + } + @@ -273,7 +280,6 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' } - console.log(bias) db.cipherChat = false let body = ({ model: aiModel === 'openrouter' ? db.openrouterRequestModel : diff --git a/src/ts/process/scripts.ts b/src/ts/process/scripts.ts index ece99770..b3b20c4f 100644 --- a/src/ts/process/scripts.ts +++ b/src/ts/process/scripts.ts @@ -9,6 +9,7 @@ import { risuChatParser as risuChatParserOrg, type simpleCharacterArgument } fro import { autoMarkPlugin } from "../plugins/automark"; import { runCharacterJS } from "../plugins/embedscript"; import { metricaPlugin } from "../plugins/metrica"; +import { OaiFixKorean } from "../plugins/fixer"; const dreg = /{{data}}/g const randomness = /\|\|\|/g @@ -69,6 +70,9 @@ export async function processScriptFull(char:character|groupChat|simpleCharacter if(db.officialplugins.metrica && (mode === 'editinput' || mode === 'editoutput' || mode === 'editprocess')){ data = metricaPlugin(data, 'imperial') } + if(db.officialplugins.oaiFixLetters && db.officialplugins.oaiFix && (mode === 'editoutput' || mode === 'editdisplay')){ + data = OaiFixKorean(data) + } data = await runCharacterJS({ code: char.virtualscript ?? null, mode, diff --git a/src/ts/storage/database.ts b/src/ts/storage/database.ts index 809faed9..4706316f 100644 --- a/src/ts/storage/database.ts +++ b/src/ts/storage/database.ts @@ -396,6 +396,9 @@ export interface Database{ automark?: boolean romanizer?: boolean metrica?: boolean + oaiFix?: boolean + oaiFixEmdash?: boolean + oaiFixLetters?: boolean } currentPluginProvider: string zoomsize:number