diff --git a/src/lib/Setting/Pages/PluginSettings.svelte b/src/lib/Setting/Pages/PluginSettings.svelte
index de0a486f..d9cce85d 100644
--- a/src/lib/Setting/Pages/PluginSettings.svelte
+++ b/src/lib/Setting/Pages/PluginSettings.svelte
@@ -24,18 +24,26 @@
-
- Romanizer (Official Plugin)
-
-
-
-
Metric Systemizer (Official Plugin)
+
+ OpenAI Fixer (Official Plugin)
+
+
+
+
+ {#if $DataBase.officialplugins.oaiFix}
+
+
+
+
+
+
+ {/if}
{#each $DataBase.plugins as plugin, i}
diff --git a/src/ts/plugins/fixer.ts b/src/ts/plugins/fixer.ts
new file mode 100644
index 00000000..e6edba2e
--- /dev/null
+++ b/src/ts/plugins/fixer.ts
@@ -0,0 +1,43 @@
+export function OaifixEmdash(bias:{[key:number]:number}){
+ const emdashes = [
+ 2001, 2345, 8713, 16620, 17223,
+ 22416, 29096, 29472, 30697, 35192,
+ 38542, 41128, 44603, 49525, 50004,
+ 50617, 51749, 51757, 55434, 60654,
+ 61311, 63750, 63938, 63977, 66101,
+ 68850, 71201, 71480, 72318, 76070,
+ 76929, 80078, 81902, 83872, 84941,
+ 85366, 86319, 87247, 87671, 88958,
+ 90863, 93830, 96197, 99563
+ ]
+
+ for (const emdash of emdashes) {
+ bias[emdash] = -100
+ }
+
+ return bias
+
+}
+
+export function OaiFixKorean(text:string){
+
+ //tokenizer problem fixes
+ const replacer = {
+ //commonly wrong english
+ '피츠': '피스',
+ '스커츠': '스커트',
+ '스파츠': '스커트',
+ '스마트폰': '스파트폰',
+ '스위츠': '스위치',
+ '해도 되': '해도 돼',
+ '해도 됩니다': '해도 돼요',
+ '에레베이터': '엘리베이터',
+ '에리베이터': '엘리베이터',
+ '에레바토르': '엘리베이터',
+ }
+
+ for (const key in replacer) {
+ text = text.replace(key, replacer[key])
+ }
+ return text
+}
\ No newline at end of file
diff --git a/src/ts/process/index.ts b/src/ts/process/index.ts
index dc46f36d..5106015d 100644
--- a/src/ts/process/index.ts
+++ b/src/ts/process/index.ts
@@ -557,29 +557,6 @@ export async function sendChat(chatProcessIndex = -1,arg:{chatAdditonalTokens?:n
currentTokens += await tokenizer.tokenizeChat(chat)
}
- if(db.officialplugins.romanizer){
- const romanizer = await import('../plugins/romanizer')
- const r = romanizer.romanizer(chats.map((v) => {
- return v.content
- }))
-
- for(let i=0;i await tokenizer.tokenizeChat(pchat)){
- chats[i] = pchat
- }
- }
-
- if(r.mostUsed !== 'roman'){
-
- unformated.postEverything.push({
- role: 'system',
- content: `user and assistant are chatting with romanized ${r.mostUsed}, but always respond with ${r.mostUsed} with ${r.mostUsed} letters.`
- })
- }
- }
-
if(nowChatroom.supaMemory && db.supaMemoryType !== 'none'){
const sp = await supaMemory(chats, currentTokens, maxContextTokens, currentChat, nowChatroom, tokenizer, {
asHyper: db.supaMemoryType !== 'subModel' && db.hypaMemory
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 6c0df107..f04809e7 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -18,6 +18,7 @@ import { Sha256 } from "@aws-crypto/sha256-js";
import { v4 } from "uuid";
import { cloneDeep } from "lodash";
import { supportsInlayImage } from "../image";
+import { OaifixEmdash } from "../plugins/fixer";
@@ -236,6 +237,12 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
}
}
+ if(db.officialplugins.oaiFix && db.officialplugins.oaiFixEmdash){
+ if(raiModel.startsWith('gpt35') || raiModel.startsWith('gpt4')){
+ bias = OaifixEmdash(bias)
+ }
+ }
+
@@ -273,7 +280,6 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
}
- console.log(bias)
db.cipherChat = false
let body = ({
model: aiModel === 'openrouter' ? db.openrouterRequestModel :
diff --git a/src/ts/process/scripts.ts b/src/ts/process/scripts.ts
index ece99770..b3b20c4f 100644
--- a/src/ts/process/scripts.ts
+++ b/src/ts/process/scripts.ts
@@ -9,6 +9,7 @@ import { risuChatParser as risuChatParserOrg, type simpleCharacterArgument } fro
import { autoMarkPlugin } from "../plugins/automark";
import { runCharacterJS } from "../plugins/embedscript";
import { metricaPlugin } from "../plugins/metrica";
+import { OaiFixKorean } from "../plugins/fixer";
const dreg = /{{data}}/g
const randomness = /\|\|\|/g
@@ -69,6 +70,9 @@ export async function processScriptFull(char:character|groupChat|simpleCharacter
if(db.officialplugins.metrica && (mode === 'editinput' || mode === 'editoutput' || mode === 'editprocess')){
data = metricaPlugin(data, 'imperial')
}
+ if(db.officialplugins.oaiFixLetters && db.officialplugins.oaiFix && (mode === 'editoutput' || mode === 'editdisplay')){
+ data = OaiFixKorean(data)
+ }
data = await runCharacterJS({
code: char.virtualscript ?? null,
mode,
diff --git a/src/ts/storage/database.ts b/src/ts/storage/database.ts
index 809faed9..4706316f 100644
--- a/src/ts/storage/database.ts
+++ b/src/ts/storage/database.ts
@@ -396,6 +396,9 @@ export interface Database{
automark?: boolean
romanizer?: boolean
metrica?: boolean
+ oaiFix?: boolean
+ oaiFixEmdash?: boolean
+ oaiFixLetters?: boolean
}
currentPluginProvider: string
zoomsize:number