Add Firefox(Bergamot) local translation (#794)

# PR Checklist
- [ ] Have you checked if it works normally in all models? *Ignore this
if it doesn't use models.*
- [ ] Have you checked if it works normally in all web, local, and node
hosted versions? If it doesn't, have you blocked it in those versions?
- [ ] Have you added type definitions?

# Description
Translation is performed using the models hosted in the repository:
https://github.com/mozilla/firefox-translations-models/

When the translation is first running, the model is downloaded from the
repository.

Testing completed for `npm dev` and Node server environment.

src/lang/cn.ts

@@ -512,6 +512,7 @@ export const languageChinese = {
     "showMenuChatList": "在菜单中显示聊天列表",
     "translatorLanguage": "翻译目标语言",
     "translatorType": "翻译器类型",
+    "htmlTranslation": "HTML 翻译",
     "deeplKey": "DeepL API 密钥",
     "deeplFreeKey": "DeepL 免费 API 密钥",
     "deeplXUrl": "DeepLX URL",

src/lang/de.ts

@@ -419,6 +419,7 @@ export const languageGerman = {
     showMenuChatList: "Menü-Chatliste anzeigen",
     translatorLanguage: "Übersetzer-Sprache",
     translatorType: "Übersetzer-Typ",
+    htmlTranslation: "HTML-Übersetzung",
     deeplKey: "DeepL API-Schlüssel",
     deeplFreeKey: "DeepL Gratis API-Schlüssel",
     deeplXUrl: "deepLX URL",

src/lang/en.ts

@@ -702,6 +702,7 @@ export const languageEnglish = {
     showMenuChatList: "Show Menu Chat List",
     translatorLanguage: "Translator Language",
     translatorType: "Translator Type",
+    htmlTranslation: "HTML Translate",
     deeplKey: "deepL API Key",
     deeplFreeKey: "deepL Free API Key",
     deeplXUrl: "deepLX URL",

src/lang/es.ts

@@ -474,6 +474,7 @@ export const languageSpanish = {
     showMenuChatList: "Mostrar Menú de Lista de Chats",
     translatorLanguage: "Idioma del Traductor",
     translatorType: "Tipo de Traductor",
+    htmlTranslation: "Traducción de HTML",
     deeplKey: "Clave API de DeepL",
     deeplFreeKey: "Clave API Gratis de DeepL",
     deeplXUrl: "URL de DeepLX",

src/lang/ko.ts

@@ -651,6 +651,7 @@ export const languageKorean = {
     "showMenuChatList": "메뉴에서 채팅 리스트 보이기",
     "translatorLanguage": "번역기 언어",
     "translatorType": "번역기 타입",
+    "htmlTranslation": "HTML 번역",
     "deeplKey": "deepL API 키",
     "deeplFreeKey": "deepL 무료 API 키",
     "deeplXUrl": "deepLX URL",

src/lang/vi.ts

@@ -390,6 +390,7 @@ export const LanguageVietnamese = {
     "showMenuChatList": "Hiển thị Menu Danh sách Trò chuyện",
     "translatorLanguage": "Ngôn ngữ dịch",
     "translatorType": "Loại dịch giả",
+    "htmlTranslation": "Dịch HTML",
     "deeplKey": "Khóa API deepL",
     "deeplFreeKey": "Khóa API miễn phí deepL",
     "deeplXUrl": "deepLX URL",

src/lang/zh-Hant.ts

@@ -518,6 +518,7 @@ export const languageChineseTraditional = {
     "showMenuChatList": "在選單中顯示聊天列表",
     "translatorLanguage": "翻譯目標語言",
     "translatorType": "翻譯器類型",
+    "htmlTranslation": "HTML 翻譯",
     "deeplKey": "DeepL API 金鑰",
     "deeplFreeKey": "DeepL 免費 API 金鑰",
     "deeplXUrl": "DeepLX URL",

src/lib/Setting/Pages/LanguageSettings.svelte

@@ -90,6 +90,7 @@
         <OptionInput value="deepl" >DeepL</OptionInput>
         <OptionInput value="llm" >Ax. Model</OptionInput>
         <OptionInput value="deeplX" >DeepL X</OptionInput>
+        <OptionInput value="bergamot" >Firefox</OptionInput>
     </SelectInput>
 
     {#if DBState.db.translatorType === 'deepl'}
@@ -135,6 +136,11 @@
         </SelectInput>
     {/if}
 
+    {#if DBState.db.translatorType === 'bergamot'}
+        <div class="flex items-center mt-4">
+            <Check bind:check={DBState.db.htmlTranslation} name={language.htmlTranslation}/>
+        </div>
+    {/if}
 
     <div class="flex items-center mt-2">
         <Check bind:check={DBState.db.autoTranslate} name={language.autoTranslation}/>

src/ts/storage/database.svelte.ts

@@ -335,6 +335,7 @@ export function setDatabase(data:Database){
     data.mancerHeader ??= ''
     data.emotionProcesser ??= 'submodel'
     data.translatorType ??= 'google'
+    data.htmlTranslation ??= false
     data.deeplOptions ??= {
         key:'',
         freeApi: false
@@ -736,8 +737,9 @@ export interface Database{
     mancerHeader:string
     emotionProcesser:'submodel'|'embedding',
     showMenuChatList?:boolean,
-    translatorType:'google'|'deepl'|'none'|'llm'|'deeplX',
+    translatorType:'google'|'deepl'|'none'|'llm'|'deeplX'|'bergamot',
     translatorInputLanguage?:string
+    htmlTranslation?:boolean,
     NAIadventure?:boolean,
     NAIappendName?:boolean,
     deeplOptions:{

src/ts/translator/bergamotTranslator.ts

@@ -0,0 +1,145 @@
+import { LatencyOptimisedTranslator, TranslatorBacking } from "@browsermt/bergamot-translator";
+import { gunzipSync } from 'fflate';
+
+// Cache Translations Models
+class CacheDB {
+    private readonly dbName: string;
+    private readonly storeName: string = "cache";
+
+    constructor(dbName: string = "cache") {
+        this.dbName = dbName;
+    }
+
+    private async getDB(): Promise<IDBDatabase> {
+        return new Promise((resolve, reject) => {
+            const request = indexedDB.open(this.dbName, 1);
+
+            request.onupgradeneeded = (event: IDBVersionChangeEvent) => {
+                const db = (event.target as IDBOpenDBRequest).result;
+                if (!db.objectStoreNames.contains(this.storeName)) {
+                    db.createObjectStore(this.storeName, { keyPath: "url" });
+                }
+            };
+
+            request.onsuccess = () => resolve(request.result);
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    async load(url: string, checksum: string): Promise<ArrayBuffer | null> {
+        const db = await this.getDB();
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(this.storeName, "readonly");
+            const store = transaction.objectStore(this.storeName);
+            const request = store.get(url);
+
+            request.onsuccess = () => {
+                const result = request.result;
+                if (result && result.checksum === checksum) {
+                    resolve(result.buffer);
+                } else {
+                    resolve(null);
+                }
+            };
+
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    async save(url: string, checksum: string, buffer: ArrayBuffer): Promise<void> {
+        const db = await this.getDB();
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(this.storeName, "readwrite");
+            const store = transaction.objectStore(this.storeName);
+            const request = store.put({ url, checksum, buffer });
+
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+
+    async clear(): Promise<void> {
+        const db = await this.getDB();
+        return new Promise((resolve, reject) => {
+            const transaction = db.transaction(this.storeName, "readwrite");
+            const store = transaction.objectStore(this.storeName);
+            const request = store.clear();
+
+            request.onsuccess = () => resolve();
+            request.onerror = () => reject(request.error);
+        });
+    }
+}
+
+// Mozilla Firefox Translations Models
+class FirefoxBacking extends TranslatorBacking {
+    private cache: CacheDB;
+    downloadTimeout: number;
+
+    constructor(options?) {
+        const registryUrl = 'https://raw.githubusercontent.com/mozilla/firefox-translations-models/refs/heads/main/registry.json';
+        options = options || {};
+        options.registryUrl = options.registryUrl || registryUrl;
+        super(options);
+        this.cache = new CacheDB("firefox-translations-models");
+    }
+
+    async loadModelRegistery() {
+        const modelUrl = 'https://media.githubusercontent.com/media/mozilla/firefox-translations-models/refs/heads/main/models';
+        const registry = await super.loadModelRegistery();
+        for (const entry of registry) {
+            for(const name in entry.files) {
+                const file = entry.files[name];
+                file.name = `${modelUrl}/${file.modelType}/${entry.from}${entry.to}/${file.name}.gz`;
+            }
+        }
+        return registry;
+    }
+
+    async fetch(url, checksum, extra) {
+        const cacheBuffer = await this.cache.load(url, checksum);
+        if (cacheBuffer) { return cacheBuffer; }
+        const res = await fetch(url, {
+            credentials: 'omit',
+        });
+        // Decompress GZip
+        const buffer = await res.arrayBuffer();
+        const decomp = await decompressGZip(buffer);
+        await this.cache.save(url, checksum, decomp);
+        return decomp;
+    }
+}
+
+async function decompressGZip(buffer:ArrayBuffer) {
+    if (typeof DecompressionStream !== "undefined") {
+        const decompressor = new DecompressionStream('gzip');
+        const stream = new Response(buffer).body.pipeThrough(decompressor);
+        return await new Response(stream).arrayBuffer();
+    } else {    // GZip decompression fallback
+        return gunzipSync(new Uint8Array(buffer)).buffer;
+    }
+}
+
+let translator = null;
+let translateTask = null;
+
+// Translate
+export async function bergamotTranslate(text:string, from:string, to:string, html:boolean|null) {
+    translator ??= new LatencyOptimisedTranslator({}, new FirefoxBacking())
+    const result = await (translateTask = translate());
+    return result.target.text;
+
+    // Wait for previous tasks...
+    async function translate() {
+        await translateTask;
+        return translator.translate({
+            from: from, to: to,
+            text: text, html: html,
+        });
+    }
+}
+
+// Clear Cache
+export async function clearCache() {
+    await new CacheDB("firefox-translations-models").clear();
+}

src/ts/translator/translator.ts

@@ -10,6 +10,7 @@ import { selectedCharID } from "../stores.svelte"
 import { getModuleRegexScripts } from "../process/modules"
 import { getNodetextToSentence, sleep } from "../util"
 import { processScriptFull } from "../process/scripts"
+import { bergamotTranslate } from "./bergamotTranslator"
 import localforage from "localforage"
 import sendSound from '../../etc/send.mp3'
 
@@ -165,6 +166,9 @@ async function translateMain(text:string, arg:{from:string, to:string, host:stri
 
         return f.data.data;
     }
+    if(db.translatorType == "bergamot") {
+        return bergamotTranslate(text, arg.from, arg.to, false);
+    }
     if(db.useExperimentalGoogleTranslator){
 
         const hqAvailable = isTauri || isNodeServer || userScriptFetch
@@ -274,6 +278,11 @@ export async function translateHTML(html: string, reverse:boolean, charArg:simpl
 
         return r
     }
+    if(db.translatorType == "bergamot" && db.htmlTranslation) {
+        const from = db.aiModel.startsWith('novellist') ? 'ja' : 'en'
+        const to = db.translator || 'en'
+        return bergamotTranslate(html, from, to, true)
+    }
     const dom = new DOMParser().parseFromString(html, 'text/html');
     console.log(html)