Add custom tokenizers

2024-04-19 13:35:56 +09:00
parent b291242830
commit da272d83d8
4 changed files with 40 additions and 3 deletions
--- a/src/lang/en.ts
+++ b/src/lang/en.ts
@@ -576,4 +576,5 @@ export const languageEnglish = {
    autoContinueMinTokens: "Target Tokens (Auto Continue)",
    autoContinueChat: "Anti-Incomplete Response (Auto Continue)",
    removeIncompleteResponse: "Remove Incomplete Sentences",
+    tokenizer: "Tokenizer",
 }