From 58513ed54a0c7e53aefdba48bcb1f3de32d553ba Mon Sep 17 00:00:00 2001
From: kwaroran <amasio1234@proton.me>
Date: Wed, 24 Apr 2024 22:06:02 +0900
Subject: [PATCH] Add tokenizer playground

---
 src/lib/Playground/PlaygroundTokenizer.svelte | 31 +++++++++++++++++++
 src/ts/tokenizer.ts                           |  2 +-
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/lib/Playground/PlaygroundTokenizer.svelte b/src/lib/Playground/PlaygroundTokenizer.svelte
index e69de29b..1efff614 100644
--- a/src/lib/Playground/PlaygroundTokenizer.svelte
+++ b/src/lib/Playground/PlaygroundTokenizer.svelte
@@ -0,0 +1,31 @@
+<script lang="ts">
+    import { encode } from "src/ts/tokenizer";
+    import TextAreaInput from "../UI/GUI/TextAreaInput.svelte";
+    import { language } from 'src/lang';
+
+    let input = "";
+    let output = "";
+    let outputLength = 0;
+    const onInput = async () => {
+        try {
+            const tokenized = await encode(input);
+            const tokenizedNumArray = Array.from(tokenized)
+            outputLength = tokenizedNumArray.length;
+            output = JSON.stringify(tokenizedNumArray);
+        } catch (e) {
+            output = `Error: ${e}`
+        }
+    }
+</script>
+
+<h2 class="text-4xl text-textcolor my-6 font-black relative">{language.tokenizer}</h2>
+
+<span class="text-textcolor text-lg">Input</span>
+
+<TextAreaInput onInput={onInput} bind:value={input} height="32" />
+
+<span class="text-textcolor text-lg">Result</span>
+
+<TextAreaInput value={output} height="32" />
+
+<span class="text-textcolor2 text-lg">{outputLength} {language.tokens}</span>
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index e6d52ff8..e02650c2 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -18,7 +18,7 @@ export const tokenizerList = [
     ['novellist', 'Novellist'],
 ] as const
 
-async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
+export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
     let db = get(DataBase)
     if(db.aiModel === 'openrouter' || db.aiModel === 'reverse_proxy'){
         switch(db.customTokenizer){