Add tokenizer playground

src/lib/Playground/PlaygroundTokenizer.svelte

@@ -0,0 +1,31 @@
+<script lang="ts">
+    import { encode } from "src/ts/tokenizer";
+    import TextAreaInput from "../UI/GUI/TextAreaInput.svelte";
+    import { language } from 'src/lang';
+
+    let input = "";
+    let output = "";
+    let outputLength = 0;
+    const onInput = async () => {
+        try {
+            const tokenized = await encode(input);
+            const tokenizedNumArray = Array.from(tokenized)
+            outputLength = tokenizedNumArray.length;
+            output = JSON.stringify(tokenizedNumArray);
+        } catch (e) {
+            output = `Error: ${e}`
+        }
+    }
+</script>
+
+<h2 class="text-4xl text-textcolor my-6 font-black relative">{language.tokenizer}</h2>
+
+<span class="text-textcolor text-lg">Input</span>
+
+<TextAreaInput onInput={onInput} bind:value={input} height="32" />
+
+<span class="text-textcolor text-lg">Result</span>
+
+<TextAreaInput value={output} height="32" />
+
+<span class="text-textcolor2 text-lg">{outputLength} {language.tokens}</span>

src/ts/tokenizer.ts

@@ -18,7 +18,7 @@ export const tokenizerList = [
     ['novellist', 'Novellist'],
 ] as const
 
-async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
+export async function encode(data:string):Promise<(number[]|Uint32Array|Int32Array)>{
     let db = get(DataBase)
     if(db.aiModel === 'openrouter' || db.aiModel === 'reverse_proxy'){
         switch(db.customTokenizer){