Add playground and fix embeddings

src/lang/en.ts

@@ -580,4 +580,8 @@ export const languageEnglish = {
     chatFormating: "Chat Formating",
     useInstructPrompt: "Use Instruction Prompt",
     hanuraiMemory: "HanuraiMemory",
+    playground: "Playground",
+    embedding: "Embedding",
+    syntax: "Syntax",
+    run: "Run",
 }

src/lib/ChatScreens/DefaultChatScreen.svelte

@@ -2,7 +2,7 @@
 	import Suggestion from './Suggestion.svelte';
 	import AdvancedChatEditor from './AdvancedChatEditor.svelte';
     import { CameraIcon, DatabaseIcon, DicesIcon, GlobeIcon, ImagePlusIcon, LanguagesIcon, Laugh, MenuIcon, MicOffIcon, PackageIcon, RefreshCcwIcon, ReplyIcon, Send, StepForwardIcon } from "lucide-svelte";
-    import { CurrentCharacter, CurrentChat, CurrentUsername, selectedCharID, CurrentUserIcon, CurrentShowMemoryLimit,CurrentSimpleCharacter } from "../../ts/stores";
+    import { CurrentCharacter, CurrentChat, CurrentUsername, selectedCharID, CurrentUserIcon, CurrentShowMemoryLimit,CurrentSimpleCharacter, PlaygroundStore } from "../../ts/stores";
     import Chat from "./Chat.svelte";
     import { DataBase, type Message, type character, type groupChat } from "../../ts/storage/database";
     import { getCharImage } from "../../ts/characters";
@@ -25,6 +25,7 @@
     import { processMultiCommand } from 'src/ts/process/command';
     import { postChatFile } from 'src/ts/process/files/multisend';
   import { getInlayImage } from 'src/ts/process/files/image';
+  import PlaygroundMenu from '../Playground/PlaygroundMenu.svelte';
 
     let messageInput:string = ''
     let messageInputTranslate:string = ''
@@ -391,7 +392,11 @@
     openMenu = false
 }}>
     {#if $selectedCharID < 0}
+        {#if $PlaygroundStore === 0}
             <MainMenu />
+        {:else}
+            <PlaygroundMenu />
+        {/if}
     {:else}
         <div class="h-full w-full flex flex-col-reverse overflow-y-auto relative default-chat-screen"  on:scroll={(e) => {
             //@ts-ignore  
@@ -721,17 +726,6 @@
     {/if}
 </div>
 <style>
-    .loadmove {
-        animation: spin 1s linear infinite;
-        border-radius: 50%;
-        border: 0.4rem solid rgba(0,0,0,0);
-        width: 1rem;
-        height: 1rem;
-        border-top: 0.4rem solid var(--risu-theme-borderc);
-        border-left: 0.4rem solid var(--risu-theme-borderc);
-        /* transition colors */
-        transition: border-color 0.5s;
-    }
 
     .chat-process-stage-1{
         border-top: 0.4rem solid #60a5fa;

src/lib/Playground/PlaygroundEmbedding.svelte

@@ -0,0 +1,65 @@
+<script lang="ts">
+    import { language } from "src/lang";
+    import TextInput from "../UI/GUI/TextInput.svelte";
+    import OptionInput from "../UI/GUI/OptionInput.svelte";
+    import SelectInput from "../UI/GUI/SelectInput.svelte";
+    import Button from "../UI/GUI/Button.svelte";
+    import { HypaProcesser } from "src/ts/process/memory/hypamemory";
+
+    let query = "";
+    let model = "MiniLM";
+    let data:string[] = [];
+    let dataresult:[string, number][] = [];
+    let running = false;
+
+    const run = async () => {
+        if(running) return;
+        running = true;
+        const processer = new HypaProcesser(model as any);
+        await processer.addText(data);
+        console.log(processer.vectors)
+        dataresult = await processer.similaritySearchScored(query);
+        running = false;
+    }
+</script>
+  
+<h2 class="text-4xl text-textcolor my-6 font-black relative">{language.embedding}</h2>
+
+<span class="text-textcolor text-lg">Model</span>
+<SelectInput bind:value={model}>
+    <OptionInput value="MiniLM">MiniLM L6 v2</OptionInput>
+    <OptionInput value="nomic">Nomic Embed Text v1.5</OptionInput>
+</SelectInput>
+
+<span class="text-textcolor text-lg">Query</span>
+<TextInput bind:value={query} size="lg" fullwidth />
+
+<span class="text-textcolor text-lg mt-6">Data</span>
+{#each data as item}
+    <TextInput bind:value={item} size="lg" fullwidth marginBottom />
+{/each}
+<Button styled="outlined" on:click={() => {
+    data.push("");
+    data = data
+}}>+</Button>
+
+<span class="text-textcolor text-lg mt-6">Result</span>
+{#if dataresult.length === 0}
+    <span class="text-textcolor2 text-lg">No result</span>
+{/if}
+{#each dataresult as [item, score]}
+    <div class="flex justify-between">
+        <span>{item}</span>
+        <span>{score.toFixed(2)}</span>
+    </div>
+{/each}
+
+<Button className="mt-6 flex justify-center" size="lg" on:click={() => {
+    run()
+}}>
+    {#if running}
+        <div class="loadmove" />
+    {:else}
+        {language.run?.toLocaleUpperCase()}
+    {/if}
+</Button>

src/lib/Playground/PlaygroundMenu.svelte

@@ -0,0 +1,78 @@
+<script lang="ts">
+    import { ArrowLeft } from "lucide-svelte";
+    import { language } from "src/lang";
+    import { PlaygroundStore, SizeStore } from "src/ts/stores";
+    import PlaygroundEmbedding from "./PlaygroundEmbedding.svelte";
+    import PlaygroundTokenizer from "./PlaygroundTokenizer.svelte";
+    import PlaygroundJinja from "./PlaygroundJinja.svelte";
+    import PlaygroundSyntax from "./PlaygroundSyntax.svelte";
+    import PlaygroundRegex from "./PlaygroundRegex.svelte";
+</script>
+
+<div class="h-full w-full flex flex-col overflow-y-auto items-center">
+    {#if $PlaygroundStore === 1}
+        <h2 class="text-4xl text-textcolor my-6 font-black relative">{language.playground}</h2>
+        <div class="grid grid-cols-1 gap-4 md:grid-cols-2 w-full max-w-4xl">
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" on:click={() => {
+                PlaygroundStore.set(2)
+            }}>
+                <h1 class="text-2xl font-bold text-start">{language.Chat}</h1>
+            </button>
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" on:click={() => {
+                PlaygroundStore.set(3)
+            }}>
+                <h1 class="text-2xl font-bold text-start">{language.embedding}</h1>
+            </button>
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" on:click={() => {
+                PlaygroundStore.set(4)
+            }}>
+                <h1 class="text-2xl font-bold text-start">{language.tokenizer}</h1>
+            </button>
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" on:click={() => {
+                PlaygroundStore.set(5)
+            }}>
+                <h1 class="text-2xl font-bold text-start">{language.syntax}</h1>
+            </button>
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" on:click={() => {
+                PlaygroundStore.set(6)
+            }}>
+                <h1 class="text-2xl font-bold text-start">Jinja</h1>
+            </button>
+            <button class="bg-darkbg rounded-md p-6 flex flex-col transition-shadow hover:ring-1" on:click={() => {
+                PlaygroundStore.set(7)
+            }}>
+                <h1 class="text-2xl font-bold text-start">Regex</h1>
+            </button>
+        </div>
+    {:else}
+        {#if $SizeStore.w < 1024}
+            <div class="mt-14"></div>
+        {/if}
+        <div class="w-full max-w-4xl flex flex-col">
+            <div class="flex items-center mt-4">
+                <button class="mr-2 text-textcolor2 hover:text-green-500" on:click={() => ($PlaygroundStore = 1)}>
+                <ArrowLeft/>
+                </button>
+            </div>
+
+            {#if $PlaygroundStore === 2}
+                <!-- <PlaygroundChat/> -->
+            {/if}
+            {#if $PlaygroundStore === 3}
+                <PlaygroundEmbedding/>
+            {/if}
+            {#if $PlaygroundStore === 4}
+                <PlaygroundTokenizer/>
+            {/if}
+            {#if $PlaygroundStore === 5}
+                <PlaygroundSyntax/>
+            {/if}
+            {#if $PlaygroundStore === 6}
+                <PlaygroundJinja/>
+            {/if}
+            {#if $PlaygroundStore === 7}
+                <PlaygroundRegex/>
+            {/if}
+        </div>
+    {/if}
+</div>

src/lib/SideBars/Sidebar.svelte

@@ -9,14 +9,17 @@
     sideBarStore,
     CurrentCharacter,
 
-    OpenRealmStore
+    OpenRealmStore,
+
+    PlaygroundStore
+
 
   } from "../../ts/stores";
   import { DataBase, setDatabase, type folder } from "../../ts/storage/database";
   import BarIcon from "./BarIcon.svelte";
   import SidebarIndicator from "./SidebarIndicator.svelte";
   import {
-    X,
+    ShellIcon,
     Settings,
     ListIcon,
     LayoutGridIcon,
@@ -366,6 +369,8 @@
         onClick={() => {
           reseter();
           selectedCharID.set(-1)
+          PlaygroundStore.set(0)
+          OpenRealmStore.set(false)
         }}><HomeIcon /></BarIcon>
       <div class="mt-2"></div>
       <BarIcon
@@ -565,6 +570,20 @@
         }
       }} on:dragenter={preventAll} />
     {/each}
+        <div class="flex flex-col items-center space-y-2 px-2 mb-2">
+      <BaseRoundedButton
+        onClick={() => {
+          reseter()
+          if($selectedCharID === -1 && $PlaygroundStore !== 0){
+            PlaygroundStore.set(0)
+            return
+          }
+          selectedCharID.set(-1)
+          PlaygroundStore.set(1)
+        }}
+        ><ShellIcon size="24" /></BaseRoundedButton
+      >
+    </div>
     <div class="flex flex-col items-center space-y-2 px-2">
       <BaseRoundedButton
         onClick={async () => {
@@ -597,17 +616,6 @@
         ></BaseRoundedButton
       >
     </div>
-    {#if $DataBase.tpo}
-      <div class="flex flex-col items-center space-y-2 px-2 mt-2 mb-2">
-        <BaseRoundedButton
-          onClick={() => {
-            reseter()
-            sideBarMode = sideBarMode === 2 ? 0 : 2
-          }}
-          ><MilestoneIcon size="24" /></BaseRoundedButton
-        >
-      </div>
-    {/if}
   </div>
 </div>
 <div

src/lib/UI/GUI/Button.svelte

@@ -1,8 +1,8 @@
 <button
     on:click
     class="{
-        styled === 'primary' ?
+        styled === 'primary' ? ((selected ? 'bg-borderc' : 'bg-darkbutton') + " hover:bg-borderc focus:ring-borderc border-darkborderc")
-        ((selected ? 'bg-borderc' : 'bg-darkbutton') + " hover:bg-borderc focus:ring-borderc border-darkborderc")
+        : styled === 'outlined' ? 'bg-transparent hover:bg-darkbg focus:ring-borderc border-darkborderc text-textcolor2'
         : ((selected ? 'bg-red-800' : 'bg-red-700') + ' hover:bg-red-500 focus:ring-red-600 border-red-600')
     } border text-textcolor rounded-md shadow-sm focus:outline-none focus:ring-2 transition-colors duration-200{className ? (" " + className) : ""}"
     class:px-4 = {size == "md"}
@@ -18,7 +18,7 @@
 </button>
 <script lang="ts">
     export let selected = false
-    export let styled:'primary'|'danger' = 'primary'
+    export let styled:'primary'|'danger'|'outlined' = 'primary'
     export let className = ""
     export let size: "sm" | "md" | "lg" = "md"
 </script>

src/styles.css

@@ -180,3 +180,15 @@ html, body{
 .x-risu-button-default{
   @apply border text-textcolor rounded-md shadow-sm focus:outline-none focus:ring-2 transition-colors duration-200 border-darkborderc px-4 py-2 bg-darkbutton hover:bg-borderc focus:ring-borderc
 }
+
+.loadmove {
+  animation: spin 1s linear infinite;
+  border-radius: 50%;
+  border: 0.4rem solid rgba(0,0,0,0);
+  width: 1rem;
+  height: 1rem;
+  border-top: 0.4rem solid var(--risu-theme-borderc);
+  border-left: 0.4rem solid var(--risu-theme-borderc);
+  /* transition colors */
+  transition: border-color 0.5s;
+}

src/ts/process/transformers.ts

@@ -57,7 +57,16 @@ export const runEmbedding = async (texts: string[], model:EmbeddingModel = 'Xeno
         extractor = await pipeline('feature-extraction', model);
     }
     let result = await extractor(texts, { pooling: 'mean', normalize: true });
-    return (result.data as Float32Array[]) ?? null;
+    console.log(texts, result)
+    const data = result.data as Float32Array
+
+    const lenPerText = data.length / texts.length
+    let res:Float32Array[] = []
+    for(let i = 0; i < texts.length; i++){
+        res.push(data.subarray(i * lenPerText, (i + 1) * lenPerText))
+    }
+    console.log(res)
+    return res ?? [];
 }
 
 export const runImageEmbedding = async (dataurl:string) => {

src/ts/storage/globalApi.ts

@@ -986,7 +986,7 @@ export function checkCharOrder() {
         const charId = db.characters[i].chaId
         charIdList.push(charId)
         if(!ordered.includes(charId)){
-            if(charId !== '§temp'){
+            if(charId !== '§temp' && charId !== '§playground'){
                 db.characterOrder.push(charId)
             }
         }

src/ts/stores.ts

@@ -40,6 +40,7 @@ export const ShowVN = writable(false)
 export const SettingsMenuIndex = writable(-1)
 export const CurrentVariablePointer = writable({} as {[key:string]: string|number|boolean})
 export const OpenRealmStore = writable(false)
+export const PlaygroundStore = writable(0)
 
 function createSimpleCharacter(char:character|groupChat){
     if((!char) || char.type === 'group'){