From ccda92cc4907263c3980560df1ec4eb386ab5a10 Mon Sep 17 00:00:00 2001
From: kwaroran <amasio1234@proton.me>
Date: Sun, 1 Sep 2024 19:30:06 +0900
Subject: [PATCH] Add autopilot

---
 src/lib/SideBars/DevTool.svelte | 135 ++++++++++++++++++++++++++++++--
 src/ts/tokenizer.ts             |  56 ++++++++++++-
 2 files changed, 182 insertions(+), 9 deletions(-)

diff --git a/src/lib/SideBars/DevTool.svelte b/src/lib/SideBars/DevTool.svelte
index 085dadcb..74f05a78 100644
--- a/src/lib/SideBars/DevTool.svelte
+++ b/src/lib/SideBars/DevTool.svelte
@@ -1,13 +1,22 @@
 <script lang="ts">
-    import { CurrentChat } from "src/ts/stores";
-  import TextInput from "../UI/GUI/TextInput.svelte";
-  import NumberInput from "../UI/GUI/NumberInput.svelte";
-  import Button from "../UI/GUI/Button.svelte";
-  import { getRequestLog } from "src/ts/storage/globalApi";
-  import { alertMd } from "src/ts/alert";
-  import Arcodion from "../UI/Arcodion.svelte";
+    import { CurrentCharacter, CurrentChat, selectedCharID } from "src/ts/stores";
+    import TextInput from "../UI/GUI/TextInput.svelte";
+    import NumberInput from "../UI/GUI/NumberInput.svelte";
+    import Button from "../UI/GUI/Button.svelte";
+    import { getRequestLog } from "src/ts/storage/globalApi";
+    import { alertMd } from "src/ts/alert";
+    import Arcodion from "../UI/Arcodion.svelte";
+    import { getCharToken, getChatToken } from "src/ts/tokenizer";
+    import { tokenizePreset } from "src/ts/process/prompt";
+    import { DataBase, setDatabase } from "src/ts/storage/database";
+  import TextAreaInput from "../UI/GUI/TextAreaInput.svelte";
+  import { FolderUpIcon, PlusIcon, TrashIcon } from "lucide-svelte";
+  import { selectSingleFile } from "src/ts/util";
+  import { file } from "jszip";
+  import { doingChat, sendChat } from "src/ts/process";
 
     
+    let autopilot = []
 </script>
 
 <Arcodion styled name={"Variables"}>
@@ -29,6 +38,116 @@
     </div>
 </Arcodion>
 
-<Button on:click={() => {
+<Arcodion styled name={"Tokens"}>
+    <div class="rounded-md border border-darkborderc grid grid-cols-2 gap-2 p-2">
+        {#await getCharToken($CurrentCharacter)}
+            <span>Character Persistant</span>
+            <div class="p-2 text-center">Loading...</div>
+            <span>Character Dynamic</span>
+            <div class="p-2 text-center">Loading...</div>
+        {:then token}
+            <span>Character Persistant</span>
+            <div class="p-2 text-center">{token.persistant} Tokens</div>
+            <span>Character Dynamic</span>
+            <div class="p-2 text-center">{token.dynamic} Tokens</div>
+        {/await}
+        {#await getChatToken($CurrentChat)}
+            <span>Current Chat</span>
+            <div class="p-2 text-center">Loading...</div>
+        {:then token}
+            <span>Current Chat</span>
+            <div class="p-2 text-center">{token} Tokens</div>
+        {/await}
+        {#if $DataBase.promptTemplate}
+            {#await tokenizePreset($DataBase.promptTemplate)}
+                <span>Prompt Template</span>
+                <div class="p-2 text-center">Loading...</div>
+            {:then token}
+                <span>Prompt Template</span>
+                <div class="p-2 text-center">{token} Tokens</div>
+            {/await}
+        {/if}
+    </div>
+    <span class="text-sm text-textcolor2">This is a estimate. The actual token count may be different.</span>
+</Arcodion>
+
+<Arcodion styled name={"Autopilot"}>
+    <div class="flex flex-col p-2 border border-darkborderc rounded-md">
+        {#each autopilot as text}
+            <TextAreaInput bind:value={text} />
+        {/each}
+    </div>
+    <div class="flex justify-end">
+        <button class="text-textcolor2 hover:text-textcolor" on:click={() => {
+            autopilot.pop()
+            autopilot = autopilot
+        }}>
+            <TrashIcon />
+        </button>
+
+        <button class="text-textcolor2 hover:text-textcolor" on:click={() => {
+            autopilot.push('')
+            autopilot = autopilot
+        }}>
+            <PlusIcon />
+        </button>
+
+        <button class="text-textcolor2 hover:text-textcolor" on:click={async () => {
+            const selected = await selectSingleFile([
+                'txt', 'csv', 'json'
+            ])
+            if(!selected){
+                return
+            }
+            const file = new TextDecoder().decode(selected.data)
+            if(selected.name.endsWith('.json')){
+                const parsed = JSON.parse(file)
+                if(Array.isArray(parsed)){
+                    autopilot = parsed
+                }
+            }
+            if(selected.name.endsWith('.csv')){
+                autopilot = file.split('\n').map(x => {
+                    return x.replace(/\r/g, '')
+                        .replace(/\\n/g, '\n')
+                        .replace(/\\t/g, '\t')
+                        .replace(/\\r/g, '\r')
+                })
+            }
+            if(selected.name.endsWith('.txt')){
+                autopilot = file.split('\n')
+            }
+        }}>
+            <FolderUpIcon />
+        </button>
+    </div>
+    <Button className="mt-2" on:click={async () => {
+        if($doingChat){
+            return
+        }
+        for(let i=0;i<autopilot.length;i++){
+            const db = ($DataBase)
+            let currentChar = db.characters[$selectedCharID]
+            let currentChat = currentChar.chats[currentChar.chatPage]
+            currentChat.message.push({
+                role: 'user',
+                data: autopilot[i]
+            })
+            currentChar.chats[currentChar.chatPage] = currentChat
+            db.characters[$selectedCharID] = currentChar
+            if($doingChat){
+                return
+            }
+            currentChar.chats[currentChar.chatPage] = currentChat
+            db.characters[$selectedCharID] = currentChar
+            doingChat.set(false)
+            await sendChat(i);
+            currentChar = db.characters[$selectedCharID]
+            currentChat = currentChar.chats[currentChar.chatPage]
+        }
+        doingChat.set(false)
+    }}>Run</Button>
+</Arcodion>
+<Button className="mt-2" on:click={() => {
     alertMd(getRequestLog())
 }}>Request Log</Button>
\ No newline at end of file
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index 7afe4fa0..1a8cf376 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -1,12 +1,13 @@
 import type { Tiktoken } from "@dqbd/tiktoken";
 import type { Tokenizer } from "@mlc-ai/web-tokenizers";
-import { DataBase, type character } from "./storage/database";
+import { DataBase, type groupChat, type character, type Chat } from "./storage/database";
 import { get } from "svelte/store";
 import type { MultiModal, OpenAIChat } from "./process";
 import { supportsInlayImage } from "./process/files/image";
 import { risuChatParser } from "./parser";
 import { tokenizeGGUFModel } from "./process/models/local";
 import { globalFetch } from "./storage/globalApi";
+import { CurrentCharacter } from "./stores";
 
 
 export const tokenizerList = [
@@ -342,4 +343,57 @@ export async function strongBan(data:string, bias:{[key:number]:number}) {
     }
     localStorage.setItem('strongBan_' + data, JSON.stringify(bias))
     return bias
+}
+
+export async function getCharToken(char?:character|groupChat|null){
+    let persistant = 0
+    let dynamic = 0
+
+    if(!char){
+        const c = get(CurrentCharacter)
+        char = c
+    }
+    if(char.type === 'group'){
+        return {persistant:0, dynamic:0}
+    }
+
+    const basicTokenize = async (data:string) => {
+        data = data.replace(/{{char}}/g, char.name).replace(/<char>/g, char.name)
+        return await tokenize(data)
+    }
+
+    persistant += await basicTokenize(char.desc)
+    persistant += await basicTokenize(char.personality ?? '')
+    persistant += await basicTokenize(char.scenario ?? '')
+    for(const lore of char.globalLore){
+        let cont = lore.content.split('\n').filter((line) => {
+            if(line.startsWith('@@')){
+                return false
+            }
+            if(line === ''){
+                return false
+            }
+            return true
+        }).join('\n')
+        dynamic += await basicTokenize(cont)
+    }
+
+    return {persistant, dynamic}
+}
+
+export async function getChatToken(chat:Chat) {
+    let persistant = 0
+
+    const chatTokenizer = new ChatTokenizer(0, 'name')
+    const chatf = chat.message.map((d) => {
+        return {
+            role: d.role === 'user' ? 'user' : 'assistant',
+            content: d.data,
+        } as OpenAIChat
+    })
+    for(const chat of chatf){
+        persistant += await chatTokenizer.tokenizeChat(chat)
+    }
+
+    return persistant
 }
\ No newline at end of file