gptq processing

.gitignore

@@ -30,4 +30,9 @@ src-others/
 /src-taurl/gen/
 /build/
 pycache/
-test.ts
+test.ts
+
+
+
+vite.config.js.timestamp-*
+vite.config.ts.timestamp-*

src/lib/UI/ModelList.svelte

@@ -115,16 +115,9 @@
                 {/if}
             </Arcodion>
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('reverse_proxy')}}>Reverse Proxy</button>
-            <!-- <button class="hover:bg-selected px-6 py-2 text-lg" on:click={async () => {
+            <button class="hover:bg-selected px-6 py-2 text-lg" on:click={async () => {
-                const res = (await checkLocalModel())
+                changeModel('local_gptq')
-                if(res === 'success'){
+            }}>Local Model GPTQ</button>
-                    changeModel('local_gptq')
-                }
-                else{
-                    alertError("python 3.10, cuda 11.7 and git must be installed to run it. " + res)
-                }
-                // changeModel('local_gptq')
-            }}>Local Model GPTQ</button> -->
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('textgen_webui')}}>Oobabooga WebUI</button>
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('mancer')}}>Mancer</button>
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('palm2')}}>Google PaLM2</button>

src/ts/alert.ts

@@ -79,7 +79,7 @@ export function alertToast(msg:string){
 }
 
 export function alertWait(msg:string){
-
+    console.log(msg)
     alertStore.set({
         'type': 'wait',
         'msg': msg

src/ts/process/models/local.ts

@@ -21,7 +21,8 @@ export async function startLocalModelServer(){
 }
 
 export async function checkLocalServerInstalled() {
-    const p = await path.join(await path.appDataDir(), 'local_server')
+    console.log(await path.appDataDir())
+    const p = await path.join(await path.appDataDir(), 'local_server', 'key.txt')
     return await exists(p)
 }
 
@@ -76,6 +77,17 @@ interface LocalGeneratorItem {
     max_new_tokens?: number;
 }
 
+export async function checkServerRunning() {
+    try {
+        console.log("checking server")
+        const res = await fetch("http://localhost:7239/")
+        console.log(res)
+        return res.ok   
+    } catch (error) {
+        return false
+    }
+}
+
 
 export async function loadExllamaFull(){
 
@@ -89,17 +101,20 @@ export async function loadExllamaFull(){
         }
         while(true){
             //check is server is running by fetching the status
-            try {
+            if(await checkLocalServerInstalled()){
-                const res = await globalFetch("http://localhost:7239/")
+                await sleep(1000)
-                if(res.ok){
+                try {
-                    break
+                    const res = await fetch("http://localhost:7239/")
-                }
+                    if(res.status === 200){
-            } catch (error) {}
+                        break
+                    }
+                } catch (error) {}
+            }
             await sleep(1000)
         }
 
         const body:LocalLoaderItem = {
-            dir: "exllama",
+            dir: "C:\\Users\\blueb\\Downloads\\model",
         }
     
         alertWait("Loading Local Model")
@@ -117,6 +132,10 @@ export async function loadExllamaFull(){
 export async function runLocalModel(prompt:string){
     const db = get(DataBase)
 
+    if(!serverRunning){
+        await loadExllamaFull()
+    }
+
     const body:LocalGeneratorItem = {
         prompt: prompt,
         temperature: db.temperature,
@@ -126,6 +145,8 @@ export async function runLocalModel(prompt:string){
         max_new_tokens: db.maxResponse
     }
 
+    console.log("generating")
+
     const gen = await globalFetch("http://localhost:7239/generate/", {
         body: body
     })

src/ts/process/request.ts

@@ -10,6 +10,7 @@ import { createDeep } from "./deepai";
 import { hubURL } from "../characterCards";
 import { NovelAIBadWordIds, stringlizeNAIChat } from "./models/nai";
 import { tokenizeNum } from "../tokenizer";
+import { runLocalModel } from "./models/local";
 
 interface requestDataArgument{
     formated: OpenAIChat[]
@@ -957,6 +958,13 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 }
 
 
+            }
+            if(aiModel.startsWith('local_')){
+                console.log('running local model')
+                const suggesting = model === "submodel"
+                const proompt = stringlizeChatOba(formated, currentChar.name, suggesting, arg.continue)
+                const stopStrings = getStopStrings(suggesting)
+                await runLocalModel(proompt)
             }
             return {
                 type: 'fail',