Refactor code for local models

src-tauri/src-python/main.py

@@ -66,17 +66,39 @@ def stream_chat_llamacpp(item:LlamaItem):
     chunks = app.llm.create_completion(
         prompt = item.prompt,
         temperature = item.temperature,
-        top_p = item.top_p,
+        # top_p = item.top_p,
-        top_k = item.top_k,
+        # top_k = item.top_k,
-        max_tokens = item.max_tokens,
+        # max_tokens = item.max_tokens,
-        presence_penalty = item.presence_penalty,
+        # presence_penalty = item.presence_penalty,
-        frequency_penalty = item.frequency_penalty,
+        # frequency_penalty = item.frequency_penalty,
-        repeat_penalty = item.repeat_penalty,
+        # repeat_penalty = item.repeat_penalty,
-        stop=item.stop,
+        # stop=item.stop,
-        stream=True
+        stream=False,
     )
+    if(type(chunks) == str):
+        print(chunks, end="")
+        yield chunks
+        return
+    if(type(chunks) == bytes):
+        print(chunks.decode('utf-8'), end="")
+        yield chunks.decode('utf-8')
+        return
+    if(type(chunks) == dict and "choices" in chunks):
+        print(chunks["choices"][0]["text"], end="")
+        yield chunks["choices"][0]["text"]
+        return
+
     for chunk in chunks:
+        if(type(chunk) == str):
+            print(chunk, end="")
+            yield chunk
+            continue
+        if(type(chunk) == bytes):
+            print(chunk.decode('utf-8'), end="")
+            yield chunk.decode('utf-8')
+            continue
         cont:CompletionChunk  = chunk
+        print(cont)
         encoded = cont["choices"][0]["text"]
         print(encoded, end="")
         yield encoded

src/lib/UI/ModelList.svelte

@@ -5,6 +5,8 @@
     import { language } from "src/lang";
     import Help from "../Others/Help.svelte";
     import CheckInput from "./GUI/CheckInput.svelte";
+    import { isTauri } from "src/ts/storage/globalApi";
+    import {open} from '@tauri-apps/api/dialog'
 
     export let value = ""
     export let onChange: (v:string) => void = (v) => {}
@@ -89,6 +91,10 @@
                     const split = name.split(":::")
                     return `${split[1]}`
                 }
+                if(name.startsWith('local_')){
+                    const realName = name.replace('local_', '').split(/(\\|\/)/g).at(-1)
+                    return `GGUF ${realName}`
+                }
                 return name
         }
     }
@@ -143,9 +149,17 @@
                 {/if}
             </Arcodion>
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('reverse_proxy')}}>Reverse Proxy</button>
-            {#if import.meta.env.DEV}
+            {#if $DataBase.tpo && isTauri}
                 <button class="hover:bg-selected px-6 py-2 text-lg" on:click={async () => {
-                    changeModel('local_') // TODO: Fix this
+                    const selected = await open({
+                        filters: [{
+                            name: 'Model File',
+                            extensions: ['gguf']
+                        }]
+                    });
+                    if(selected){
+                        changeModel('local_' + selected)
+                    }
                 }}>Local GGUF Model <Help key="experimental"/> </button>
             {/if}
             <button class="hover:bg-selected px-6 py-2 text-lg" on:click={() => {changeModel('ooba')}}>Oobabooga</button>

src/ts/process/request.ts

@@ -1685,6 +1685,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 const suggesting = model === "submodel"
                 const proompt = stringlizeChatOba(formated, currentChar.name, suggesting, arg.continue)
                 const stopStrings = getStopStrings(suggesting)
+                console.log(stopStrings)
                 const modelPath = aiModel.replace('local_', '')
                 const res = await runGGUFModel({
                     prompt: proompt,