From fe31610ca084d4344ec1af9e851e417e8ca5022e Mon Sep 17 00:00:00 2001
From: kwaroran <amasio1234@proton.me>
Date: Wed, 6 Mar 2024 22:14:24 +0900
Subject: [PATCH] Add claude streaming & remove unused parameters

---
 package.json                             |  1 +
 pnpm-lock.yaml                           |  8 +++
 src/lib/Setting/Pages/BotSettings.svelte | 13 ++++-
 src/ts/process/request.ts                | 66 ++++++++++++++++++++++++
 4 files changed, 86 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 3acae88a..0ff4055c 100644
--- a/package.json
+++ b/package.json
@@ -36,6 +36,7 @@
     "cors": "^2.8.5",
     "crc": "^4.3.2",
     "dompurify": "^3.0.8",
+    "eventsource-parser": "^1.1.2",
     "exifr": "^7.1.3",
     "express": "^4.18.2",
     "fflate": "^0.8.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index b1cec9bb..1da8a385 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -65,6 +65,9 @@ dependencies:
   dompurify:
     specifier: ^3.0.8
     version: 3.0.8
+  eventsource-parser:
+    specifier: ^1.1.2
+    version: 1.1.2
   exifr:
     specifier: ^7.1.3
     version: 7.1.3
@@ -2923,6 +2926,11 @@ packages:
     resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
     dev: false
 
+  /eventsource-parser@1.1.2:
+    resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==}
+    engines: {node: '>=14.18'}
+    dev: false
+
   /execa@5.1.1:
     resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==}
     engines: {node: '>=10'}
diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte
index f0168737..80aa9195 100644
--- a/src/lib/Setting/Pages/BotSettings.svelte
+++ b/src/lib/Setting/Pages/BotSettings.svelte
@@ -151,7 +151,7 @@
     <span class="text-textcolor">Claude {language.apiKey}</span>
     <TextInput marginBottom={true} size={"sm"} placeholder="..." bind:value={$DataBase.claudeAPIKey}/>
     {#if $DataBase.useExperimental}
-        <Check name="Claude Aws" bind:check={$DataBase.claudeAws}> <Help key="experimental" /></Check>
+        <Check name="AWS Claude" bind:check={$DataBase.claudeAws}> <Help key="experimental" /></Check>
     {/if}
 {/if}
 {#if $DataBase.aiModel.startsWith('mistral') || $DataBase.subModel.startsWith('mistral')}
@@ -235,7 +235,7 @@
     <TextInput marginBottom={false} size={"sm"} bind:value={$DataBase.openAIKey} placeholder="sk-XXXXXXXXXXXXXXXXXXXX"/>
 
 {/if}
-{#if $DataBase.aiModel.startsWith('gpt') || $DataBase.aiModel === 'reverse_proxy' || $DataBase.aiModel === 'openrouter'}
+{#if $DataBase.aiModel.startsWith('gpt') || $DataBase.aiModel === 'reverse_proxy' || $DataBase.aiModel === 'openrouter' || $DataBase.aiModel.startsWith('claude-3')}
     <div class="flex items-center mt-2 mb-4">
         <Check bind:check={$DataBase.useStreaming} name={`Response ${language.streaming}`}/>
     </div>
@@ -501,7 +501,16 @@
     <span class="text-textcolor">Typical P</span>
     <SliderInput min={0} max={1} step={0.01} bind:value={$DataBase.ainconfig.typical_p}/>
     <span class="text-textcolor2 mb-6 text-sm">{($DataBase.ainconfig.typical_p).toFixed(2)}</span>
+{:else if $DataBase.aiModel.startsWith('claude')}
+    <span class="text-textcolor">Top P <Help key="topP"/></span>
+    <SliderInput min={0} max={1} step={0.01} bind:value={$DataBase.top_p}/>
+    <span class="text-textcolor2 mb-6 text-sm">{($DataBase.top_p).toFixed(2)}</span>
+    <span class="text-textcolor mt-2">{language.autoSuggest} <Help key="autoSuggest"/></span>
+    <TextAreaInput height="20" autocomplete="off" bind:value={$DataBase.autoSuggestPrompt} />
+    <span class="text-textcolor2 mb-6 text-sm">{tokens.autoSuggest} {language.tokens}</span>
 {:else}
+
+
     <span class="text-textcolor">Top P <Help key="topP"/></span>
     <SliderInput min={0} max={1} step={0.01} bind:value={$DataBase.top_p}/>
     <span class="text-textcolor2 mb-6 text-sm">{($DataBase.top_p).toFixed(2)}</span>
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 674bfe4a..8eb3b001 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -22,6 +22,7 @@ import { OaifixBias } from "../plugins/fixer";
 import { Capacitor } from "@capacitor/core";
 import { getFreeOpenRouterModel } from "../model/openrouter";
 import { runTransformers } from "./embedding/transformers";
+import {createParser, type ParsedEvent, type ReconnectInterval} from 'eventsource-parser'
 
 
 
@@ -1490,6 +1491,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                     temperature: temperature,
                     top_p: db.top_p,
                     top_k: db.top_k,
+                    stream: db.useStreaming ?? false
                 }
 
                 if(systemPrompt === ''){
@@ -1578,6 +1580,70 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                     }
                 }
 
+                if(db.useStreaming){
+                    
+                    const res = await fetchNative(replacerURL, {
+                        body: JSON.stringify(body),
+                        headers: {
+                            "Content-Type": "application/json",
+                            "x-api-key": apiKey,
+                            "anthropic-version": "2023-06-01",
+                            "accept": "application/json",
+                        },
+                        method: "POST"
+                    })
+
+                    if(res.status !== 200){
+                        return {
+                            type: 'fail',
+                            result: await textifyReadableStream(res.body)
+                        }
+                    }
+
+
+                    const stream = new ReadableStream<StreamResponseChunk>({
+                        async start(controller){
+                            let text = ''
+                            const decoder = new TextDecoder()
+                            const parser = createParser((e) => {
+                                if(e.type === 'event'){
+                                    switch(e.event){
+                                        case 'content_block_delta': {
+                                            if(e.data){
+                                                text += JSON.parse(e.data).delta?.text
+                                                controller.enqueue({
+                                                    "0": text
+                                                })
+                                            }
+                                            break
+                                        }
+        
+                                    }
+                                }
+                                if(e.type === 'reconnect-interval'){
+                                    //TODO: handle reconnect interval
+                                }
+                            })
+                            const reader = res.body.getReader()
+                            while(true){
+                                const {done, value} = await reader.read()
+                                if(done){
+                                    break
+                                }
+                                parser.feed(decoder.decode(value))
+                            }
+                            controller.close()
+                        },
+                        cancel(){
+                        }
+                    })
+
+                    return {
+                        type: 'streaming',
+                        result: stream
+                    }
+
+                }
                 const res = await globalFetch(replacerURL, {
                     body: body,
                     headers: {