diff --git a/package.json b/package.json
index 3acae88a..0ff4055c 100644
--- a/package.json
+++ b/package.json
@@ -36,6 +36,7 @@
"cors": "^2.8.5",
"crc": "^4.3.2",
"dompurify": "^3.0.8",
+ "eventsource-parser": "^1.1.2",
"exifr": "^7.1.3",
"express": "^4.18.2",
"fflate": "^0.8.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index b1cec9bb..1da8a385 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -65,6 +65,9 @@ dependencies:
dompurify:
specifier: ^3.0.8
version: 3.0.8
+ eventsource-parser:
+ specifier: ^1.1.2
+ version: 1.1.2
exifr:
specifier: ^7.1.3
version: 7.1.3
@@ -2923,6 +2926,11 @@ packages:
resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
dev: false
+ /eventsource-parser@1.1.2:
+ resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==}
+ engines: {node: '>=14.18'}
+ dev: false
+
/execa@5.1.1:
resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==}
engines: {node: '>=10'}
diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte
index f0168737..80aa9195 100644
--- a/src/lib/Setting/Pages/BotSettings.svelte
+++ b/src/lib/Setting/Pages/BotSettings.svelte
@@ -151,7 +151,7 @@
Claude {language.apiKey}
{#if $DataBase.useExperimental}
-
+
{/if}
{/if}
{#if $DataBase.aiModel.startsWith('mistral') || $DataBase.subModel.startsWith('mistral')}
@@ -235,7 +235,7 @@
{/if}
-{#if $DataBase.aiModel.startsWith('gpt') || $DataBase.aiModel === 'reverse_proxy' || $DataBase.aiModel === 'openrouter'}
+{#if $DataBase.aiModel.startsWith('gpt') || $DataBase.aiModel === 'reverse_proxy' || $DataBase.aiModel === 'openrouter' || $DataBase.aiModel.startsWith('claude-3')}
@@ -501,7 +501,16 @@
Typical P
{($DataBase.ainconfig.typical_p).toFixed(2)}
+{:else if $DataBase.aiModel.startsWith('claude')}
+ Top P
+
+ {($DataBase.top_p).toFixed(2)}
+ {language.autoSuggest}
+
+ {tokens.autoSuggest} {language.tokens}
{:else}
+
+
Top P
{($DataBase.top_p).toFixed(2)}
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 674bfe4a..8eb3b001 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -22,6 +22,7 @@ import { OaifixBias } from "../plugins/fixer";
import { Capacitor } from "@capacitor/core";
import { getFreeOpenRouterModel } from "../model/openrouter";
import { runTransformers } from "./embedding/transformers";
+import {createParser, type ParsedEvent, type ReconnectInterval} from 'eventsource-parser'
@@ -1490,6 +1491,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
temperature: temperature,
top_p: db.top_p,
top_k: db.top_k,
+ stream: db.useStreaming ?? false
}
if(systemPrompt === ''){
@@ -1578,6 +1580,70 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
}
}
+ if(db.useStreaming){
+
+ const res = await fetchNative(replacerURL, {
+ body: JSON.stringify(body),
+ headers: {
+ "Content-Type": "application/json",
+ "x-api-key": apiKey,
+ "anthropic-version": "2023-06-01",
+ "accept": "application/json",
+ },
+ method: "POST"
+ })
+
+ if(res.status !== 200){
+ return {
+ type: 'fail',
+ result: await textifyReadableStream(res.body)
+ }
+ }
+
+
+ const stream = new ReadableStream({
+ async start(controller){
+ let text = ''
+ const decoder = new TextDecoder()
+ const parser = createParser((e) => {
+ if(e.type === 'event'){
+ switch(e.event){
+ case 'content_block_delta': {
+ if(e.data){
+ text += JSON.parse(e.data).delta?.text
+ controller.enqueue({
+ "0": text
+ })
+ }
+ break
+ }
+
+ }
+ }
+ if(e.type === 'reconnect-interval'){
+ //TODO: handle reconnect interval
+ }
+ })
+ const reader = res.body.getReader()
+ while(true){
+ const {done, value} = await reader.read()
+ if(done){
+ break
+ }
+ parser.feed(decoder.decode(value))
+ }
+ controller.close()
+ },
+ cancel(){
+ }
+ })
+
+ return {
+ type: 'streaming',
+ result: stream
+ }
+
+ }
const res = await globalFetch(replacerURL, {
body: body,
headers: {