[feat] added streaming (#71)

package.json

@@ -33,7 +33,8 @@
     "rollup": "^3.21.3",
     "showdown": "^2.1.0",
     "sweetalert2": "^11.7.3",
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "web-streams-polyfill": "^3.2.1"
   },
   "devDependencies": {
     "@sveltejs/vite-plugin-svelte": "^2.0.0",

pnpm-lock.yaml

@@ -48,6 +48,7 @@ specifiers:
   vite: ^4.2.1
   vite-plugin-top-level-await: ^1.3.0
   vite-plugin-wasm: ^3.2.2
+  web-streams-polyfill: ^3.2.1
 
 dependencies:
   '@dqbd/tiktoken': 1.0.4
@@ -71,6 +72,7 @@ dependencies:
   showdown: 2.1.0
   sweetalert2: 11.7.3
   uuid: 9.0.0
+  web-streams-polyfill: 3.2.1
 
 devDependencies:
   '@sveltejs/vite-plugin-svelte': 2.0.4_svelte@3.58.0+vite@4.2.1
@@ -2275,6 +2277,11 @@ packages:
       xml-name-validator: 4.0.0
     dev: false
 
+  /web-streams-polyfill/3.2.1:
+    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
+    engines: {node: '>= 8'}
+    dev: false
+
   /webidl-conversions/7.0.0:
     resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==}
     engines: {node: '>=12'}

src-tauri/tauri.conf.json

@@ -8,7 +8,7 @@
   },
   "package": {
     "productName": "RisuAI",
-    "version": "1.10.1"
+    "version": "1.11.0"
   },
   "tauri": {
     "allowlist": {

src/lang/en.ts

@@ -248,5 +248,5 @@ export const languageEnglish = {
     useExperimental: "Able Experimental Features",
     showMemoryLimit: "Show Memory Limit",
     roundIcons: "Round Icons",
-    useStreaming: "Use Streaming"
+    streaming: "Streaming"
 }

src/lang/ko.ts

@@ -225,5 +225,14 @@ export const languageKorean = {
     selective: "멀티플 키",
     SecondaryKeys: '두번째 키',
     useGlobalSettings: "글로벌 설정 사용",
-    recursiveScanning: "재귀 검색"
+    recursiveScanning: "재귀 검색",
+    creator: "제작자",
+    CharVersion: "캐릭터 버전",
+    Speech: "음성",
+    ToggleSuperMemory: "SupaMemory 토글",
+    SuperMemory:"SupaMemory",
+    useExperimental: "실험적 요소 보이기",
+    showMemoryLimit: "기억 한계치 보이기",
+    roundIcons: "둥근 아이콘",
+    streaming: "스트리밍"
 }

src/lib/SideBars/Settings.svelte

@@ -121,7 +121,11 @@
     {/if}
     {#if $DataBase.aiModel === 'gpt35' || $DataBase.aiModel === 'gpt4' || $DataBase.subModel === 'gpt4' || $DataBase.subModel === 'gpt35'}
         <span class="text-neutral-200">OpenAI {language.apiKey} <Help key="oaiapikey"/></span>
-        <input class="text-neutral-200 mb-4 p-2 bg-transparent input-text focus:bg-selected text-sm" placeholder="sk-XXXXXXXXXXXXXXXXXXXX" bind:value={$DataBase.openAIKey}>
+        <input class="text-neutral-200 p-2 bg-transparent input-text focus:bg-selected text-sm" placeholder="sk-XXXXXXXXXXXXXXXXXXXX" bind:value={$DataBase.openAIKey}>
+        <div class="flex items-center mt-2 mb-4">
+            <Check bind:check={$DataBase.useStreaming}/>
+            <span>OpenAI {language.streaming}</span>
+        </div>
     {/if}
     {#if $DataBase.aiModel === 'custom'}
         <span class="text-neutral-200 mt-2">{language.plugin}</span>

src/main.ts

@@ -1,12 +1,17 @@
 import "./styles.css";
 import App from "./App.svelte";
 import { loadData } from "./ts/globalApi";
-
+import { ReadableStream, WritableStream, TransformStream } from "web-streams-polyfill/ponyfill/es2018";
 import { Buffer as BufferPolyfill } from 'buffer'
 import { initHotkey } from "./ts/hotkey";
+
+//Polyfills
 declare var Buffer: typeof BufferPolyfill;
 globalThis.Buffer = BufferPolyfill
-
+//@ts-ignore
+globalThis.WritableStream = globalThis.WritableStream ?? WritableStream
+globalThis.ReadableStream = globalThis.ReadableStream ?? ReadableStream
+globalThis.TransformStream = globalThis.TransformStream ?? TransformStream
 
 const app = new App({
   target: document.getElementById("app"),

src/ts/database.ts

@@ -7,7 +7,7 @@ import { cloneDeep } from 'lodash';
 
 export const DataBase = writable({} as any as Database)
 export const loadedStore = writable(false)
-export let appVer = '1.10.1'
+export let appVer = '1.11.0'
 
 
 export function setDatabase(data:Database){

src/ts/process/index.ts

@@ -328,7 +328,8 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
     const req = await requestChatData({
         formated: formated,
         bias: bias,
-        currentChar: currentChar
+        currentChar: currentChar,
+        useStreaming: true
     }, 'model')
 
     let result = ''
@@ -339,7 +340,23 @@ export async function sendChat(chatProcessIndex = -1):Promise<boolean> {
         return false
     }
     else if(req.type === 'streaming'){
-        
+        const reader = req.result.getReader()
+        const msgIndex = db.characters[selectedChar].chats[selectedChat].message.length
+        db.characters[selectedChar].chats[selectedChat].message.push({
+            role: 'char',
+            data: "",
+            saying: currentChar.chaId
+        })
+        while(true){
+            const readed = (await reader.read())
+            if(readed.value){
+                db.characters[selectedChar].chats[selectedChat].message[msgIndex].data =readed.value
+                setDatabase(db)
+            }
+            if(readed.done){
+                break
+            }   
+        }
     }
     else{
         const result2 = processScriptFull(currentChar, reformatContent(req.result), 'editoutput')

src/ts/process/request.ts

@@ -13,7 +13,8 @@ interface requestDataArgument{
     temperature?: number
     maxTokens?:number
     PresensePenalty?: number
-    frequencyPenalty?: number
+    frequencyPenalty?: number,
+    useStreaming?:boolean
 }
 
 type requestDataResponse = {
@@ -21,7 +22,7 @@ type requestDataResponse = {
     result: string
 }|{
     type: "streaming",
-    result: ReadableStreamDefaultReader<Uint8Array>
+    result: ReadableStream<string>
 }
 
 export async function requestChatData(arg:requestDataArgument, model:'model'|'submodel'):Promise<requestDataResponse> {
@@ -60,6 +61,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 presence_penalty: arg.PresensePenalty ?? (db.PresensePenalty / 100),
                 frequency_penalty: arg.frequencyPenalty ?? (db.frequencyPenalty / 100),
                 logit_bias: bias,
+                stream: false
             })
 
             let replacerURL = replacer === '' ? 'https://api.openai.com/v1/chat/completions' : replacer
@@ -71,19 +73,59 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
                 replacerURL += 'chat/completions'
             }
 
-            if(db.useStreaming){
+            if(db.useStreaming && arg.useStreaming){
+                body.stream = true
                 const da = await fetch(replacerURL, {
                     body: JSON.stringify(body),
+                    method: "POST",
                     headers: {
-                        "Authorization": "Bearer " + db.openAIKey
+                        "Authorization": "Bearer " + db.openAIKey,
+                        "Content-Type": "application/json"
                     },
                 })
 
-                const reader = da.body.getReader()
+                if(da.status !== 200){
+                    return {
+                        type: "fail",
+                        result: await da.text()
+                    }
+                }
+
+                let dataUint = new Uint8Array([])
+
+                const transtream = new TransformStream<Uint8Array, string>(  {
+                    async transform(chunk, control) {
+                        dataUint = Buffer.from(new Uint8Array([...dataUint, ...chunk]))
+                        try {
+                            const datas = dataUint.toString().split('\n')
+                            let readed = ''
+                            for(const data of datas){
+                                if(data.startsWith("data: ")){
+                                    try {
+                                        const rawChunk = data.replace("data: ", "")
+                                        if(rawChunk === "[DONE]"){
+                                            control.enqueue(readed)
+                                            return
+                                        }
+                                        const chunk = JSON.parse(rawChunk).choices[0].delta.content
+                                        if(chunk){
+                                            readed += chunk
+                                        }
+                                    } catch (error) {}
+                                }
+                            }
+                            control.enqueue(readed)
+                        } catch (error) {
+                            
+                        }
+                    }
+                },)
+
+                da.body.pipeTo(transtream.writable)
 
                 return {
                     type: 'streaming',
-                    result: reader
+                    result: transtream.readable
                 }
             }
 

src/ts/update.ts

@@ -46,7 +46,7 @@ function versionStringToNumber(versionString:string):number {
     return Number(
       versionString
         .split(".")
-        .map((component) => component.padStart(2, "0"))
+        .map((component) => component.padStart(4, "0"))
         .join("")
     );
 }

version.json

@@ -1 +1 @@
-{"version":"1.10.1"}
+{"version":"1.11.0"}