diff --git a/package.json b/package.json index 48e017f5..9931e119 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "mobile-drag-drop": "3.0.0-rc.0", "msgpackr": "^1.10.1", "node-html-parser": "^6.1.12", + "ollama": "^0.5.0", "pdfjs-dist": "^4.0.379", "peerjs": "^1.5.2", "png-chunk-text": "^1.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b78b2226..332197b1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -122,6 +122,9 @@ dependencies: node-html-parser: specifier: ^6.1.12 version: 6.1.12 + ollama: + specifier: ^0.5.0 + version: 0.5.0 pdfjs-dist: specifier: ^4.0.379 version: 4.0.379 @@ -4390,6 +4393,12 @@ packages: resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} dev: false + /ollama@0.5.0: + resolution: {integrity: sha512-CRtRzsho210EGdK52GrUMohA2pU+7NbgEaBG3DcYeRmvQthDO7E2LHOkLlUUeaYUlNmEd8icbjC02ug9meSYnw==} + dependencies: + whatwg-fetch: 3.6.20 + dev: false + /on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} @@ -6100,6 +6109,10 @@ packages: iconv-lite: 0.6.3 dev: false + /whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + dev: false + /whatwg-mimetype@4.0.0: resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} engines: {node: '>=18'} diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte index 16ed3a84..e9557e88 100644 --- a/src/lib/Setting/Pages/BotSettings.svelte +++ b/src/lib/Setting/Pages/BotSettings.svelte @@ -203,6 +203,13 @@ Risu {language.apiKey} {/if} +{#if $DataBase.aiModel === 'ollama-hosted'} + Ollama URL + + + Ollama Model + +{/if} {#if $DataBase.aiModel === 'openrouter' || $DataBase.subModel === 'openrouter'} Openrouter Key diff --git a/src/lib/UI/ModelList.svelte b/src/lib/UI/ModelList.svelte index c10f5e28..ab0152e3 100644 --- a/src/lib/UI/ModelList.svelte +++ b/src/lib/UI/ModelList.svelte @@ -132,6 +132,7 @@ {/each} {/await} + {#if showUnrec} diff --git a/src/ts/model/names.ts b/src/ts/model/names.ts index 188ef173..a87a3aca 100644 --- a/src/ts/model/names.ts +++ b/src/ts/model/names.ts @@ -95,6 +95,8 @@ export function getModelName(name:string){ return 'GPT-4 Turbo (20240409)' case 'gemini-1.5-pro-latest': return 'Gemini 1.5 Pro' + case 'ollama-hosted': + return 'Ollama' default: if(name.startsWith("horde:::")){ const split = name.split(":::") diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 4faadb67..e654dea9 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -23,6 +23,7 @@ import { Capacitor } from "@capacitor/core"; import { getFreeOpenRouterModel } from "../model/openrouter"; import { runTransformers } from "./transformers"; import {createParser, type ParsedEvent, type ReconnectInterval} from 'eventsource-parser' +import {Ollama} from 'ollama/dist/browser.mjs' @@ -1494,6 +1495,38 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' result: resp.replace(/\\n/g, '\n') } } + case 'ollama-hosted':{ + const ollama = new Ollama({host: 'http://localhost:11434'}) + + const response = await ollama.chat({ + model: db.ollamaModel, + messages: formated.map((v) => { + return { + role: v.role, + content: v.content + } + }).filter((v) => { + return v.role === 'assistant' || v.role === 'user' || v.role === 'system' + }), + stream: true + }) + + const readableStream = new ReadableStream({ + async start(controller){ + for await(const chunk of response){ + controller.enqueue({ + "0": chunk.message.content + }) + } + controller.close() + } + }) + + return { + type: 'streaming', + result: readableStream + } + } default:{ if(raiModel.startsWith('claude-3')){ let replacerURL = (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.anthropic.com/v1/messages') diff --git a/src/ts/storage/database.ts b/src/ts/storage/database.ts index ecd3c272..272e48a8 100644 --- a/src/ts/storage/database.ts +++ b/src/ts/storage/database.ts @@ -383,6 +383,8 @@ export function setDatabase(data:Database){ data.heightMode ??= 'normal' data.antiClaudeOverload ??= false data.maxSupaChunkSize ??= 1200 + data.ollamaURL ??= '' + data.ollamaModel ??= '' changeLanguage(data.language) DataBase.set(data) @@ -618,6 +620,8 @@ export interface Database{ noWaitForTranslate:boolean antiClaudeOverload:boolean maxSupaChunkSize:number + ollamaURL:string + ollamaModel:string } export interface customscript{