import { get } from "svelte/store"; import type { OpenAIChat, OpenAIChatFull } from "."; import { DataBase, setDatabase, type character } from "../storage/database"; import { pluginProcess } from "../plugins/plugins"; import { language } from "../../lang"; import { stringlizeAINChat, stringlizeChat, stringlizeChatOba, getStopStrings, unstringlizeAIN, unstringlizeChat } from "./stringlize"; import { addFetchLog, globalFetch, isNodeServer, isTauri } from "../storage/globalApi"; import { sleep } from "../util"; import { createDeep } from "./deepai"; import { hubURL } from "../characterCards"; import { NovelAIBadWordIds, stringlizeNAIChat } from "./models/nai"; import { strongBan, tokenizeNum } from "../tokenizer"; import { runGGUFModel } from "./models/local"; import { risuChatParser } from "../parser"; import { SignatureV4 } from "@smithy/signature-v4"; import { HttpRequest } from "@smithy/protocol-http"; import { Sha256 } from "@aws-crypto/sha256-js"; import { v4 } from "uuid"; import { cloneDeep } from "lodash"; import { supportsInlayImage } from "./files/image"; import { OaifixBias } from "../plugins/fixer"; import { Capacitor } from "@capacitor/core"; import { getFreeOpenRouterModel } from "../model/openrouter"; import { runTransformers } from "./embedding/transformers"; interface requestDataArgument{ formated: OpenAIChat[] bias: {[key:number]:number} biasString?: [string,number][] currentChar?: character temperature?: number maxTokens?:number PresensePenalty?: number frequencyPenalty?: number, useStreaming?:boolean isGroupChat?:boolean useEmotion?:boolean continue?:boolean } type requestDataResponse = { type: 'success'|'fail' result: string noRetry?: boolean, special?: { emotion?: string } }|{ type: "streaming", result: ReadableStream, special?: { emotion?: string } }|{ type: "multiline", result: ['user'|'char',string][], special?: { emotion?: string } } interface StreamResponseChunk{[key:string]:string} interface OaiFunctions { name: string; description: string; parameters: { type: string; properties: { [key:string]: { type: string; enum: string[] }; }; required: string[]; }; } export async function requestChatData(arg:requestDataArgument, model:'model'|'submodel', abortSignal:AbortSignal=null):Promise { const db = get(DataBase) let trys = 0 while(true){ const da = await requestChatDataMain(arg, model, abortSignal) if(da.type !== 'fail' || da.noRetry){ return da } trys += 1 if(trys > db.requestRetrys){ return da } } } interface OpenAITextContents { type: 'text' text: string } interface OpenAIImageContents { type: 'image' image_url: { url: string detail: string } } type OpenAIContents = OpenAITextContents|OpenAIImageContents export interface OpenAIChatExtra { role: 'system'|'user'|'assistant'|'function' content: string|OpenAIContents[] memo?:string name?:string removable?:boolean attr?:string[] } export async function requestChatDataMain(arg:requestDataArgument, model:'model'|'submodel', abortSignal:AbortSignal=null):Promise { const db = get(DataBase) let formated = cloneDeep(arg.formated) let maxTokens = arg.maxTokens ??db.maxResponse let temperature = arg.temperature ?? (db.temperature / 100) let bias = arg.bias let currentChar = arg.currentChar arg.continue = arg.continue ?? false let biasString = arg.biasString ?? [] const aiModel = (model === 'model' || (!db.advancedBotSettings)) ? db.aiModel : db.subModel let raiModel = aiModel if(aiModel === 'reverse_proxy'){ if(db.proxyRequestModel.startsWith('claude')){ raiModel = db.proxyRequestModel } if(db.forceProxyAsOpenAI){ raiModel = 'reverse_proxy' } } console.log(formated) switch(raiModel){ case 'gpt35': case 'gpt35_0613': case 'gpt35_16k': case 'gpt35_16k_0613': case 'gpt4': case 'gpt45': case 'gpt4_32k': case 'gpt4_0613': case 'gpt4_32k_0613': case 'gpt4_1106': case 'gpt4_0125': case 'gpt35_0125': case 'gpt35_1106': case 'gpt35_0301': case 'gpt4_0314': case 'gptvi4_1106': case 'openrouter': case 'mistral-tiny': case 'mistral-small': case 'mistral-medium': case 'mistral-small-latest': case 'mistral-medium-latest': case 'mistral-large-latest': case 'reverse_proxy':{ let formatedChat:OpenAIChatExtra[] = [] if(db.inlayImage){ let pendingImages:OpenAIImageContents[] = [] for(let i=0;i 0 && m.role === 'user'){ let v:OpenAIChatExtra = cloneDeep(m) let contents:OpenAIContents[] = pendingImages contents.push({ "type": "text", "text": m.content }) v.content = contents formatedChat.push(v) pendingImages = [] } else{ formatedChat.push(m) } } } } else{ formatedChat = formated } let oobaSystemPrompts:string[] = [] for(let i=0;i 0){ formatedChat.push({ role: 'system', content: oobaSystemPrompts.join('\n') }) } if(db.newOAIHandle){ formatedChat = formatedChat.filter(m => { return m.content !== '' }) } for(let i=0;i 0){ // @ts-ignore body.seed = db.generationSeed } if(db.putUserOpen){ // @ts-ignore body.user = getOpenUserString() } if(aiModel === 'openrouter'){ if(db.top_k !== 0){ //@ts-ignore body.top_k = db.top_k } if(db.openrouterFallback){ //@ts-ignore body.route = "fallback" } //@ts-ignore body.transforms = db.openrouterMiddleOut ? ['middle-out'] : [] } if(aiModel === 'reverse_proxy' && db.reverseProxyOobaMode){ const OobaBodyTemplate = db.reverseProxyOobaArgs const keys = Object.keys(OobaBodyTemplate) for(const key of keys){ if(OobaBodyTemplate[key] !== undefined && OobaBodyTemplate[key] !== null){ // @ts-ignore body[key] = OobaBodyTemplate[key] } } } if(supportsInlayImage()){ // inlay models doesn't support logit_bias // @ts-ignore delete body.logit_bias } let replacerURL = aiModel === 'openrouter' ? "https://openrouter.ai/api/v1/chat/completions" : (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.openai.com/v1/chat/completions') let risuIdentify = false if(replacerURL.startsWith("risu::")){ risuIdentify = true replacerURL = replacerURL.replace("risu::", '') } if(aiModel === 'reverse_proxy' && db.autofillRequestUrl){ if(replacerURL.endsWith('v1')){ replacerURL += '/chat/completions' } else if(replacerURL.endsWith('v1/')){ replacerURL += 'chat/completions' } else if(!(replacerURL.endsWith('completions') || replacerURL.endsWith('completions/'))){ if(replacerURL.endsWith('/')){ replacerURL += 'v1/chat/completions' } else{ replacerURL += '/v1/chat/completions' } } } let headers = { "Authorization": "Bearer " + (aiModel === 'reverse_proxy' ? db.proxyKey : (aiModel === 'openrouter' ? db.openrouterKey : db.openAIKey)), "Content-Type": "application/json" } if(aiModel === 'openrouter'){ headers["X-Title"] = 'RisuAI' headers["HTTP-Referer"] = 'https://risuai.xyz' } if(risuIdentify){ headers["X-Proxy-Risu"] = 'RisuAI' } const multiGen = (db.genTime > 1 && aiModel.startsWith('gpt') && (!arg.continue)) if(multiGen){ // @ts-ignore body.n = db.genTime } let throughProxi = (!isTauri) && (!isNodeServer) && (!db.usePlainFetch) && (!Capacitor.isNativePlatform()) if(db.useStreaming && arg.useStreaming){ body.stream = true let urlHost = new URL(replacerURL).host if(urlHost.includes("localhost") || urlHost.includes("172.0.0.1") || urlHost.includes("0.0.0.0")){ if(!isTauri){ return { type: 'fail', result: 'You are trying local request on streaming. this is not allowed dude to browser/os security policy. turn off streaming.', } } } const da = (throughProxi) ? await fetch(hubURL + `/proxy2`, { body: JSON.stringify(body), headers: { "risu-header": encodeURIComponent(JSON.stringify(headers)), "risu-url": encodeURIComponent(replacerURL), "Content-Type": "application/json", "x-risu-tk": "use" }, method: "POST", signal: abortSignal }) : await fetch(replacerURL, { body: JSON.stringify(body), method: "POST", headers: headers, signal: abortSignal }) if(da.status !== 200){ return { type: "fail", result: await da.text() } } if (!da.headers.get('Content-Type').includes('text/event-stream')){ return { type: "fail", result: await da.text() } } addFetchLog({ body: body, response: "Streaming", success: true, url: replacerURL, }) let dataUint = new Uint8Array([]) const transtream = new TransformStream( { async transform(chunk, control) { dataUint = Buffer.from(new Uint8Array([...dataUint, ...chunk])) try { const datas = dataUint.toString().split('\n') let readed:{[key:string]:string} = {} for(const data of datas){ if(data.startsWith("data: ")){ try { const rawChunk = data.replace("data: ", "") if(rawChunk === "[DONE]"){ control.enqueue(readed) return } const choices = JSON.parse(rawChunk).choices for(const choice of choices){ const chunk = choice.delta.content if(chunk){ if(multiGen){ const ind = choice.index.toString() if(!readed[ind]){ readed[ind] = "" } readed[ind] += chunk } else{ if(!readed["0"]){ readed["0"] = "" } readed["0"] += chunk } } } } catch (error) {} } } control.enqueue(readed) } catch (error) { } } },) da.body.pipeTo(transtream.writable) return { type: 'streaming', result: transtream.readable } } const res = await globalFetch(replacerURL, { body: body, headers: headers, abortSignal, useRisuToken:throughProxi }) const dat = res.data as any if(res.ok){ try { if(multiGen && dat.choices){ return { type: 'multiline', result: dat.choices.map((v) => { return ["char",v.message.content] }) } } const msg:OpenAIChatFull = (dat.choices[0].message) return { type: 'success', result: msg.content } } catch (error) { return { type: 'fail', result: (language.errors.httpError + `${JSON.stringify(dat)}`) } } } else{ if(dat.error && dat.error.message){ return { type: 'fail', result: (language.errors.httpError + `${dat.error.message}`) } } else{ return { type: 'fail', result: (language.errors.httpError + `${JSON.stringify(res.data)}`) } } } break } case 'novelai': case 'novelai_kayra':{ console.log(arg.continue) const proompt = stringlizeNAIChat(formated, currentChar?.name ?? '', arg.continue) let logit_bias_exp:{ sequence: number[], bias: number, ensure_sequence_finish: false, generate_once: true }[] = [] for(let i=0;i m.content?.trim()).map(m => { let author = ''; if(m.role == 'system'){ m.content = m.content.trim(); } console.log(m.role +":"+m.content); switch (m.role) { case 'user': author = 'User'; break; case 'assistant': author = 'Assistant'; break; case 'system': author = 'Instruction'; break; default: author = m.role; break; } return `\n## ${author}\n${m.content.trim()}`; //return `\n\n${author}: ${m.content.trim()}`; }).join("") + `\n## Response\n`; const response = await globalFetch( "https://api.openai.com/v1/completions", { body: { model: "gpt-3.5-turbo-instruct", prompt: prompt, max_tokens: maxTokens, temperature: temperature, top_p: 1, stop:["User:"," User:", "user:", " user:"], presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), frequency_penalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), }, headers: { "Content-Type": "application/json", "Authorization": "Bearer " + db.openAIKey }, }); if(!response.ok){ return { type: 'fail', result: (language.errors.httpError + `${JSON.stringify(response.data)}`) } } const text:string = response.data.choices[0].text return { type: 'success', result: text.replace(/##\n/g, '') } } case "textgen_webui": case 'mancer':{ let streamUrl = db.textgenWebUIStreamURL.replace(/\/api.*/, "/api/v1/stream") let blockingUrl = db.textgenWebUIBlockingURL.replace(/\/api.*/, "/api/v1/generate") let bodyTemplate:any const suggesting = model === "submodel" const proompt = stringlizeChatOba(formated, currentChar.name, suggesting, arg.continue) let stopStrings = getStopStrings(suggesting) if(db.localStopStrings){ stopStrings = db.localStopStrings.map((v) => { return risuChatParser(v.replace(/\\n/g, "\n")) }) } bodyTemplate = { 'max_new_tokens': db.maxResponse, 'do_sample': db.ooba.do_sample, 'temperature': (db.temperature / 100), 'top_p': db.ooba.top_p, 'typical_p': db.ooba.typical_p, 'repetition_penalty': db.ooba.repetition_penalty, 'encoder_repetition_penalty': db.ooba.encoder_repetition_penalty, 'top_k': db.ooba.top_k, 'min_length': db.ooba.min_length, 'no_repeat_ngram_size': db.ooba.no_repeat_ngram_size, 'num_beams': db.ooba.num_beams, 'penalty_alpha': db.ooba.penalty_alpha, 'length_penalty': db.ooba.length_penalty, 'early_stopping': false, 'truncation_length': maxTokens, 'ban_eos_token': db.ooba.ban_eos_token, 'stopping_strings': stopStrings, 'seed': -1, add_bos_token: db.ooba.add_bos_token, topP: db.top_p, prompt: proompt } const headers = (aiModel === 'textgen_webui') ? {} : { 'X-API-KEY': db.mancerHeader } if(db.useStreaming && arg.useStreaming){ const oobaboogaSocket = new WebSocket(streamUrl); const statusCode = await new Promise((resolve) => { oobaboogaSocket.onopen = () => resolve(0) oobaboogaSocket.onerror = () => resolve(1001) oobaboogaSocket.onclose = ({ code }) => resolve(code) }) if(abortSignal.aborted || statusCode !== 0) { oobaboogaSocket.close() return ({ type: "fail", result: abortSignal.reason || `WebSocket connection failed to '${streamUrl}' failed!`, }) } const close = () => { oobaboogaSocket.close() } const stream = new ReadableStream({ start(controller){ let readed = ""; oobaboogaSocket.onmessage = async (event) => { const json = JSON.parse(event.data); if (json.event === "stream_end") { close() controller.close() return } if (json.event !== "text_stream") return readed += json.text controller.enqueue(readed) }; oobaboogaSocket.send(JSON.stringify(bodyTemplate)); }, cancel(){ close() } }) oobaboogaSocket.onerror = close oobaboogaSocket.onclose = close abortSignal.addEventListener("abort", close) return { type: 'streaming', result: stream } } const res = await globalFetch(blockingUrl, { body: bodyTemplate, headers: headers, abortSignal }) const dat = res.data as any if(res.ok){ try { let result:string = dat.results[0].text if(suggesting){ result = "\n" + db.autoSuggestPrefix + result } return { type: 'success', result: unstringlizeChat(result, formated, currentChar?.name ?? '') } } catch (error) { return { type: 'fail', result: (language.errors.httpError + `${error}`) } } } else{ return { type: 'fail', result: (language.errors.httpError + `${JSON.stringify(res.data)}`) } } } case 'ooba': { const suggesting = model === "submodel" const proompt = stringlizeChatOba(formated, currentChar.name, suggesting, arg.continue) let stopStrings = getStopStrings(suggesting) if(db.localStopStrings){ stopStrings = db.localStopStrings.map((v) => { return risuChatParser(v.replace(/\\n/g, "\n")) }) } let bodyTemplate:Record = { 'prompt': proompt, presence_penalty: arg.PresensePenalty || (db.PresensePenalty / 100), frequency_penalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), logit_bias: {}, max_tokens: maxTokens, stop: stopStrings, temperature: temperature, top_p: db.top_p, } const url = new URL(db.textgenWebUIBlockingURL) url.pathname = "/v1/completions" const urlStr = url.toString() const OobaBodyTemplate = db.reverseProxyOobaArgs const keys = Object.keys(OobaBodyTemplate) for(const key of keys){ if(OobaBodyTemplate[key] !== undefined && OobaBodyTemplate[key] !== null){ bodyTemplate[key] = OobaBodyTemplate[key] } } const response = await globalFetch(urlStr, { body: bodyTemplate, }) if(!response.ok){ return { type: 'fail', result: (language.errors.httpError + `${JSON.stringify(response.data)}`) } } const text:string = response.data.choices[0].text return { type: 'success', result: text.replace(/##\n/g, '') } } case 'custom':{ const d = await pluginProcess({ bias: bias, prompt_chat: formated, temperature: (db.temperature / 100), max_tokens: maxTokens, presence_penalty: (db.PresensePenalty / 100), frequency_penalty: (db.frequencyPenalty / 100) }) if(!d){ return { type: 'fail', result: (language.errors.unknownModel) } } else if(!d.success){ return { type: 'fail', result: d.content } } else{ return { type: 'success', result: d.content } } break } case 'palm2': case 'palm2_unicorn':{ const bodyData = { "instances": [ { "content": stringlizeChat(formated, currentChar?.name ?? '', arg.continue) } ], "parameters": { "candidateCount": 1, "maxOutputTokens": maxTokens, "stopSequences": [ "system:", "user:", "assistant:" ], "temperature": temperature, } }; const API_ENDPOINT="us-central1-aiplatform.googleapis.com" const PROJECT_ID=db.google.projectId const MODEL_ID= aiModel === 'palm2' ? 'text-bison' : 'palm2_unicorn' ? 'text-unicorn' : '' const LOCATION_ID="us-central1" const url = `https://${API_ENDPOINT}/v1/projects/${PROJECT_ID}/locations/${LOCATION_ID}/publishers/google/models/${MODEL_ID}:predict`; const res = await globalFetch(url, { body: bodyData, headers: { "Content-Type": "application/json", "Authorization": "Bearer " + db.google.accessToken }, abortSignal }) if(res.ok){ console.log(res.data) if(res.data.predictions){ let output:string = res.data.predictions[0].content const ind = output.search(/(system note)|(user)|(assistant):/gi) if(ind >= 0){ output = output.substring(0, ind) } return { type: 'success', result: output } } else{ return { type: 'fail', result: `${JSON.stringify(res.data)}` } } } else{ return { type: 'fail', result: `${JSON.stringify(res.data)}` } } } case 'gemini-pro': case 'gemini-pro-vision': case 'gemini-ultra': case 'gemini-ultra-vision':{ interface GeminiPart{ text?:string "inlineData"?: { "mimeType": string, "data": string }, } interface GeminiChat { role: "USER"|"MODEL" parts:|GeminiPart[] } let reformatedChat:GeminiChat[] = [] let pendingImage = '' for(let i=0;i { if(data?.candidates?.[0]?.content?.parts?.[0]?.text){ fullRes += data.candidates[0].content.parts[0].text } else if(data?.errors){ return { type: 'fail', result: `${JSON.stringify(data.errors)}` } } else{ return { type: 'fail', result: `${JSON.stringify(data)}` } } } // traverse responded data if it contains multipart contents if (typeof (res.data)[Symbol.iterator] === 'function') { for(const data of res.data){ processDataItem(data) } } else { processDataItem(res.data) } return { type: 'success', result: fullRes } } case "kobold":{ const proompt = stringlizeChat(formated, currentChar?.name ?? '', arg.continue) const url = new URL(db.koboldURL) if(url.pathname.length < 3){ url.pathname = 'api/v1/generate' } const da = await globalFetch(url.toString(), { method: "POST", body: { "prompt": proompt, "temperature": (db.temperature / 100), "top_p": 0.9 }, headers: { "content-type": "application/json", }, abortSignal }) if(!da.ok){ return { type: "fail", result: da.data, noRetry: true } } const data = da.data return { type: 'success', result: data.results[0].text } } case "novellist": case "novellist_damsel":{ const auth_key = db.novellistAPI; const api_server_url = 'https://api.tringpt.com/'; const logit_bias:string[] = [] const logit_bias_values:string[] = [] for(let i=0;i>") + db.ainconfig.stoptokens, logit_bias: (logit_bias.length > 0) ? logit_bias.join("<<|>>") : undefined, logit_bias_values: (logit_bias_values.length > 0) ? logit_bias_values.join("|") : undefined, }; const response = await globalFetch(api_server_url + '/api', { method: 'POST', headers: headers, body: send_body }); if(!response.ok){ return { type: 'fail', result: response.data } } if(response.data.error){ return { 'type': 'fail', 'result': `${response.data.error.replace("token", "api key")}` } } const result = response.data.data[0]; const unstr = unstringlizeAIN(result, formated, currentChar?.name ?? '') return { 'type': 'multiline', 'result': unstr } } case "deepai":{ for(let i=0;i { if(claudeChat.length > 0 && claudeChat[claudeChat.length-1].role === chat.role){ claudeChat[claudeChat.length-1].content += "\n\n" + chat.content } else{ claudeChat.push(chat) } } for(const chat of formated){ switch(chat.role){ case 'user':{ addClaudeChat({ role: 'user', content: chat.content }) break } case 'assistant':{ addClaudeChat({ role: 'assistant', content: chat.content }) break } case 'system':{ if(claudeChat.length === 0){ systemPrompt = chat.content } else{ addClaudeChat({ role: 'user', content: "System: " + chat.content }) } break } case 'function':{ //ignore function for now break } } } if(claudeChat.length === 0 && systemPrompt === ''){ return { type: 'fail', result: 'No input' } } if(claudeChat.length === 0 && systemPrompt !== ''){ claudeChat.push({ role: 'user', content: systemPrompt }) systemPrompt = '' } if(claudeChat[0].role !== 'user'){ claudeChat.unshift({ role: 'user', content: 'Start' }) } console.log(claudeChat, formated) let body = { model: raiModel, messages: claudeChat, system: systemPrompt, max_tokens: maxTokens, temperature: temperature, top_p: db.top_p, top_k: db.top_k, } if(systemPrompt === ''){ delete body.system } const res = await globalFetch(replacerURL, { body: body, headers: { "Content-Type": "application/json", "x-api-key": apiKey, "anthropic-version": "2023-06-01", "accept": "application/json" }, method: "POST" }) if(!res.ok){ return { type: 'fail', result: JSON.stringify(res.data) } } if(res.data.error){ return { type: 'fail', result: JSON.stringify(res.data.error) } } return { type: 'success', result: res.data.content[0].text } } else if(raiModel.startsWith('claude')){ let replacerURL = (aiModel === 'reverse_proxy') ? (db.forceReplaceUrl) : ('https://api.anthropic.com/v1/complete') let apiKey = (aiModel === 'reverse_proxy') ? db.proxyKey : db.claudeAPIKey if(aiModel === 'reverse_proxy'){ if(replacerURL.endsWith('v1')){ replacerURL += '/complete' } else if(replacerURL.endsWith('v1/')){ replacerURL += 'complete' } else if(!(replacerURL.endsWith('complete') || replacerURL.endsWith('complete/'))){ if(replacerURL.endsWith('/')){ replacerURL += 'v1/complete' } else{ replacerURL += '/v1/complete' } } } for(let i=0;i { let prefix = '' switch (v.role){ case "assistant": prefix = "\n\nAssistant: " break case "user": prefix = "\n\nHuman: " break case "system": prefix = "\n\nSystem: " break } latestRole = v.role if(raiModel.startsWith('claude-2') && (!raiModel.startsWith('claude-2.0'))){ if(v.role === 'system' && i === 0){ prefix = '' } } return prefix + v.content }).join('') if(latestRole !== 'assistant'){ requestPrompt += '\n\nAssistant: ' } const bedrock = db.claudeAws if(bedrock && aiModel !== 'reverse_proxy'){ function getCredentialParts(key:string) { const [accessKeyId, secretAccessKey, region] = key.split(":"); if (!accessKeyId || !secretAccessKey || !region) { throw new Error("The key assigned to this request is invalid."); } return { accessKeyId, secretAccessKey, region }; } const { accessKeyId, secretAccessKey, region } = getCredentialParts(apiKey); const AMZ_HOST = "bedrock-runtime.%REGION%.amazonaws.com"; const host = AMZ_HOST.replace("%REGION%", region); const stream = false const LATEST_AWS_V2_MINOR_VERSION = 1; let awsModel = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`; const pattern = /^(claude-)?(instant-)?(v)?(\d+)(\.(\d+))?(-\d+k)?$/i; const match = raiModel.match(pattern); if (match) { const [, , instant, v, major, dot, minor] = match; if (instant) { awsModel = "anthropic.claude-instant-v1"; } // There's only one v1 model else if (major === "1") { awsModel = "anthropic.claude-v1"; } // Try to map Anthropic API v2 models to AWS v2 models else if (major === "2") { if (minor === "0") { awsModel = "anthropic.claude-v2"; } else if (!v && !dot && !minor) { awsModel = "anthropic.claude-v2"; } } } const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}` const params = { prompt : requestPrompt.startsWith("\n\nHuman: ") ? requestPrompt : "\n\nHuman: " + requestPrompt, //model: raiModel, max_tokens_to_sample: maxTokens, stop_sequences: ["\n\nHuman:", "\n\nSystem:", "\n\nAssistant:"], temperature: temperature, top_p: db.top_p, } const rq = new HttpRequest({ method: "POST", protocol: "https:", hostname: host, path: `/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`, headers: { ["Host"]: host, ["Content-Type"]: "application/json", ["accept"]: "application/json", //"anthropic-version": "2023-06-01", }, body: JSON.stringify(params), }); const signer = new SignatureV4({ sha256: Sha256, credentials: { accessKeyId, secretAccessKey }, region, service: "bedrock", }); const signed = await signer.sign(rq); const da = await globalFetch(url, { method: "POST", body: params, headers: signed.headers, plainFetchForce: true }) if((!da.ok) || (da.data.error)){ return { type: 'fail', result: `${JSON.stringify(da.data)}` } } const res = da.data return { type: "success", result: res.completion, } } const da = await globalFetch(replacerURL, { method: "POST", body: { prompt : "\n\nHuman: " + requestPrompt, model: raiModel, max_tokens_to_sample: maxTokens, stop_sequences: ["\n\nHuman:", "\n\nSystem:", "\n\nAssistant:"], temperature: temperature, }, headers: { "Content-Type": "application/json", "x-api-key": apiKey, "anthropic-version": "2023-06-01", "accept": "application/json" }, useRisuToken: aiModel === 'reverse_proxy' }) if((!da.ok) || (da.data.error)){ return { type: 'fail', result: `${JSON.stringify(da.data)}` } } const res = da.data return { type: "success", result: res.completion, } } if(aiModel.startsWith("horde:::")){ const proompt = stringlizeChat(formated, currentChar?.name ?? '', arg.continue) const realModel = aiModel.split(":::")[1] const argument = { "prompt": proompt, "params": { "n": 1, "max_context_length": db.maxContext + 100, "max_length": db.maxResponse, "singleline": false, "temperature": db.temperature / 100, "top_k": db.top_k, "top_p": db.top_p, }, "trusted_workers": false, "workerslow_workers": true, "_blacklist": false, "dry_run": false, "models": [realModel, realModel.trim(), ' ' + realModel, realModel + ' '] } if(realModel === 'auto'){ delete argument.models } let apiKey = '0000000000' if(db.hordeConfig.apiKey.length > 2){ apiKey = db.hordeConfig.apiKey } const da = await fetch("https://stablehorde.net/api/v2/generate/text/async", { body: JSON.stringify(argument), method: "POST", headers: { "content-type": "application/json", "apikey": apiKey }, signal: abortSignal }) if(da.status !== 202){ return { type: "fail", result: await da.text() } } const json:{ id:string, kudos:number, message:string } = await da.json() let warnMessage = "" if(json.message){ warnMessage = "with " + json.message } while(true){ await sleep(2000) const data = await (await fetch("https://stablehorde.net/api/v2/generate/text/status/" + json.id)).json() if(!data.is_possible){ fetch("https://stablehorde.net/api/v2/generate/text/status/" + json.id, { method: "DELETE" }) return { type: 'fail', result: "Response not possible" + warnMessage, noRetry: true } } if(data.done && Array.isArray(data.generations) && data.generations.length > 0){ const generations:{text:string}[] = data.generations if(generations && generations.length > 0){ return { type: "success", result: unstringlizeChat(generations[0].text, formated, currentChar?.name ?? '') } } return { type: 'fail', result: "No Generations when done", noRetry: true } } } } if(aiModel.startsWith('hf:::')){ const realModel = aiModel.split(":::")[1] const suggesting = model === "submodel" const proompt = stringlizeChatOba(formated, currentChar.name, suggesting, arg.continue) const v = await runTransformers(proompt, realModel, { temperature: temperature, max_new_tokens: maxTokens, top_k: db.ooba.top_k, top_p: db.ooba.top_p, repetition_penalty: db.ooba.repetition_penalty, typical_p: db.ooba.typical_p, }) return { type: 'success', result: unstringlizeChat(v.generated_text, formated, currentChar?.name ?? '') } } if(aiModel.startsWith('local_')){ console.log('running local model') const suggesting = model === "submodel" const proompt = stringlizeChatOba(formated, currentChar.name, suggesting, arg.continue) const stopStrings = getStopStrings(suggesting) console.log(stopStrings) const modelPath = aiModel.replace('local_', '') const res = await runGGUFModel({ prompt: proompt, modelPath: modelPath, temperature: temperature, top_p: db.top_p, top_k: db.top_k, maxTokens: maxTokens, presencePenalty: arg.PresensePenalty || (db.PresensePenalty / 100), frequencyPenalty: arg.frequencyPenalty || (db.frequencyPenalty / 100), repeatPenalty: 0, maxContext: db.maxContext, stop: stopStrings, }) let decoded = '' const transtream = new TransformStream({ async transform(chunk, control) { const decodedChunk = new TextDecoder().decode(chunk) decoded += decodedChunk control.enqueue({ "0": decoded }) } }) res.pipeTo(transtream.writable) return { type: 'streaming', result: transtream.readable } } return { type: 'fail', result: (language.errors.unknownModel) } } } } let userString = '' let requestedTimes = 999 let refreshTime = 0 function getOpenUserString(){ if(refreshTime < Date.now() && requestedTimes > 2 ){ refreshTime = Date.now() + (300000 * Math.random()) + 60000 userString = v4() requestedTimes = 0 } requestedTimes += 1 console.log(userString) return userString }