From 44c81e10f9bfe483e638fa00bab2e34a58bfa60c Mon Sep 17 00:00:00 2001 From: kwaroran Date: Mon, 17 Jul 2023 23:37:05 +0900 Subject: [PATCH] [feat] generate by web --- src/lang/en.ts | 6 +- src/lib/Others/AlertComp.svelte | 2 +- src/lib/SideBars/Sidebar.svelte | 7 + src/ts/creator/creator.ts | 320 ++++++++++++++++++++++++++++++++ src/ts/parser.ts | 3 +- src/ts/storage/globalApi.ts | 7 +- 6 files changed, 340 insertions(+), 5 deletions(-) create mode 100644 src/ts/creator/creator.ts diff --git a/src/lang/en.ts b/src/lang/en.ts index 436fdf54..8006ff04 100644 --- a/src/lang/en.ts +++ b/src/lang/en.ts @@ -339,5 +339,9 @@ export const languageEnglish = { persona: "Persona", icon:"Icon", account: "Account", - remove: "Remove" + remove: "Remove", + creationSuccess: "Creation Success", + noweb: "This feature cannot be used on web version.", + createBotInternet: "Create Bot from Internet with AI", + createBotInternetAlert: "Please provide the character's name and the corresponding series/game." } \ No newline at end of file diff --git a/src/lib/Others/AlertComp.svelte b/src/lib/Others/AlertComp.svelte index b5ff5c98..fb4f31f2 100644 --- a/src/lib/Others/AlertComp.svelte +++ b/src/lib/Others/AlertComp.svelte @@ -90,7 +90,7 @@ }) }}>OK {:else if $alertStore.type === 'input'} - + + {/if} diff --git a/src/ts/creator/creator.ts b/src/ts/creator/creator.ts new file mode 100644 index 00000000..65e91e15 --- /dev/null +++ b/src/ts/creator/creator.ts @@ -0,0 +1,320 @@ +import { language } from "src/lang"; +import { alertError, alertInput, alertNormal, alertSelect, alertStore } from "../alert"; +import { requestChatData } from "../process/request"; +import { checkCharOrder, globalFetch, isNodeServer, isTauri, saveAsset } from "../storage/globalApi"; +import { tokenize } from "../tokenizer"; +import { createBlankChar } from "../characters"; +import { DataBase, setDatabase, type character } from "../storage/database"; +import { get } from "svelte/store"; +import { sleep } from "../util"; + + +async function createBotFromWebMain(prompt:string):Promise<{ ok: false; data:string }|{ok:"creation";data:character}>{ + + + + const usp = new URLSearchParams() + usp.append("q","fandom " + prompt) + + + if(prompt.toLocaleLowerCase().includes('risu') && (!prompt.toLocaleLowerCase().includes('arisu'))){ + return { + ok: false, + data: "You are playing Risu now" + } + } + + const bd = await globalFetch("https://lite.duckduckgo.com/lite/", { + body: usp, + rawResponse: true, + headers: { + 'Content-Type': "application/x-www-form-urlencoded" + } + }) + if(!bd.ok){ + return { + ok: false, + data: bd.data + } + } + const searchDom = new DOMParser() + const searchDoc = searchDom.parseFromString(Buffer.from(bd.data).toString('utf-8'), 'text/html') + console.log(searchDoc) + const links = searchDoc.querySelectorAll(".link-text") + let url = '' + const fandomURL = /(.+?)\.fandom\.com\/wiki\//g + const wikiggURL = /(.+?)\.wiki\.gg\/wiki\//g + + for(const link of links){ + try { + let lurl = link.innerHTML.trim() + if(fandomURL.test(lurl) || wikiggURL.test(lurl)){ + if(!lurl.startsWith("https://")){ + lurl = "https://" + lurl + } + const surl = new URL(lurl) + const charname = surl.pathname.split("/")[2].toLocaleLowerCase() + console.log(charname) + if(charname.includes('main') ||charname.includes('home') || charname.includes('wiki')){ + continue + } + url = lurl + break + } + } catch (error) { + console.error(error) + } + } + + + + const surl = new URL(url) + const urlPathName = surl.pathname.split("/") + + while(urlPathName.length > 3){ + urlPathName.pop() + } + surl.pathname = urlPathName.join('/') + url = surl.toString() + + if(url === ''){ + return { + ok: false, + data: "Cannot find the character, is the character not popular, or did you misspelled?" + } + } + + let val = '' + let imgID = '' + const charname = url.split('/').at(-1) + + if(charname.toLocaleLowerCase().startsWith('main') || charname.toLocaleLowerCase().startsWith('wiki')){ + return { + ok: false, + data: "Cannot find the character, is the character not popular, or did you misspelled?" + } + } + + try { + const v = await globalFetch(url, {rawResponse: true, method: 'GET'}) + if(!v.ok){ + throw '' + } + const parser = new DOMParser() + + const vdom = parser.parseFromString(Buffer.from(v.data).toString(), 'text/html') + const imgDoms = vdom.querySelectorAll("#mw-content-text .mw-parser-output img") + + let qurl = '' + let level = 0 + for(const dom of imgDoms){ + const thisLevel = dom.className.includes("thumbnail") ? 2 : 1 + if(thisLevel > level){ + qurl = dom.getAttribute("src") || qurl + level = thisLevel + } + } + + if(qurl){ + const v = await globalFetch(qurl, {rawResponse: true, method: 'GET'}) + if(!v.ok){ + throw '' + } + imgID = await saveAsset(v.data) + } + + + await sleep(2000) + } catch (error) { + console.error(error) + } + + + try { + const vurl = `https://bluearchive.fandom.com/api.php?action=visualeditor&format=json&paction=wikitext&page=${charname}&uselang=en&formatversion=2` + const fv = await globalFetch(vurl) + if(!fv.ok){ + throw '' + } + val = fv.data?.visualeditor?.content ?? '' + if(val === ''){ + throw '' + } + } catch (error) { + const rurl = url + '?action=edit' + const v = await globalFetch(rurl, {rawResponse: true, method: 'GET'}) + + if(!v.ok){ + console.log(v) + return { + ok: false, + data: "Failed on Reading Site" + } + } + + const parser = new DOMParser() + const vdom = parser.parseFromString(Buffer.from(v.data).toString(), 'text/html') + const ta:HTMLTextAreaElement = vdom.querySelector('textarea#wpTextbox1') + + if((!ta) || (!ta.value)){ + console.log(vdom.body.outerHTML) + return { + ok: false, + data: "Data cannot be found inside site." + } + } + + val = ta.value + } + + let tokns = await tokenize(val) + if (tokns > 3200){ + const v = val.split('\n') + let chunks:[string,string,number][] = [["main","",0]] + + for(const a of v){ + if(a.startsWith('==') && (a.endsWith('=='))){ + chunks.push([a, a + "\n", 0]) + } + chunks.at(-1)[1] += a + "\n" + } + + for(let i=0;i 3200){ + + const ind = chunks.length-1 + + tokns -= chunks[ind][2] + + chunks.splice(ind, 1) + } + + val = chunks.map((v) => v[1]).join("\n") + } + + + alertStore.set({ + type: 'wait', + msg: 'Loading..' + }) + const rqv = val + "\n\n[[This was a character's wiki page data.]]" + const ch = await requestChatData({ + formated: [{ + role: 'system', + content: rqv + },{ + role: 'system', + content: "\n\n*Name*:\n*Age*:\n*gender*: \n*race*:\n*Hair style, color*:\n*color, shape of eye*:\n*Personality*:\n*Dress*:\n*Height (cm)*:\n*weight(kg)*:\n*Job*:\n*Specialty*:\n*Features*: \n*Likes*:\n*Dislikes*:\n*Character's background*: \n\n[[This is a format that you must convert to. output the latest information If there is older information. If it is unknown, output as unknown. only output the converted result. now, output the converted result.]]" + }], + maxTokens: 600, + temperature: 0, + bias: {} + }, 'submodel') + + if(ch.type === 'multiline' || ch.type === 'fail' || ch.type === 'streaming'){ + return { + ok: false, + data: "Request Fail: " + ch.result + } + } + + const char = createBlankChar() + + char.name = charname.replaceAll("_"," ") + char.desc = ch.result + char.creatorNotes = `Generated by RisuAI, Data from ` + url + if(imgID){ + char.image = imgID + } + return { + ok: "creation", + data: char + } + + +} + +async function createFirstMsg(charDesc:string) { + const v = await requestChatData({ + formated: [{ + role: "system", + content: charDesc + `\n[This was the character's description]` + },{ + role: "system", + content: "Create and describe the situation where the character and {{user}} meet, reflecting about information of character and prompt. Line from {{user}} is not allowed." + }], + bias: {}, + maxTokens: 600, + temperature: 0 + }, 'submodel') + + return v +} + +async function createBotFromWeb() { + if((!isTauri) && (!isNodeServer)){ + alertNormal(language.noweb) + return + } + let search = (await alertInput(language.createBotInternetAlert)).split("#")[0] + if(search.length < 3){ + return + } + alertStore.set({ + type: 'wait', + msg: 'Fetching..' + }) + const d = await createBotFromWebMain(search) + if(d.ok === 'creation'){ + const db = get(DataBase) + const cha = d.data + const fm = await createFirstMsg(cha.desc) + if(fm.type === 'multiline' || fm.type === 'fail' || fm.type === 'streaming'){ + return { + ok: false, + data: "Request Fail: " + fm.result + } + } + cha.firstMessage = surroundTextWithAsterisks(fm.result) + db.characters.push(d.data) + checkCharOrder() + setDatabase(db) + alertNormal(language.creationSuccess) + } + else{ + alertError(d.data) + } +} + +export const BotCreator = { + createBotFromWeb +} + +function surroundTextWithAsterisks(fulltext:string) { + let result:string[] = [] + + const splited = fulltext.split("\n") + for(const text of splited){ + if(!text){ + result.push(text) + continue + } + let output = ''; + let parts = text.split('"'); + for(let i = 0; i < parts.length; i++) { + let part = parts[i]; + if(i % 2 === 0) { + let trimmed = part.trim(); + output += trimmed ? '*' + trimmed + '*' : part; + } else { + output += '"' + part + '"'; + } + } + result.push(output) + } + + return result.join('\n'); +} diff --git a/src/ts/parser.ts b/src/ts/parser.ts index aa5e86f1..29b486d6 100644 --- a/src/ts/parser.ts +++ b/src/ts/parser.ts @@ -105,13 +105,14 @@ export async function ParseMarkdown(data:string, char:(character | groupChat) = return DOMPurify.sanitize(mconverted.parse(data), { ADD_TAGS: ["iframe"], ADD_ATTR: ["allow", "allowfullscreen", "frameborder", "scrolling"], + FORBID_ATTR: ["href"] }) } export function parseMarkdownSafe(data:string) { return DOMPurify.sanitize(safeConvertor.makeHtml(data), { FORBID_TAGS: ["a", "style"], - FORBID_ATTR: ["style"] + FORBID_ATTR: ["style", "href"] }) } diff --git a/src/ts/storage/globalApi.ts b/src/ts/storage/globalApi.ts index 971e84a5..4d6d7dcf 100644 --- a/src/ts/storage/globalApi.ts +++ b/src/ts/storage/globalApi.ts @@ -571,7 +571,8 @@ export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body } } - const body = Body.json(arg.body) + const body = (!arg.body) ? null : + (arg.body instanceof URLSearchParams) ? (Body.text(arg.body.toString())) : (Body.json(arg.body)) const headers = arg.headers ?? {} const d = await TauriFetch(url, { body: body, @@ -581,7 +582,8 @@ export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body secs: db.timeOut, nanos: 0 }, - responseType: arg.rawResponse ? ResponseType.Binary : ResponseType.JSON + responseType: arg.rawResponse ? ResponseType.Binary : ResponseType.JSON, + }) if(arg.rawResponse){ addFetchLog("Uint8Array Response", d.ok) @@ -680,6 +682,7 @@ export async function globalFetch(url:string, arg:{plainFetchForce?:boolean,body } } } catch (error) { + console.error(error) return { ok:false, data: `${error}`,