Add text file post functionality

src/lib/ChatScreens/DefaultChatScreen.svelte

@@ -651,6 +651,10 @@
                                 messageInput += res.data
                                 updateInputSizeAll()
                             }
+                            if(res?.type === 'text'){
+                                messageInput += `{{file::${res.name}::${res.data}}}`
+                                updateInputSizeAll()
+                            }
                         }}>
                             <ImagePlusIcon />
                             <span class="ml-2">{language.postFile}</span>

src/styles.css

@@ -162,4 +162,16 @@ html, body{
 
 .text-bordered{
   -webkit-text-stroke: 1px #000;
+}
+
+.x-risu-risu-file{
+  padding: 1rem;
+  border: 1px solid var(--risu-theme-selected);
+  border-radius: 0.5rem;
+  color: var(--FontColorStandard);
+  min-width: 0;
+  max-width: 20rem;
+  overflow: hidden;
+  white-space: nowrap;
+  text-overflow: ellipsis;
 }

src/ts/parser.ts

@@ -393,6 +393,7 @@ type matcherArg = {
     var?:{[key:string]:string}
     tokenizeAccurate?:boolean
     consistantChar?:boolean
+    displaying?:boolean
 }
 const matcher = (p1:string,matcherArg:matcherArg) => {
     if(p1.length > 100000){
@@ -680,6 +681,12 @@ const matcher = (p1:string,matcherArg:matcherArg) => {
             case 'not':{
                 return (Number(arra[1]) === 0) ? '1' : '0'
             }
+            case 'file':{
+                if(matcherArg.displaying){
+                    return `<br><div class="risu-file">${arra[1]}</div><br>`
+                }
+                return Buffer.from(arra[2], 'base64').toString('utf-8') 
+            }
         }
     }
     if(p1.startsWith('random')){
@@ -852,7 +859,8 @@ export function risuChatParser(da:string, arg:{
         rmVar: arg.rmVar ?? false,
         db: db,
         var: arg.var ?? null,
-        tokenizeAccurate: arg.tokenizeAccurate ?? false
+        tokenizeAccurate: arg.tokenizeAccurate ?? false,
+        displaying: arg.visualize ?? false,
     }
     let pef = performance.now()
     while(pointer < da.length){

src/ts/process/files/multisend.ts

@@ -93,9 +93,10 @@ async function sendPofile(arg:sendFileArg){
 }
 
 async function sendPDFFile(arg:sendFileArg) {
-    const pdfjsLib = (await import('pdfjs-dist')).default;
+    const pdfjsLib = (await import('pdfjs-dist'));
+    const pdfjsWorker = await import('pdfjs-dist/build/pdf.worker?worker&url');
+    pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsWorker.default;
     const pdf = await pdfjsLib.getDocument({data: arg.file}).promise;
-    const db = get(DataBase)
     const texts:string[] = []
     for(let i = 1; i<=pdf.numPages; i++){
         const page = await pdf.getPage(i);
@@ -105,30 +106,40 @@ async function sendPDFFile(arg:sendFileArg) {
             texts.push(item.str)
         }
     }
+    console.log(texts)
     const hypa = new HypaProcesser('MiniLM')
     hypa.addText(texts)
-    let currentChar = db.characters[get(selectedCharID)]
-    let currentChat = currentChar.chats[currentChar.chatPage]
     const result = await hypa.similaritySearch(arg.query)
-    let message = arg.query
+    let message = ''
     for(let i = 0; i<result.length; i++){
-        message += "\n" + texts[result[i]]
+        message += "\n" + result[i]
         if(i>5){
             break
         }
     }
-    currentChat.message.push({
-        role: 'user',
-        data: message
-    })
-
-    currentChar.chats[currentChar.chatPage] = currentChat
-    db.characters[get(selectedCharID)] = currentChar
-    setDatabase(db)
-    await sendChat(-1)
+    console.log(message)
+    return Buffer.from(`<File>\n${message}\n</File>\n`).toString('base64')
 }
 
-type postFileResult = postFileResultImage | postFileResultVoid
+async function sendTxtFile(arg:sendFileArg) {
+    const lines = arg.file.split('\n').filter((a) => {
+        return a !== ''
+    })
+    const hypa = new HypaProcesser('MiniLM')
+    hypa.addText(lines)
+    const result = await hypa.similaritySearch(arg.query)
+    let message = ''
+    for(let i = 0; i<result.length; i++){
+        message += "\n" + result[i]
+        if(i>5){
+            break
+        }
+    }
+    console.log(message)
+    return Buffer.from(`<File>\n${message}\n</File>\n`).toString('base64')
+}
+
+type postFileResult = postFileResultImage | postFileResultVoid | postFileResultText
 
 type postFileResultImage = {
     data: string,
@@ -139,6 +150,11 @@ type postFileResultVoid = {
     type: 'void',
 }
 
+type postFileResultText = {
+    data: string,
+    type: 'text',
+    name: string
+}
 export async function postChatFile(query:string):Promise<postFileResult>{
     const file = await selectSingleFile([
         //image format
@@ -147,7 +163,8 @@ export async function postChatFile(query:string):Promise<postFileResult>{
         'png',
         'webp',
         'po',
-        'pdf'
+        // 'pdf',
+        'txt'
     ])
 
     if(!file){
@@ -168,12 +185,13 @@ export async function postChatFile(query:string):Promise<postFileResult>{
             }
         }
         case 'pdf':{
-            await sendPDFFile({
-                file: BufferToText(file.data),
-                query: query
-            })
             return {
-                type: 'void'
+                type: 'text',
+                data: await sendPDFFile({
+                    file: BufferToText(file.data),
+                    query: query
+                }),
+                name: file.name
             }
         }
         case 'jpg':
@@ -186,6 +204,16 @@ export async function postChatFile(query:string):Promise<postFileResult>{
                 type: 'image'
             }
         }
+        case 'txt':{
+            return {
+                type: 'text',
+                data: await sendTxtFile({
+                    file: BufferToText(file.data),
+                    query: query
+                }),
+                name: file.name
+            }
+        }
     }
 
     return