From 3dec23a99c39545df175086c0222cf17ea976a18 Mon Sep 17 00:00:00 2001 From: kwaroran Date: Mon, 15 Jan 2024 03:40:29 +0900 Subject: [PATCH] Update token length check in embedding transformer and error message in supaMemory --- src/ts/process/embedding/transformers.ts | 2 +- src/ts/process/memory/supaMemory.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ts/process/embedding/transformers.ts b/src/ts/process/embedding/transformers.ts index f4898500..627b9db5 100644 --- a/src/ts/process/embedding/transformers.ts +++ b/src/ts/process/embedding/transformers.ts @@ -57,7 +57,7 @@ export const runEmbedding = async (text: string):Promise => { } const tokenizer = await AutoTokenizer.from_pretrained('Xenova/all-MiniLM-L6-v2'); const tokens = tokenizer.encode(text) - if (tokens.length > 256) { + if (tokens.length > 1024) { let chunks:string[] = [] let chunk:number[] = [] for (let i = 0; i < tokens.length; i++) { diff --git a/src/ts/process/memory/supaMemory.ts b/src/ts/process/memory/supaMemory.ts index 7874d5a5..905e1b03 100644 --- a/src/ts/process/memory/supaMemory.ts +++ b/src/ts/process/memory/supaMemory.ts @@ -178,7 +178,7 @@ export async function supaMemory( if(db.supaMemoryType === 'distilbart'){ try { const sum = await runSummarizer(stringlizedChat) - return sum[0].summary_text + return sum } catch (error) { return { currentTokens: currentTokens, @@ -274,7 +274,7 @@ export async function supaMemory( return { currentTokens: currentTokens, chats: chats, - error: "Not Enough Tokens" + error: "Not Enough Tokens to summarize in SupaMemory" } } maxChunkSize = maxChunkSize * 0.7