mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix extractor initialization in runEmbedding function

Browse Source
This commit is contained in:
kwaroran
2024-01-05 23:33:27 +09:00
parent b4272e9e69
commit b6787b93a7
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/ts/process/embedding/transformers.ts
View File

@@ -47,9 +47,12 @@ export const runSummarizer = async (text: string) => {
return v return v
} }
let extractor:Pipeline = null
export const runEmbedding = async (text: string):Promise<Float32Array> => { export const runEmbedding = async (text: string):Promise<Float32Array> => {
await loadTransformer() await loadTransformer()
let extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2'); if(!extractor){
extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
}
const tokenizer = await AutoTokenizer.from_pretrained('Xenova/all-MiniLM-L6-v2'); const tokenizer = await AutoTokenizer.from_pretrained('Xenova/all-MiniLM-L6-v2');
const tokens = tokenizer.encode(text) const tokens = tokenizer.encode(text)
if (tokens.length > 256) { if (tokens.length > 256) {