mincomk/risuai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add nomic embedding

Browse Source
This commit is contained in:
kwaroran
2024-04-23 20:41:08 +09:00
parent bb0ad4c57e
commit 376fa1641b
3 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/ts/process/transformers.ts
View File

@@ -50,12 +50,12 @@ export const runSummarizer = async (text: string) => {
}
let extractor:FeatureExtractionPipeline = null
export const runEmbedding = async (text: string):Promise<Float32Array> => {
export const runEmbedding = async (text: string, model:'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5' = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array> => {
await initTransformers()
if(!extractor){
extractor = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
extractor = await pipeline('feature-extraction', model);
}
const tokenizer = await AutoTokenizer.from_pretrained('Xenova/all-MiniLM-L6-v2');
const tokenizer = await AutoTokenizer.from_pretrained(model);
const tokens = tokenizer.encode(text)
if (tokens.length > 1024) {
let chunks:string[] = []