Refactor HypaProcesser instantiation to remove hardcoded model name and add models and others
This commit is contained in:
kwaroran
@@ -23,6 +23,7 @@
|
||||
"@capacitor/filesystem": "^5.2.0",
|
||||
"@dqbd/tiktoken": "^1.0.7",
|
||||
"@huggingface/jinja": "^0.2.2",
|
||||
"@huggingface/transformers": "^3.1.1",
|
||||
"@mlc-ai/web-tokenizers": "^0.1.2",
|
||||
"@risuai/ccardlib": "^0.4.1",
|
||||
"@smithy/protocol-http": "^3.0.12",
|
||||
@@ -37,7 +38,6 @@
|
||||
"@tauri-apps/plugin-shell": "~2",
|
||||
"@tauri-apps/plugin-updater": "~2",
|
||||
"@types/markdown-it": "^14.1.1",
|
||||
"@xenova/transformers": "^2.17.1",
|
||||
"blueimp-md5": "^2.19.0",
|
||||
"body-parser": "^1.20.2",
|
||||
"buffer": "^6.0.3",
|
||||
|
||||
386
pnpm-lock.yaml
generated
386
pnpm-lock.yaml
generated
@@ -29,6 +29,9 @@ importers:
|
||||
'@huggingface/jinja':
|
||||
specifier: ^0.2.2
|
||||
version: 0.2.2
|
||||
'@huggingface/transformers':
|
||||
specifier: ^3.1.1
|
||||
version: 3.1.1
|
||||
'@mlc-ai/web-tokenizers':
|
||||
specifier: ^0.1.2
|
||||
version: 0.1.2
|
||||
@@ -71,9 +74,6 @@ importers:
|
||||
'@types/markdown-it':
|
||||
specifier: ^14.1.1
|
||||
version: 14.1.1
|
||||
'@xenova/transformers':
|
||||
specifier: ^2.17.1
|
||||
version: 2.17.1
|
||||
blueimp-md5:
|
||||
specifier: ^2.19.0
|
||||
version: 2.19.0
|
||||
@@ -421,6 +421,9 @@ packages:
|
||||
'@dqbd/tiktoken@1.0.7':
|
||||
resolution: {integrity: sha512-bhR5k5W+8GLzysjk8zTMVygQZsgvf7W1F0IlL4ZQ5ugjo5rCyiwGM5d8DYriXspytfu98tv59niang3/T+FoDw==}
|
||||
|
||||
'@emnapi/runtime@1.3.1':
|
||||
resolution: {integrity: sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==}
|
||||
|
||||
'@esbuild/aix-ppc64@0.21.5':
|
||||
resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==}
|
||||
engines: {node: '>=12'}
|
||||
@@ -563,10 +566,122 @@ packages:
|
||||
resolution: {integrity: sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@huggingface/jinja@0.3.2':
|
||||
resolution: {integrity: sha512-F2FvuIc+w1blGsaqJI/OErRbWH6bVJDCBI8Rm5D86yZ2wlwrGERsfIaru7XUv9eYC3DMP3ixDRRtF0h6d8AZcQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@huggingface/transformers@3.1.1':
|
||||
resolution: {integrity: sha512-/OpCiSKIowo5w5rJAOH3pgZKvpT6DOfDYw9br9Fp8w3qm4oyxc6dOhrxdRLVrmqLbE8rp5dKCePUW34ZBsinsg==}
|
||||
|
||||
'@hutson/parse-repository-url@3.0.2':
|
||||
resolution: {integrity: sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q==}
|
||||
engines: {node: '>=6.9.0'}
|
||||
|
||||
'@img/sharp-darwin-arm64@0.33.5':
|
||||
resolution: {integrity: sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
'@img/sharp-darwin-x64@0.33.5':
|
||||
resolution: {integrity: sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
'@img/sharp-libvips-darwin-arm64@1.0.4':
|
||||
resolution: {integrity: sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
'@img/sharp-libvips-darwin-x64@1.0.4':
|
||||
resolution: {integrity: sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
'@img/sharp-libvips-linux-arm64@1.0.4':
|
||||
resolution: {integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-libvips-linux-arm@1.0.5':
|
||||
resolution: {integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-libvips-linux-s390x@1.0.4':
|
||||
resolution: {integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-libvips-linux-x64@1.0.4':
|
||||
resolution: {integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-libvips-linuxmusl-arm64@1.0.4':
|
||||
resolution: {integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-libvips-linuxmusl-x64@1.0.4':
|
||||
resolution: {integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-linux-arm64@0.33.5':
|
||||
resolution: {integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-linux-arm@0.33.5':
|
||||
resolution: {integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-linux-s390x@0.33.5':
|
||||
resolution: {integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-linux-x64@0.33.5':
|
||||
resolution: {integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-linuxmusl-arm64@0.33.5':
|
||||
resolution: {integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-linuxmusl-x64@0.33.5':
|
||||
resolution: {integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
'@img/sharp-wasm32@0.33.5':
|
||||
resolution: {integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [wasm32]
|
||||
|
||||
'@img/sharp-win32-ia32@0.33.5':
|
||||
resolution: {integrity: sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [ia32]
|
||||
os: [win32]
|
||||
|
||||
'@img/sharp-win32-x64@0.33.5':
|
||||
resolution: {integrity: sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
'@ionic/cli-framework-output@2.2.8':
|
||||
resolution: {integrity: sha512-TshtaFQsovB4NWRBydbNFawql6yul7d5bMiW1WYYf17hd99V6xdDdk3vtF51bw6sLkxON3bDQpWsnUc9/hVo3g==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
@@ -607,6 +722,10 @@ packages:
|
||||
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
'@isaacs/fs-minipass@4.0.1':
|
||||
resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@jridgewell/gen-mapping@0.3.3':
|
||||
resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==}
|
||||
engines: {node: '>=6.0.0'}
|
||||
@@ -1113,9 +1232,6 @@ packages:
|
||||
'@types/lodash@4.14.202':
|
||||
resolution: {integrity: sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ==}
|
||||
|
||||
'@types/long@4.0.2':
|
||||
resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==}
|
||||
|
||||
'@types/markdown-it@14.1.1':
|
||||
resolution: {integrity: sha512-4NpsnpYl2Gt1ljyBGrKMxFYAYvpqbnnkgP/i/g+NLpjEUa3obn1XJCur9YbEXKDAkaXqsR1LbDnGEJ0MmKFxfg==}
|
||||
|
||||
@@ -1164,9 +1280,6 @@ packages:
|
||||
'@types/wicg-file-system-access@2020.9.8':
|
||||
resolution: {integrity: sha512-ggMz8nOygG7d/stpH40WVaNvBwuyYLnrg5Mbyf6bmsj/8+gb6Ei4ZZ9/4PNpcPNTT8th9Q8sM8wYmWGjMWLX/A==}
|
||||
|
||||
'@xenova/transformers@2.17.1':
|
||||
resolution: {integrity: sha512-zo702tQAFZXhzeD2GCYUNUqeqkoueOdiSbQWa4s0q7ZE4z8WBIwIsMMPGobpgdqjQ2u0Qulo08wuqVEUrBXjkQ==}
|
||||
|
||||
'@xml-tools/parser@1.0.11':
|
||||
resolution: {integrity: sha512-aKqQ077XnR+oQtHJlrAflaZaL7qZsulWc/i/ZEooar5JiWj1eLt0+Wg28cpa+XLney107wXqneC+oG1IZvxkTA==}
|
||||
|
||||
@@ -1434,6 +1547,10 @@ packages:
|
||||
resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
chownr@3.0.0:
|
||||
resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
clean-stack@2.2.0:
|
||||
resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==}
|
||||
engines: {node: '>=6'}
|
||||
@@ -1750,6 +1867,10 @@ packages:
|
||||
resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
detect-libc@2.0.3:
|
||||
resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
didyoumean@1.2.2:
|
||||
resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==}
|
||||
|
||||
@@ -2401,8 +2522,8 @@ packages:
|
||||
lodash@4.17.21:
|
||||
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
|
||||
|
||||
long@4.0.0:
|
||||
resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==}
|
||||
long@5.2.3:
|
||||
resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==}
|
||||
|
||||
lru-cache@10.1.0:
|
||||
resolution: {integrity: sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==}
|
||||
@@ -2543,10 +2664,18 @@ packages:
|
||||
resolution: {integrity: sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==}
|
||||
engines: {node: '>=16 || 14 >=14.17'}
|
||||
|
||||
minipass@7.1.2:
|
||||
resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
|
||||
engines: {node: '>=16 || 14 >=14.17'}
|
||||
|
||||
minizlib@2.1.2:
|
||||
resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
minizlib@3.0.1:
|
||||
resolution: {integrity: sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==}
|
||||
engines: {node: '>= 18'}
|
||||
|
||||
mkdirp-classic@0.5.3:
|
||||
resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
|
||||
|
||||
@@ -2555,6 +2684,11 @@ packages:
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
mkdirp@3.0.1:
|
||||
resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==}
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
ml-array-mean@1.1.6:
|
||||
resolution: {integrity: sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==}
|
||||
|
||||
@@ -2728,18 +2862,15 @@ packages:
|
||||
resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
onnx-proto@4.0.4:
|
||||
resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==}
|
||||
onnxruntime-common@1.20.1:
|
||||
resolution: {integrity: sha512-YiU0s0IzYYC+gWvqD1HzLc46Du1sXpSiwzKb63PACIJr6LfL27VsXSXQvt68EzD3V0D5Bc0vyJTjmMxp0ylQiw==}
|
||||
|
||||
onnxruntime-common@1.14.0:
|
||||
resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==}
|
||||
|
||||
onnxruntime-node@1.14.0:
|
||||
resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==}
|
||||
onnxruntime-node@1.20.1:
|
||||
resolution: {integrity: sha512-di/I4HDXRw+FLgq+TyHmQEDd3cEp9iFFZm0r4uJ1Wd7b/WE1VXtKWo8yemex347c6GNF/3Pv86ZfPhIWxORr0w==}
|
||||
os: [win32, darwin, linux]
|
||||
|
||||
onnxruntime-web@1.14.0:
|
||||
resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==}
|
||||
onnxruntime-web@1.20.1:
|
||||
resolution: {integrity: sha512-TePF6XVpLL1rWVMIl5Y9ACBQcyCNFThZON/jgElNd9Txb73CIEGlklhYR3UEr1cp5r0rbGI6nDwwrs79g7WjoA==}
|
||||
|
||||
open@8.4.2:
|
||||
resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==}
|
||||
@@ -2963,9 +3094,9 @@ packages:
|
||||
resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
protobufjs@6.11.4:
|
||||
resolution: {integrity: sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==}
|
||||
hasBin: true
|
||||
protobufjs@7.4.0:
|
||||
resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==}
|
||||
engines: {node: '>=12.0.0'}
|
||||
|
||||
proxy-addr@2.0.7:
|
||||
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
|
||||
@@ -3105,6 +3236,10 @@ packages:
|
||||
engines: {node: '>=14'}
|
||||
hasBin: true
|
||||
|
||||
rimraf@5.0.10:
|
||||
resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==}
|
||||
hasBin: true
|
||||
|
||||
rollup@3.29.4:
|
||||
resolution: {integrity: sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw==}
|
||||
engines: {node: '>=14.18.0', npm: '>=8.0.0'}
|
||||
@@ -3164,6 +3299,11 @@ packages:
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
semver@7.6.3:
|
||||
resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==}
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
send@0.18.0:
|
||||
resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==}
|
||||
engines: {node: '>= 0.8.0'}
|
||||
@@ -3189,6 +3329,10 @@ packages:
|
||||
resolution: {integrity: sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==}
|
||||
engines: {node: '>=14.15.0'}
|
||||
|
||||
sharp@0.33.5:
|
||||
resolution: {integrity: sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==}
|
||||
engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0}
|
||||
|
||||
shebang-command@2.0.0:
|
||||
resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
|
||||
engines: {node: '>=8'}
|
||||
@@ -3434,6 +3578,10 @@ packages:
|
||||
resolution: {integrity: sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
tar@7.4.3:
|
||||
resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
temp-dir@2.0.0:
|
||||
resolution: {integrity: sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==}
|
||||
engines: {node: '>=8'}
|
||||
@@ -3807,6 +3955,10 @@ packages:
|
||||
yallist@4.0.0:
|
||||
resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
|
||||
|
||||
yallist@5.0.0:
|
||||
resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
yaml@2.4.2:
|
||||
resolution: {integrity: sha512-B3VqDZ+JAg1nZpaEmWtTXUlBneoGx6CPM9b0TENK6aoSu5t73dItudwdgmi6tHlIZZId4dZ9skcAQ2UbcyAeVA==}
|
||||
engines: {node: '>= 14'}
|
||||
@@ -3989,6 +4141,11 @@ snapshots:
|
||||
|
||||
'@dqbd/tiktoken@1.0.7': {}
|
||||
|
||||
'@emnapi/runtime@1.3.1':
|
||||
dependencies:
|
||||
tslib: 2.6.2
|
||||
optional: true
|
||||
|
||||
'@esbuild/aix-ppc64@0.21.5':
|
||||
optional: true
|
||||
|
||||
@@ -4060,8 +4217,92 @@ snapshots:
|
||||
|
||||
'@huggingface/jinja@0.2.2': {}
|
||||
|
||||
'@huggingface/jinja@0.3.2': {}
|
||||
|
||||
'@huggingface/transformers@3.1.1':
|
||||
dependencies:
|
||||
'@huggingface/jinja': 0.3.2
|
||||
onnxruntime-node: 1.20.1
|
||||
onnxruntime-web: 1.20.1
|
||||
sharp: 0.33.5
|
||||
|
||||
'@hutson/parse-repository-url@3.0.2': {}
|
||||
|
||||
'@img/sharp-darwin-arm64@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-darwin-arm64': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-darwin-x64@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-darwin-x64': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-darwin-arm64@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-darwin-x64@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-linux-arm64@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-linux-arm@1.0.5':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-linux-s390x@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-linux-x64@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-linuxmusl-arm64@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-libvips-linuxmusl-x64@1.0.4':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-linux-arm64@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-linux-arm64': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-linux-arm@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-linux-arm': 1.0.5
|
||||
optional: true
|
||||
|
||||
'@img/sharp-linux-s390x@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-linux-s390x': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-linux-x64@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-linux-x64': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-linuxmusl-arm64@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-linuxmusl-arm64': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-linuxmusl-x64@0.33.5':
|
||||
optionalDependencies:
|
||||
'@img/sharp-libvips-linuxmusl-x64': 1.0.4
|
||||
optional: true
|
||||
|
||||
'@img/sharp-wasm32@0.33.5':
|
||||
dependencies:
|
||||
'@emnapi/runtime': 1.3.1
|
||||
optional: true
|
||||
|
||||
'@img/sharp-win32-ia32@0.33.5':
|
||||
optional: true
|
||||
|
||||
'@img/sharp-win32-x64@0.33.5':
|
||||
optional: true
|
||||
|
||||
'@ionic/cli-framework-output@2.2.8':
|
||||
dependencies:
|
||||
'@ionic/utils-terminal': 2.3.5
|
||||
@@ -4161,6 +4402,10 @@ snapshots:
|
||||
wrap-ansi: 8.1.0
|
||||
wrap-ansi-cjs: wrap-ansi@7.0.0
|
||||
|
||||
'@isaacs/fs-minipass@4.0.1':
|
||||
dependencies:
|
||||
minipass: 7.1.2
|
||||
|
||||
'@jridgewell/gen-mapping@0.3.3':
|
||||
dependencies:
|
||||
'@jridgewell/set-array': 1.1.2
|
||||
@@ -4633,8 +4878,6 @@ snapshots:
|
||||
|
||||
'@types/lodash@4.14.202': {}
|
||||
|
||||
'@types/long@4.0.2': {}
|
||||
|
||||
'@types/markdown-it@14.1.1':
|
||||
dependencies:
|
||||
'@types/linkify-it': 5.0.0
|
||||
@@ -4681,14 +4924,6 @@ snapshots:
|
||||
|
||||
'@types/wicg-file-system-access@2020.9.8': {}
|
||||
|
||||
'@xenova/transformers@2.17.1':
|
||||
dependencies:
|
||||
'@huggingface/jinja': 0.2.2
|
||||
onnxruntime-web: 1.14.0
|
||||
sharp: 0.32.6
|
||||
optionalDependencies:
|
||||
onnxruntime-node: 1.14.0
|
||||
|
||||
'@xml-tools/parser@1.0.11':
|
||||
dependencies:
|
||||
chevrotain: 7.1.1
|
||||
@@ -4983,6 +5218,8 @@ snapshots:
|
||||
|
||||
chownr@2.0.0: {}
|
||||
|
||||
chownr@3.0.0: {}
|
||||
|
||||
clean-stack@2.2.0: {}
|
||||
|
||||
cliui@6.0.0:
|
||||
@@ -5308,6 +5545,8 @@ snapshots:
|
||||
|
||||
detect-libc@2.0.2: {}
|
||||
|
||||
detect-libc@2.0.3: {}
|
||||
|
||||
didyoumean@1.2.2: {}
|
||||
|
||||
diff@4.0.2: {}
|
||||
@@ -6015,7 +6254,7 @@ snapshots:
|
||||
|
||||
lodash@4.17.21: {}
|
||||
|
||||
long@4.0.0: {}
|
||||
long@5.2.3: {}
|
||||
|
||||
lru-cache@10.1.0: {}
|
||||
|
||||
@@ -6143,15 +6382,24 @@ snapshots:
|
||||
|
||||
minipass@7.0.4: {}
|
||||
|
||||
minipass@7.1.2: {}
|
||||
|
||||
minizlib@2.1.2:
|
||||
dependencies:
|
||||
minipass: 3.3.6
|
||||
yallist: 4.0.0
|
||||
|
||||
minizlib@3.0.1:
|
||||
dependencies:
|
||||
minipass: 7.1.2
|
||||
rimraf: 5.0.10
|
||||
|
||||
mkdirp-classic@0.5.3: {}
|
||||
|
||||
mkdirp@1.0.4: {}
|
||||
|
||||
mkdirp@3.0.1: {}
|
||||
|
||||
ml-array-mean@1.1.6:
|
||||
dependencies:
|
||||
ml-array-sum: 1.1.6
|
||||
@@ -6349,25 +6597,21 @@ snapshots:
|
||||
dependencies:
|
||||
mimic-fn: 2.1.0
|
||||
|
||||
onnx-proto@4.0.4:
|
||||
onnxruntime-common@1.20.1: {}
|
||||
|
||||
onnxruntime-node@1.20.1:
|
||||
dependencies:
|
||||
protobufjs: 6.11.4
|
||||
onnxruntime-common: 1.20.1
|
||||
tar: 7.4.3
|
||||
|
||||
onnxruntime-common@1.14.0: {}
|
||||
|
||||
onnxruntime-node@1.14.0:
|
||||
dependencies:
|
||||
onnxruntime-common: 1.14.0
|
||||
optional: true
|
||||
|
||||
onnxruntime-web@1.14.0:
|
||||
onnxruntime-web@1.20.1:
|
||||
dependencies:
|
||||
flatbuffers: 1.12.0
|
||||
guid-typescript: 1.0.9
|
||||
long: 4.0.0
|
||||
onnx-proto: 4.0.4
|
||||
onnxruntime-common: 1.14.0
|
||||
long: 5.2.3
|
||||
onnxruntime-common: 1.20.1
|
||||
platform: 1.3.6
|
||||
protobufjs: 7.4.0
|
||||
|
||||
open@8.4.2:
|
||||
dependencies:
|
||||
@@ -6581,7 +6825,7 @@ snapshots:
|
||||
kleur: 3.0.3
|
||||
sisteransi: 1.0.5
|
||||
|
||||
protobufjs@6.11.4:
|
||||
protobufjs@7.4.0:
|
||||
dependencies:
|
||||
'@protobufjs/aspromise': 1.1.2
|
||||
'@protobufjs/base64': 1.1.2
|
||||
@@ -6593,9 +6837,8 @@ snapshots:
|
||||
'@protobufjs/path': 1.1.2
|
||||
'@protobufjs/pool': 1.1.0
|
||||
'@protobufjs/utf8': 1.1.0
|
||||
'@types/long': 4.0.2
|
||||
'@types/node': 18.19.7
|
||||
long: 4.0.0
|
||||
long: 5.2.3
|
||||
|
||||
proxy-addr@2.0.7:
|
||||
dependencies:
|
||||
@@ -6741,6 +6984,10 @@ snapshots:
|
||||
dependencies:
|
||||
glob: 9.3.5
|
||||
|
||||
rimraf@5.0.10:
|
||||
dependencies:
|
||||
glob: 10.3.10
|
||||
|
||||
rollup@3.29.4:
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.3
|
||||
@@ -6803,6 +7050,8 @@ snapshots:
|
||||
dependencies:
|
||||
lru-cache: 6.0.0
|
||||
|
||||
semver@7.6.3: {}
|
||||
|
||||
send@0.18.0:
|
||||
dependencies:
|
||||
debug: 2.6.9
|
||||
@@ -6855,6 +7104,32 @@ snapshots:
|
||||
tar-fs: 3.0.4
|
||||
tunnel-agent: 0.6.0
|
||||
|
||||
sharp@0.33.5:
|
||||
dependencies:
|
||||
color: 4.2.3
|
||||
detect-libc: 2.0.3
|
||||
semver: 7.6.3
|
||||
optionalDependencies:
|
||||
'@img/sharp-darwin-arm64': 0.33.5
|
||||
'@img/sharp-darwin-x64': 0.33.5
|
||||
'@img/sharp-libvips-darwin-arm64': 1.0.4
|
||||
'@img/sharp-libvips-darwin-x64': 1.0.4
|
||||
'@img/sharp-libvips-linux-arm': 1.0.5
|
||||
'@img/sharp-libvips-linux-arm64': 1.0.4
|
||||
'@img/sharp-libvips-linux-s390x': 1.0.4
|
||||
'@img/sharp-libvips-linux-x64': 1.0.4
|
||||
'@img/sharp-libvips-linuxmusl-arm64': 1.0.4
|
||||
'@img/sharp-libvips-linuxmusl-x64': 1.0.4
|
||||
'@img/sharp-linux-arm': 0.33.5
|
||||
'@img/sharp-linux-arm64': 0.33.5
|
||||
'@img/sharp-linux-s390x': 0.33.5
|
||||
'@img/sharp-linux-x64': 0.33.5
|
||||
'@img/sharp-linuxmusl-arm64': 0.33.5
|
||||
'@img/sharp-linuxmusl-x64': 0.33.5
|
||||
'@img/sharp-wasm32': 0.33.5
|
||||
'@img/sharp-win32-ia32': 0.33.5
|
||||
'@img/sharp-win32-x64': 0.33.5
|
||||
|
||||
shebang-command@2.0.0:
|
||||
dependencies:
|
||||
shebang-regex: 3.0.0
|
||||
@@ -7126,6 +7401,15 @@ snapshots:
|
||||
mkdirp: 1.0.4
|
||||
yallist: 4.0.0
|
||||
|
||||
tar@7.4.3:
|
||||
dependencies:
|
||||
'@isaacs/fs-minipass': 4.0.1
|
||||
chownr: 3.0.0
|
||||
minipass: 7.1.2
|
||||
minizlib: 3.0.1
|
||||
mkdirp: 3.0.1
|
||||
yallist: 5.0.0
|
||||
|
||||
temp-dir@2.0.0: {}
|
||||
|
||||
tempy@1.0.1:
|
||||
@@ -7422,6 +7706,8 @@ snapshots:
|
||||
|
||||
yallist@4.0.0: {}
|
||||
|
||||
yallist@5.0.0: {}
|
||||
|
||||
yaml@2.4.2: {}
|
||||
|
||||
yargs-parser@18.1.3:
|
||||
|
||||
@@ -28,8 +28,15 @@
|
||||
|
||||
<span class="text-textcolor text-lg">Model</span>
|
||||
<SelectInput bind:value={model}>
|
||||
<OptionInput value="MiniLM">MiniLM L6 v2</OptionInput>
|
||||
<OptionInput value="nomic">Nomic Embed Text v1.5</OptionInput>
|
||||
<OptionInput value="MiniLM">MiniLM L6 v2 (CPU)</OptionInput>
|
||||
<OptionInput value="nomic">Nomic Embed Text v1.5 (CPU)</OptionInput>
|
||||
<OptionInput value="nomicGPU">Nomic Embed Text v1.5 (GPU)</OptionInput>
|
||||
<OptionInput value="bgeSmallEn">BGE Small English (CPU)</OptionInput>
|
||||
<OptionInput value="bgeSmallEnGPU">BGE Small English (GPU)</OptionInput>
|
||||
<OptionInput value="bgem3">BGE Medium 3 (CPU)</OptionInput>
|
||||
<OptionInput value="bgem3GPU">BGE Medium 3 (GPU)</OptionInput>
|
||||
<OptionInput value="openai3small">OpenAI text-embedding-3-small</OptionInput>
|
||||
<OptionInput value="openai3large">OpenAI text-embedding-3-large</OptionInput>
|
||||
<OptionInput value="custom">Custom (OpenAI-compatible)</OptionInput>
|
||||
</SelectInput>
|
||||
|
||||
|
||||
@@ -6,9 +6,12 @@
|
||||
let input = $state("");
|
||||
let output = $state("");
|
||||
let outputLength = $state(0);
|
||||
let time = $state(0)
|
||||
const onInput = async () => {
|
||||
try {
|
||||
const start = performance.now();
|
||||
const tokenized = await encode(input);
|
||||
time = performance.now() - start;
|
||||
const tokenizedNumArray = Array.from(tokenized)
|
||||
outputLength = tokenizedNumArray.length;
|
||||
output = JSON.stringify(tokenizedNumArray);
|
||||
@@ -29,3 +32,4 @@
|
||||
<TextAreaInput value={output} />
|
||||
|
||||
<span class="text-textcolor2 text-lg">{outputLength} {language.tokens}</span>
|
||||
<span class="text-textcolor2 text-lg">{time} ms</span>
|
||||
|
||||
@@ -426,12 +426,6 @@
|
||||
{/if}
|
||||
<span class="text-textcolor">{language.summarizationPrompt} <Help key="summarizationPrompt" /></span>
|
||||
<TextAreaInput size="sm" bind:value={DBState.db.supaMemoryPrompt} placeholder="Leave it blank to use default"/>
|
||||
<span class="text-textcolor">{language.HypaMemory} Model</span>
|
||||
<SelectInput className="mt-2 mb-2" bind:value={DBState.db.hypaModel}>
|
||||
<OptionInput value="MiniLM">MiniLM-L6-v2 (Free / Local)</OptionInput>
|
||||
<OptionInput value="nomic">Nomic (Free / Local)</OptionInput>
|
||||
<OptionInput value="ada">OpenAI Ada (Davinci / Curie Only)</OptionInput>
|
||||
</SelectInput>
|
||||
<span class="text-textcolor">{language.hypaChunkSize}</span>
|
||||
<NumberInput size="sm" marginBottom bind:value={DBState.db.hypaChunkSize} min={100} />
|
||||
<span class="text-textcolor">{language.hypaAllocatedTokens}</span>
|
||||
@@ -454,17 +448,31 @@
|
||||
<span class="text-textcolor">{language.SuperMemory} Prompt</span>
|
||||
<TextInput size="sm" marginBottom bind:value={DBState.db.supaMemoryPrompt} placeholder="Leave it blank to use default"/>
|
||||
{/if}
|
||||
{#if DBState.db.hypaMemory}
|
||||
<span class="text-textcolor">{language.HypaMemory} Model</span>
|
||||
<SelectInput className="mt-2 mb-2" bind:value={DBState.db.hypaModel}>
|
||||
<OptionInput value="MiniLM" >MiniLM-L6-v2 (Free / Local)</OptionInput>
|
||||
<OptionInput value="ada" >OpenAI Ada (Davinci / Curie Only)</OptionInput>
|
||||
</SelectInput>
|
||||
{/if}
|
||||
<div class="flex">
|
||||
<Check bind:check={DBState.db.hypaMemory} name={language.enable + ' ' + language.HypaMemory}/>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<span class="text-textcolor">{language.embedding}</span>
|
||||
<SelectInput className="mt-2 mb-2" bind:value={DBState.db.hypaModel}>
|
||||
{#if 'gpu' in navigator}
|
||||
<OptionInput value="nomicGPU">Nomic Embed Text v1.5 (GPU)</OptionInput>
|
||||
<OptionInput value="bgeSmallEnGPU">BGE Small English (GPU)</OptionInput>
|
||||
<OptionInput value="bgem3GPU">BGE Medium 3 (GPU)</OptionInput>
|
||||
{/if}
|
||||
<OptionInput value="MiniLM">MiniLM L6 v2 (CPU)</OptionInput>
|
||||
<OptionInput value="nomic">Nomic Embed Text v1.5 (CPU)</OptionInput>
|
||||
<OptionInput value="bgeSmallEn">BGE Small English (CPU)</OptionInput>
|
||||
<OptionInput value="bgem3">BGE Medium 3 (CPU)</OptionInput>
|
||||
<OptionInput value="openai3small">OpenAI text-embedding-3-small</OptionInput>
|
||||
<OptionInput value="openai3large">OpenAI text-embedding-3-large</OptionInput>
|
||||
<OptionInput value="ada">OpenAI Ada</OptionInput>
|
||||
</SelectInput>
|
||||
|
||||
{#if DBState.db.hypaModel === 'openai3small' || DBState.db.hypaModel === 'openai3large' || DBState.db.hypaModel === 'ada'}
|
||||
<span class="text-textcolor">OpenAI API Key</span>
|
||||
<TextInput size="sm" marginBottom bind:value={DBState.db.supaMemoryKey}/>
|
||||
{/if}
|
||||
|
||||
</Arcodion>
|
||||
{/if}
|
||||
@@ -3,7 +3,7 @@ import { HypaProcesser } from '../memory/hypamemory'
|
||||
import { getUserName } from "src/ts/util";
|
||||
|
||||
export async function additionalInformations(char: character,chats:Chat,){
|
||||
const processer = new HypaProcesser('MiniLM')
|
||||
const processer = new HypaProcesser()
|
||||
const db = getDatabase()
|
||||
|
||||
const info = char.additionalText
|
||||
|
||||
@@ -124,7 +124,7 @@ async function sendPDFFile(arg:sendFileArg) {
|
||||
}
|
||||
}
|
||||
console.log(texts)
|
||||
const hypa = new HypaProcesser('MiniLM')
|
||||
const hypa = new HypaProcesser()
|
||||
hypa.addText(texts)
|
||||
const result = await hypa.similaritySearch(arg.query)
|
||||
let message = ''
|
||||
@@ -142,7 +142,7 @@ async function sendTxtFile(arg:sendFileArg) {
|
||||
const lines = arg.file.split('\n').filter((a) => {
|
||||
return a !== ''
|
||||
})
|
||||
const hypa = new HypaProcesser('MiniLM')
|
||||
const hypa = new HypaProcesser()
|
||||
hypa.addText(lines)
|
||||
const result = await hypa.similaritySearch(arg.query)
|
||||
let message = ''
|
||||
@@ -157,7 +157,7 @@ async function sendTxtFile(arg:sendFileArg) {
|
||||
}
|
||||
|
||||
async function sendXMLFile(arg:sendFileArg) {
|
||||
const hypa = new HypaProcesser('MiniLM')
|
||||
const hypa = new HypaProcesser()
|
||||
let nodeTexts:string[] = []
|
||||
const parser = new DOMParser();
|
||||
const xmlDoc = parser.parseFromString(arg.file, "text/xml");
|
||||
|
||||
@@ -1394,7 +1394,7 @@ export async function sendChat(chatProcessIndex = -1,arg:{
|
||||
}
|
||||
|
||||
if(DBState.db.emotionProcesser === 'embedding'){
|
||||
const hypaProcesser = new HypaProcesser('MiniLM')
|
||||
const hypaProcesser = new HypaProcesser()
|
||||
await hypaProcesser.addText(emotionList.map((v) => 'emotion:' + v))
|
||||
let searched = (await hypaProcesser.similaritySearchScored(result)).map((v) => {
|
||||
v[0] = v[0].replace("emotion:",'')
|
||||
|
||||
@@ -213,7 +213,7 @@ export async function runLua(code:string, arg:{
|
||||
if(!LuaLowLevelIds.has(id)){
|
||||
return
|
||||
}
|
||||
const processer = new HypaProcesser('MiniLM')
|
||||
const processer = new HypaProcesser()
|
||||
await processer.addText(value)
|
||||
return await processer.similaritySearch(source)
|
||||
})
|
||||
|
||||
@@ -13,7 +13,7 @@ export async function hanuraiMemory(chats:OpenAIChat[],arg:{
|
||||
}){
|
||||
const db = getDatabase()
|
||||
const tokenizer = arg.tokenizer
|
||||
const processer = new HypaProcesser('MiniLM')
|
||||
const processer = new HypaProcesser()
|
||||
let addTexts:string[] = []
|
||||
const queryStartIndex=chats.length-maxRecentChatQuery
|
||||
console.log(chats.length,maxRecentChatQuery,queryStartIndex)
|
||||
|
||||
@@ -3,21 +3,47 @@ import { globalFetch } from "src/ts/globalApi.svelte";
|
||||
import { runEmbedding } from "../transformers";
|
||||
import { alertError } from "src/ts/alert";
|
||||
import { appendLastPath } from "src/ts/util";
|
||||
import { getDatabase } from "src/ts/storage/database.svelte";
|
||||
|
||||
|
||||
export type HypaModel = 'ada'|'MiniLM'|'nomic'|'custom'|'nomicGPU'|'bgeSmallEn'|'bgeSmallEnGPU'|'bgem3'|'bgem3GPU'|'openai3small'|'openai3large'
|
||||
|
||||
const localModels = {
|
||||
models: {
|
||||
'MiniLM':'Xenova/all-MiniLM-L6-v2',
|
||||
'nomic':'nomic-ai/nomic-embed-text-v1.5',
|
||||
'nomicGPU':'nomic-ai/nomic-embed-text-v1.5',
|
||||
'bgeSmallEn': 'BAAI/bge-small-en-v1.5',
|
||||
'bgeSmallEnGPU': 'BAAI/bge-small-en-v1.5',
|
||||
'bgem3': 'BAAI/bge-m3',
|
||||
'bgem3GPU': 'BAAI/bge-m3',
|
||||
},
|
||||
gpuModels:[
|
||||
'nomicGPU',
|
||||
'bgeSmallEnGPU',
|
||||
'bgem3GPU'
|
||||
]
|
||||
}
|
||||
|
||||
export class HypaProcesser{
|
||||
oaikey:string
|
||||
vectors:memoryVector[]
|
||||
forage:LocalForage
|
||||
model:'ada'|'MiniLM'|'nomic'|'custom'
|
||||
model:HypaModel
|
||||
customEmbeddingUrl:string
|
||||
|
||||
constructor(model:'ada'|'MiniLM'|'nomic'|'custom',customEmbeddingUrl?:string){
|
||||
constructor(model:HypaModel|'auto' = 'auto',customEmbeddingUrl?:string){
|
||||
this.forage = localforage.createInstance({
|
||||
name: "hypaVector"
|
||||
})
|
||||
this.vectors = []
|
||||
this.model = model
|
||||
if(model === 'auto'){
|
||||
const db = getDatabase()
|
||||
this.model = db.hypaModel || 'MiniLM'
|
||||
}
|
||||
else{
|
||||
this.model = model
|
||||
}
|
||||
this.customEmbeddingUrl = customEmbeddingUrl
|
||||
}
|
||||
|
||||
@@ -39,9 +65,9 @@ export class HypaProcesser{
|
||||
|
||||
|
||||
async getEmbeds(input:string[]|string):Promise<VectorArray[]> {
|
||||
if(this.model === 'MiniLM' || this.model === 'nomic'){
|
||||
if(Object.keys(localModels.models).includes(this.model)){
|
||||
const inputs:string[] = Array.isArray(input) ? input : [input]
|
||||
let results:Float32Array[] = await runEmbedding(inputs, this.model === 'nomic' ? 'nomic-ai/nomic-embed-text-v1.5' : 'Xenova/all-MiniLM-L6-v2')
|
||||
let results:Float32Array[] = await runEmbedding(inputs, localModels.models[this.model], localModels.gpuModels.includes(this.model) ? 'webgpu' : 'wasm')
|
||||
return results
|
||||
}
|
||||
let gf = null;
|
||||
@@ -58,14 +84,21 @@ export class HypaProcesser{
|
||||
},
|
||||
})
|
||||
}
|
||||
if(this.model === 'ada'){
|
||||
if(this.model === 'ada' || this.model === 'openai3small' || this.model === 'openai3large'){
|
||||
const db = getDatabase()
|
||||
const models = {
|
||||
'ada':'text-embedding-ada-002',
|
||||
'openai3small':'text-embedding-3-small',
|
||||
'openai3large':'text-embedding-3-large'
|
||||
}
|
||||
|
||||
gf = await globalFetch("https://api.openai.com/v1/embeddings", {
|
||||
headers: {
|
||||
"Authorization": "Bearer " + this.oaikey
|
||||
"Authorization": "Bearer " + db.supaMemoryKey || this.oaikey
|
||||
},
|
||||
body: {
|
||||
"input": input,
|
||||
"model": "text-embedding-ada-002"
|
||||
"input": input,
|
||||
"model": models[this.model]
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -321,7 +321,7 @@ export async function processScriptFull(char:character|groupChat|simpleCharacter
|
||||
}
|
||||
}
|
||||
|
||||
const processer = new HypaProcesser('MiniLM')
|
||||
const processer = new HypaProcesser()
|
||||
await processer.addText(assetNames)
|
||||
const matches = data.matchAll(assetRegex)
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {env, AutoTokenizer, pipeline, type SummarizationOutput, type TextGenerationConfig, type TextGenerationOutput, FeatureExtractionPipeline, TextToAudioPipeline, type ImageToTextOutput } from '@xenova/transformers';
|
||||
import {env, AutoTokenizer, pipeline, type SummarizationOutput, type TextGenerationConfig, type TextGenerationOutput, FeatureExtractionPipeline, TextToAudioPipeline, type ImageToTextOutput } from '@huggingface/transformers';
|
||||
import { unzip } from 'fflate';
|
||||
import { globalFetch, loadAsset, saveAsset } from 'src/ts/globalApi.svelte';
|
||||
import { selectSingleFile } from 'src/ts/util';
|
||||
@@ -15,6 +15,7 @@ async function initTransformers(){
|
||||
env.useBrowserCache = false
|
||||
env.useFSCache = false
|
||||
env.useCustomCache = true
|
||||
env.allowLocalModels = true
|
||||
env.customCache = {
|
||||
put: async (url:URL|string, response:Response) => {
|
||||
await tfCache.put(url, response)
|
||||
@@ -33,10 +34,12 @@ async function initTransformers(){
|
||||
console.log('transformers loaded')
|
||||
}
|
||||
|
||||
export const runTransformers = async (baseText:string, model:string,config:TextGenerationConfig = {}) => {
|
||||
export const runTransformers = async (baseText:string, model:string,config:TextGenerationConfig, device:'webgpu'|'wasm' = 'wasm') => {
|
||||
await initTransformers()
|
||||
let text = baseText
|
||||
let generator = await pipeline('text-generation', model);
|
||||
let generator = await pipeline('text-generation', model, {
|
||||
device
|
||||
});
|
||||
let output = await generator(text, config) as TextGenerationOutput
|
||||
const outputOne = output[0]
|
||||
return outputOne
|
||||
@@ -50,16 +53,25 @@ export const runSummarizer = async (text: string) => {
|
||||
}
|
||||
|
||||
let extractor:FeatureExtractionPipeline = null
|
||||
let lastEmbeddingModelQuery:string = ''
|
||||
type EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'|'nomic-ai/nomic-embed-text-v1.5'
|
||||
export const runEmbedding = async (texts: string[], model:EmbeddingModel = 'Xenova/all-MiniLM-L6-v2'):Promise<Float32Array[]> => {
|
||||
export const runEmbedding = async (texts: string[], model:EmbeddingModel = 'Xenova/all-MiniLM-L6-v2', device:'webgpu'|'wasm'):Promise<Float32Array[]> => {
|
||||
await initTransformers()
|
||||
if(!extractor){
|
||||
extractor = await pipeline('feature-extraction', model);
|
||||
console.log('running embedding')
|
||||
let embeddingModelQuery = model + device
|
||||
if(!extractor || embeddingModelQuery !== lastEmbeddingModelQuery){
|
||||
extractor = await pipeline('feature-extraction', model, {
|
||||
device: device,
|
||||
progress_callback: (progress) => {
|
||||
console.log(progress)
|
||||
}
|
||||
});
|
||||
console.log('extractor loaded')
|
||||
}
|
||||
let result = await extractor(texts, { pooling: 'mean', normalize: true });
|
||||
console.log(texts, result)
|
||||
const data = result.data as Float32Array
|
||||
|
||||
console.log(data)
|
||||
const lenPerText = data.length / texts.length
|
||||
let res:Float32Array[] = []
|
||||
for(let i = 0; i < texts.length; i++){
|
||||
|
||||
@@ -459,7 +459,7 @@ export async function runTrigger(char:character,mode:triggerMode, arg:{
|
||||
break
|
||||
}
|
||||
|
||||
const processer = new HypaProcesser('MiniLM')
|
||||
const processer = new HypaProcesser()
|
||||
const effectValue = risuChatParser(effect.value,{chara:char})
|
||||
const source = risuChatParser(effect.source,{chara:char})
|
||||
await processer.addText(effectValue.split('§'))
|
||||
|
||||
@@ -699,7 +699,7 @@ export interface Database{
|
||||
colorSchemeName:string
|
||||
promptTemplate?:PromptItem[]
|
||||
forceProxyAsOpenAI?:boolean
|
||||
hypaModel:'ada'|'MiniLM'
|
||||
hypaModel:HypaModel
|
||||
saveTime?:number
|
||||
mancerHeader:string
|
||||
emotionProcesser:'submodel'|'embedding',
|
||||
@@ -1593,6 +1593,7 @@ import { decodeRPack, encodeRPack } from '../rpack/rpack_bg';
|
||||
import { DBState, selectedCharID } from '../stores.svelte';
|
||||
import { LLMFlags, LLMFormat } from '../model/modellist';
|
||||
import type { Parameter } from '../process/request';
|
||||
import type { HypaModel } from '../process/memory/hypamemory';
|
||||
|
||||
export async function downloadPreset(id:number, type:'json'|'risupreset'|'return' = 'json'){
|
||||
saveCurrentPreset()
|
||||
|
||||
Reference in New Issue
Block a user