diff --git a/package.json b/package.json
index e0f795ef..1bdba617 100644
--- a/package.json
+++ b/package.json
@@ -17,6 +17,7 @@
"@dqbd/tiktoken": "^1.0.4",
"@msgpack/msgpack": "3.0.0-beta2",
"@tauri-apps/api": "1.3.0",
+ "@xenova/transformers": "^2.1.1",
"body-parser": "^1.20.2",
"buffer": "^6.0.3",
"core-js": "^3.30.2",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 54c8b4d2..8e322b1e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -10,6 +10,9 @@ dependencies:
'@tauri-apps/api':
specifier: 1.3.0
version: 1.3.0
+ '@xenova/transformers':
+ specifier: ^2.1.1
+ version: 2.1.1
body-parser:
specifier: ^1.20.2
version: 1.20.2
@@ -412,6 +415,49 @@ packages:
fastq: 1.15.0
dev: true
+ /@protobufjs/aspromise@1.1.2:
+ resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
+ dev: false
+
+ /@protobufjs/base64@1.1.2:
+ resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==}
+ dev: false
+
+ /@protobufjs/codegen@2.0.4:
+ resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==}
+ dev: false
+
+ /@protobufjs/eventemitter@1.1.0:
+ resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==}
+ dev: false
+
+ /@protobufjs/fetch@1.1.0:
+ resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==}
+ dependencies:
+ '@protobufjs/aspromise': 1.1.2
+ '@protobufjs/inquire': 1.1.0
+ dev: false
+
+ /@protobufjs/float@1.0.2:
+ resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==}
+ dev: false
+
+ /@protobufjs/inquire@1.1.0:
+ resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==}
+ dev: false
+
+ /@protobufjs/path@1.1.2:
+ resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==}
+ dev: false
+
+ /@protobufjs/pool@1.1.0:
+ resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==}
+ dev: false
+
+ /@protobufjs/utf8@1.1.0:
+ resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==}
+ dev: false
+
/@rollup/plugin-virtual@3.0.1(rollup@3.21.3):
resolution: {integrity: sha512-fK8O0IL5+q+GrsMLuACVNk2x21g3yaw+sG2qn16SnUd3IlBsQyvWxLMGHmCmXRMecPjGRSZ/1LmZB4rjQm68og==}
engines: {node: '>=14.0.0'}
@@ -707,6 +753,10 @@ packages:
resolution: {integrity: sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==}
dev: true
+ /@types/long@4.0.2:
+ resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==}
+ dev: false
+
/@types/node@18.15.11:
resolution: {integrity: sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==}
@@ -742,6 +792,15 @@ packages:
resolution: {integrity: sha512-6hogE75Hl2Ov/jgp8ZhDaGmIF/q3J07GtXf8nCJCwKTHq7971po5+DId7grft09zG7plBwpF6ZU0yx9Du4/e1A==}
dev: true
+ /@xenova/transformers@2.1.1:
+ resolution: {integrity: sha512-6DTz2ob46C/CqL8bmtFIUSvI9Nfi3d4HUF4fHf1hV1J6BFE2n8rNhqZkmTgo/rHhOmaliGAeRxHjZvRXeSqZRQ==}
+ dependencies:
+ onnxruntime-web: 1.14.0
+ sharp: 0.32.1
+ optionalDependencies:
+ onnxruntime-node: 1.14.0
+ dev: false
+
/abab@2.0.6:
resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==}
dev: false
@@ -838,6 +897,14 @@ packages:
engines: {node: '>=8'}
dev: true
+ /bl@4.1.0:
+ resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
+ dependencies:
+ buffer: 5.7.1
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ dev: false
+
/body-parser@1.20.1:
resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -911,6 +978,13 @@ packages:
resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
dev: true
+ /buffer@5.7.1:
+ resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
+ dependencies:
+ base64-js: 1.5.1
+ ieee754: 1.2.1
+ dev: false
+
/buffer@6.0.3:
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
dependencies:
@@ -959,9 +1033,34 @@ packages:
fsevents: 2.3.2
dev: true
+ /chownr@1.1.4:
+ resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
+ dev: false
+
+ /color-convert@2.0.1:
+ resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
+ engines: {node: '>=7.0.0'}
+ dependencies:
+ color-name: 1.1.4
+ dev: false
+
/color-name@1.1.4:
resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
- dev: true
+
+ /color-string@1.9.1:
+ resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
+ dependencies:
+ color-name: 1.1.4
+ simple-swizzle: 0.2.2
+ dev: false
+
+ /color@4.2.3:
+ resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==}
+ engines: {node: '>=12.5.0'}
+ dependencies:
+ color-convert: 2.0.1
+ color-string: 1.9.1
+ dev: false
/combined-stream@1.0.8:
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
@@ -1095,6 +1194,18 @@ packages:
resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==}
dev: false
+ /decompress-response@6.0.0:
+ resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
+ engines: {node: '>=10'}
+ dependencies:
+ mimic-response: 3.1.0
+ dev: false
+
+ /deep-extend@0.6.0:
+ resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
+ engines: {node: '>=4.0.0'}
+ dev: false
+
/deep-is@0.1.4:
resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==}
dev: false
@@ -1131,6 +1242,11 @@ packages:
engines: {node: '>=8'}
dev: true
+ /detect-libc@2.0.1:
+ resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==}
+ engines: {node: '>=8'}
+ dev: false
+
/didyoumean@1.2.2:
resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==}
dev: true
@@ -1190,6 +1306,12 @@ packages:
engines: {node: '>= 0.8'}
dev: false
+ /end-of-stream@1.4.4:
+ resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
+ dependencies:
+ once: 1.4.0
+ dev: false
+
/entities@4.4.0:
resolution: {integrity: sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==}
engines: {node: '>=0.12'}
@@ -1290,6 +1412,11 @@ packages:
resolution: {integrity: sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw==}
dev: false
+ /expand-template@2.0.3:
+ resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
+ engines: {node: '>=6'}
+ dev: false
+
/express@4.18.2:
resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==}
engines: {node: '>= 0.10.0'}
@@ -1372,6 +1499,10 @@ packages:
- supports-color
dev: false
+ /flatbuffers@1.12.0:
+ resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==}
+ dev: false
+
/form-data@4.0.0:
resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
engines: {node: '>= 6'}
@@ -1395,6 +1526,10 @@ packages:
engines: {node: '>= 0.6'}
dev: false
+ /fs-constants@1.0.0:
+ resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
+ dev: false
+
/fs.realpath@1.0.0:
resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
dev: true
@@ -1423,6 +1558,10 @@ packages:
engines: {node: '>=10'}
dev: true
+ /github-from-package@0.0.0:
+ resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
+ dev: false
+
/glob-parent@5.1.2:
resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==}
engines: {node: '>= 6'}
@@ -1474,6 +1613,10 @@ packages:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
dev: true
+ /guid-typescript@1.0.9:
+ resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==}
+ dev: false
+
/has-proto@1.0.1:
resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==}
engines: {node: '>= 0.4'}
@@ -1579,6 +1722,10 @@ packages:
/inherits@2.0.4:
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
+ /ini@1.3.8:
+ resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
+ dev: false
+
/internal-ip@7.0.0:
resolution: {integrity: sha512-qE4TeD4brqC45Vq/+VASeMiS1KRyfBkR6HT2sh9pZVVCzSjPkaCEfKFU+dL0PRv7NHJtvoKN2r82G6wTfzorkw==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -1604,6 +1751,10 @@ packages:
engines: {node: '>= 10'}
dev: true
+ /is-arrayish@0.3.2:
+ resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
+ dev: false
+
/is-binary-path@2.1.0:
resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==}
engines: {node: '>=8'}
@@ -1763,6 +1914,17 @@ packages:
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
dev: false
+ /long@4.0.0:
+ resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==}
+ dev: false
+
+ /lru-cache@6.0.0:
+ resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
+ engines: {node: '>=10'}
+ dependencies:
+ yallist: 4.0.0
+ dev: false
+
/lucide-svelte@0.130.0(svelte@3.58.0):
resolution: {integrity: sha512-KDxJsWucA8p6XnNR5JDLvy3opMyQHWBZd+0mUO+1NI573UHmt7GDETWBynmbAjqavLH/SLT2r6P587aM4IAmPw==}
peerDependencies:
@@ -1839,6 +2001,11 @@ packages:
engines: {node: '>=6'}
dev: true
+ /mimic-response@3.1.0:
+ resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
+ engines: {node: '>=10'}
+ dev: false
+
/min-indent@1.0.1:
resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==}
engines: {node: '>=4'}
@@ -1852,7 +2019,10 @@ packages:
/minimist@1.2.8:
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
- dev: true
+
+ /mkdirp-classic@0.5.3:
+ resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
+ dev: false
/mkdirp@0.5.6:
resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==}
@@ -1895,11 +2065,26 @@ packages:
hasBin: true
dev: true
+ /napi-build-utils@1.0.2:
+ resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==}
+ dev: false
+
/negotiator@0.6.3:
resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==}
engines: {node: '>= 0.6'}
dev: false
+ /node-abi@3.43.0:
+ resolution: {integrity: sha512-QB0MMv+tn9Ur2DtJrc8y09n0n6sw88CyDniWSX2cHW10goQXYPK9ZpFJOktDS4ron501edPX6h9i7Pg+RnH5nQ==}
+ engines: {node: '>=10'}
+ dependencies:
+ semver: 7.5.1
+ dev: false
+
+ /node-addon-api@6.1.0:
+ resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==}
+ dev: false
+
/node-html-parser@6.1.5:
resolution: {integrity: sha512-fAaM511feX++/Chnhe475a0NHD8M7AxDInsqQpz6x63GRF7xYNdS8Vo5dKsIVPgsOvG7eioRRTZQnWBrhDHBSg==}
dependencies:
@@ -1962,7 +2147,6 @@ packages:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
dependencies:
wrappy: 1.0.2
- dev: true
/onetime@5.1.2:
resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==}
@@ -1971,6 +2155,36 @@ packages:
mimic-fn: 2.1.0
dev: true
+ /onnx-proto@4.0.4:
+ resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==}
+ dependencies:
+ protobufjs: 6.11.3
+ dev: false
+
+ /onnxruntime-common@1.14.0:
+ resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==}
+ dev: false
+
+ /onnxruntime-node@1.14.0:
+ resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==}
+ os: [win32, darwin, linux]
+ requiresBuild: true
+ dependencies:
+ onnxruntime-common: 1.14.0
+ dev: false
+ optional: true
+
+ /onnxruntime-web@1.14.0:
+ resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==}
+ dependencies:
+ flatbuffers: 1.12.0
+ guid-typescript: 1.0.9
+ long: 4.0.0
+ onnx-proto: 4.0.4
+ onnxruntime-common: 1.14.0
+ platform: 1.3.6
+ dev: false
+
/optionator@0.8.3:
resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==}
engines: {node: '>= 0.8.0'}
@@ -2060,6 +2274,10 @@ packages:
engines: {node: '>= 6'}
dev: true
+ /platform@1.3.6:
+ resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==}
+ dev: false
+
/png-chunk-text@1.0.0:
resolution: {integrity: sha512-DEROKU3SkkLGWNMzru3xPVgxyd48UGuMSZvioErCure6yhOc/pRH2ZV+SEn7nmaf7WNf3NdIpH+UTrRdKyq9Lw==}
dev: false
@@ -2160,11 +2378,50 @@ packages:
source-map-js: 1.0.2
dev: true
+ /prebuild-install@7.1.1:
+ resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==}
+ engines: {node: '>=10'}
+ hasBin: true
+ dependencies:
+ detect-libc: 2.0.1
+ expand-template: 2.0.3
+ github-from-package: 0.0.0
+ minimist: 1.2.8
+ mkdirp-classic: 0.5.3
+ napi-build-utils: 1.0.2
+ node-abi: 3.43.0
+ pump: 3.0.0
+ rc: 1.2.8
+ simple-get: 4.0.1
+ tar-fs: 2.1.1
+ tunnel-agent: 0.6.0
+ dev: false
+
/prelude-ls@1.1.2:
resolution: {integrity: sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==}
engines: {node: '>= 0.8.0'}
dev: false
+ /protobufjs@6.11.3:
+ resolution: {integrity: sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==}
+ hasBin: true
+ requiresBuild: true
+ dependencies:
+ '@protobufjs/aspromise': 1.1.2
+ '@protobufjs/base64': 1.1.2
+ '@protobufjs/codegen': 2.0.4
+ '@protobufjs/eventemitter': 1.1.0
+ '@protobufjs/fetch': 1.1.0
+ '@protobufjs/float': 1.0.2
+ '@protobufjs/inquire': 1.1.0
+ '@protobufjs/path': 1.1.2
+ '@protobufjs/pool': 1.1.0
+ '@protobufjs/utf8': 1.1.0
+ '@types/long': 4.0.2
+ '@types/node': 18.15.11
+ long: 4.0.0
+ dev: false
+
/proxy-addr@2.0.7:
resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}
engines: {node: '>= 0.10'}
@@ -2177,6 +2434,13 @@ packages:
resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==}
dev: false
+ /pump@3.0.0:
+ resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==}
+ dependencies:
+ end-of-stream: 1.4.4
+ once: 1.4.0
+ dev: false
+
/punycode@2.3.0:
resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==}
engines: {node: '>=6'}
@@ -2227,12 +2491,31 @@ packages:
unpipe: 1.0.0
dev: false
+ /rc@1.2.8:
+ resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}
+ hasBin: true
+ dependencies:
+ deep-extend: 0.6.0
+ ini: 1.3.8
+ minimist: 1.2.8
+ strip-json-comments: 2.0.1
+ dev: false
+
/read-cache@1.0.0:
resolution: {integrity: sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==}
dependencies:
pify: 2.3.0
dev: true
+ /readable-stream@3.6.2:
+ resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
+ engines: {node: '>= 6'}
+ dependencies:
+ inherits: 2.0.4
+ string_decoder: 1.3.0
+ util-deprecate: 1.0.2
+ dev: false
+
/readdirp@3.6.0:
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
engines: {node: '>=8.10.0'}
@@ -2318,6 +2601,14 @@ packages:
xmlchars: 2.2.0
dev: false
+ /semver@7.5.1:
+ resolution: {integrity: sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==}
+ engines: {node: '>=10'}
+ hasBin: true
+ dependencies:
+ lru-cache: 6.0.0
+ dev: false
+
/send@0.18.0:
resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==}
engines: {node: '>= 0.8.0'}
@@ -2355,6 +2646,21 @@ packages:
resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
dev: false
+ /sharp@0.32.1:
+ resolution: {integrity: sha512-kQTFtj7ldpUqSe8kDxoGLZc1rnMFU0AO2pqbX6pLy3b7Oj8ivJIdoKNwxHVQG2HN6XpHPJqCSM2nsma2gOXvOg==}
+ engines: {node: '>=14.15.0'}
+ requiresBuild: true
+ dependencies:
+ color: 4.2.3
+ detect-libc: 2.0.1
+ node-addon-api: 6.1.0
+ prebuild-install: 7.1.1
+ semver: 7.5.1
+ simple-get: 4.0.1
+ tar-fs: 2.1.1
+ tunnel-agent: 0.6.0
+ dev: false
+
/shebang-command@2.0.0:
resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
engines: {node: '>=8'}
@@ -2386,6 +2692,24 @@ packages:
resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
dev: true
+ /simple-concat@1.0.1:
+ resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
+ dev: false
+
+ /simple-get@4.0.1:
+ resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
+ dependencies:
+ decompress-response: 6.0.0
+ once: 1.4.0
+ simple-concat: 1.0.1
+ dev: false
+
+ /simple-swizzle@0.2.2:
+ resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
+ dependencies:
+ is-arrayish: 0.3.2
+ dev: false
+
/sliced@1.0.1:
resolution: {integrity: sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==}
dev: false
@@ -2417,6 +2741,12 @@ packages:
engines: {node: '>= 0.8'}
dev: false
+ /string_decoder@1.3.0:
+ resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
+ dependencies:
+ safe-buffer: 5.2.1
+ dev: false
+
/strip-final-newline@2.0.0:
resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==}
engines: {node: '>=6'}
@@ -2429,6 +2759,11 @@ packages:
min-indent: 1.0.1
dev: true
+ /strip-json-comments@2.0.1:
+ resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}
+ engines: {node: '>=0.10.0'}
+ dev: false
+
/sucrase@3.31.0:
resolution: {integrity: sha512-6QsHnkqyVEzYcaiHsOKkzOtOgdJcb8i54x6AV2hDwyZcY9ZyykGZVw6L/YN98xC0evwTP6utsWWrKRaa8QlfEQ==}
engines: {node: '>=8'}
@@ -2626,6 +2961,26 @@ packages:
- ts-node
dev: true
+ /tar-fs@2.1.1:
+ resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==}
+ dependencies:
+ chownr: 1.1.4
+ mkdirp-classic: 0.5.3
+ pump: 3.0.0
+ tar-stream: 2.2.0
+ dev: false
+
+ /tar-stream@2.2.0:
+ resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
+ engines: {node: '>=6'}
+ dependencies:
+ bl: 4.1.0
+ end-of-stream: 1.4.4
+ fs-constants: 1.0.0
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ dev: false
+
/thenify-all@1.6.0:
resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==}
engines: {node: '>=0.8'}
@@ -2676,6 +3031,12 @@ packages:
resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==}
dev: true
+ /tunnel-agent@0.6.0:
+ resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
+ dependencies:
+ safe-buffer: 5.2.1
+ dev: false
+
/type-check@0.3.2:
resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==}
engines: {node: '>= 0.8.0'}
@@ -2733,7 +3094,6 @@ packages:
/util-deprecate@1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
- dev: true
/utils-merge@1.0.1:
resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==}
@@ -2868,7 +3228,6 @@ packages:
/wrappy@1.0.2:
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
- dev: true
/ws@8.13.0:
resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==}
@@ -2892,6 +3251,10 @@ packages:
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
dev: false
+ /yallist@4.0.0:
+ resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==}
+ dev: false
+
/yaml@1.10.2:
resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==}
engines: {node: '>= 6'}
diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte
index 33a1b07e..36aca3c7 100644
--- a/src/lib/Setting/Pages/BotSettings.svelte
+++ b/src/lib/Setting/Pages/BotSettings.svelte
@@ -101,6 +101,10 @@
Palm2 {language.apiKey}
{/if}
+{#if $DataBase.aiModel === 'novellist' || $DataBase.subModel === 'novellist'}
+ NovelList {language.apiKey}
+
+{/if}
{#if $DataBase.aiModel.startsWith('claude') || $DataBase.subModel.startsWith('claude')}
Claude {language.apiKey}
diff --git a/src/lib/UI/ModelList.svelte b/src/lib/UI/ModelList.svelte
index ddcd9d8a..0b07827f 100644
--- a/src/lib/UI/ModelList.svelte
+++ b/src/lib/UI/ModelList.svelte
@@ -76,6 +76,7 @@
{/each}
{/await}
+
{#if $DataBase.plugins.length > 0}
{/if}
diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts
index 7ebc7917..e3cd18c3 100644
--- a/src/ts/process/request.ts
+++ b/src/ts/process/request.ts
@@ -72,7 +72,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
: aiModel === 'gpt4' ? 'gpt-4' : 'gpt-4-32k',
messages: formated,
temperature: temperature,
- max_tokens: arg.maxTokens ?? maxTokens,
+ max_tokens: maxTokens,
presence_penalty: arg.PresensePenalty ?? (db.PresensePenalty / 100),
frequency_penalty: arg.frequencyPenalty ?? (db.frequencyPenalty / 100),
logit_bias: bias,
@@ -460,6 +460,44 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model'
result: data.results[0].text
}
}
+ case "novellist":{
+ const auth_key = db.novellistAPI;
+ const api_server_url = 'https://api.tringpt.com/';
+
+ const headers = {
+ 'Authorization': `Bearer ${auth_key}`,
+ 'Content-Type': 'application/json'
+ };
+
+ const send_body = {
+ text: stringlizeChat(formated, currentChar?.name ?? ''),
+ length: maxTokens,
+ temperature: temperature,
+ top_p: 0.7,
+ tailfree: 1.0,
+ rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100),
+ };
+
+ const response = await globalFetch(api_server_url + '/api', {
+ method: 'POST',
+ headers: headers,
+ body: send_body,
+ });
+
+ if(!response.ok){
+ return {
+ type: 'fail',
+ result: response.data
+ }
+ }
+
+ const result = response.data.data[0];
+
+ return {
+ 'type': 'success',
+ 'result': result
+ }
+ }
default:{
if(aiModel.startsWith('claude')){
for(let i=0;i= 200 && da.status < 300)
return {
- ok: da.ok,
+ ok: da.ok && da.status >= 200 && da.status < 300,
data: new Uint8Array(await da.arrayBuffer())
}
}
else{
const dat = await da.json()
- addFetchLog(dat, da.ok)
+ addFetchLog(dat, da.ok && da.status >= 200 && da.status < 300)
return {
- ok: da.ok,
+ ok: da.ok && da.status >= 200 && da.status < 300,
data: dat
}
}
@@ -455,17 +455,17 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri
})
if(arg.rawResponse){
- addFetchLog("Uint8Array Response", da.ok)
+ addFetchLog("Uint8Array Response", da.ok && da.status >= 200 && da.status < 300)
return {
- ok: da.ok,
+ ok: da.ok && da.status >= 200 && da.status < 300,
data: new Uint8Array(await da.arrayBuffer())
}
}
else{
const dat = await da.json()
- addFetchLog(dat, da.ok)
+ addFetchLog(dat, da.ok && da.status >= 200 && da.status < 300)
return {
- ok: da.ok,
+ ok: da.ok && da.status >= 200 && da.status < 300,
data: dat
}
}
@@ -567,9 +567,9 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri
,signal: arg.abortSignal
})
- addFetchLog("Uint8Array Response", da.ok)
+ addFetchLog("Uint8Array Response", da.ok && da.status >= 200 && da.status < 300)
return {
- ok: da.ok,
+ ok: da.ok && da.status >= 200 && da.status < 300,
data: new Uint8Array(await da.arrayBuffer())
}
}
@@ -586,9 +586,9 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri
})
const dat = await da.json()
- addFetchLog(dat, da.ok)
+ addFetchLog(dat, da.ok && da.status >= 200 && da.status < 300)
return {
- ok: da.ok,
+ ok: da.ok && da.status >= 200 && da.status < 300,
data: dat
}
}
diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts
index f8fd0870..32797555 100644
--- a/src/ts/tokenizer.ts
+++ b/src/ts/tokenizer.ts
@@ -1,7 +1,13 @@
import type { Tiktoken } from "@dqbd/tiktoken";
-import type { character } from "./storage/database";
+import { DataBase, type character } from "./storage/database";
+import { get } from "svelte/store";
+import { tokenizeTransformers } from "./transformers/transformer";
async function encode(data:string):Promise<(number[]|Uint32Array)>{
+ let db = get(DataBase)
+ if(db.aiModel === 'novellist'){
+ return await tokenizeTransformers('trin',data)
+ }
return await tikJS(data)
}
diff --git a/src/ts/transformers/transformer.ts b/src/ts/transformers/transformer.ts
new file mode 100644
index 00000000..b95e40b7
--- /dev/null
+++ b/src/ts/transformers/transformer.ts
@@ -0,0 +1,30 @@
+import type { PreTrainedTokenizer } from "@xenova/transformers"
+type transformerLibType = typeof import("@xenova/transformers");
+let tokenizer:PreTrainedTokenizer = null
+let transformerLib:transformerLibType
+
+const tokenizerDict = {
+ 'trin': 'naclbit/trin_tokenizer_v3',
+} as const
+
+type tokenizerTypes = keyof(typeof tokenizerDict)
+
+let tokenizerType:tokenizerTypes|'' = ''
+
+
+async function loadTransformers() {
+ if(!transformerLib){
+ transformerLib = await import('@xenova/transformers')
+ }
+}
+
+export async function tokenizeTransformers(type:tokenizerTypes, text:string) {
+ await loadTransformers()
+ if(tokenizerType !== type){
+ const AutoTokenizer = transformerLib.AutoTokenizer
+ tokenizer = await AutoTokenizer.from_pretrained(tokenizerDict[type])
+ tokenizerType = type
+ }
+
+ return tokenizer.encode(text)
+}