From f37ae13d8381b755758960e05b79f88a4769c311 Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 08:57:42 +0900 Subject: [PATCH 1/7] [feat] novellist api --- package.json | 1 + pnpm-lock.yaml | 373 ++++++++++++++++++++++- src/lib/Setting/Pages/BotSettings.svelte | 4 + src/lib/UI/ModelList.svelte | 1 + src/ts/process/request.ts | 40 ++- src/ts/storage/database.ts | 3 +- src/ts/storage/globalApi.ts | 24 +- src/ts/tokenizer.ts | 8 +- src/ts/transformers/transformer.ts | 30 ++ 9 files changed, 464 insertions(+), 20 deletions(-) create mode 100644 src/ts/transformers/transformer.ts diff --git a/package.json b/package.json index e0f795ef..1bdba617 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@dqbd/tiktoken": "^1.0.4", "@msgpack/msgpack": "3.0.0-beta2", "@tauri-apps/api": "1.3.0", + "@xenova/transformers": "^2.1.1", "body-parser": "^1.20.2", "buffer": "^6.0.3", "core-js": "^3.30.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 54c8b4d2..8e322b1e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,6 +10,9 @@ dependencies: '@tauri-apps/api': specifier: 1.3.0 version: 1.3.0 + '@xenova/transformers': + specifier: ^2.1.1 + version: 2.1.1 body-parser: specifier: ^1.20.2 version: 1.20.2 @@ -412,6 +415,49 @@ packages: fastq: 1.15.0 dev: true + /@protobufjs/aspromise@1.1.2: + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + dev: false + + /@protobufjs/base64@1.1.2: + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + dev: false + + /@protobufjs/codegen@2.0.4: + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + dev: false + + /@protobufjs/eventemitter@1.1.0: + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + dev: false + + /@protobufjs/fetch@1.1.0: + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + dev: false + + /@protobufjs/float@1.0.2: + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + dev: false + + /@protobufjs/inquire@1.1.0: + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + dev: false + + /@protobufjs/path@1.1.2: + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + dev: false + + /@protobufjs/pool@1.1.0: + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + dev: false + + /@protobufjs/utf8@1.1.0: + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + dev: false + /@rollup/plugin-virtual@3.0.1(rollup@3.21.3): resolution: {integrity: sha512-fK8O0IL5+q+GrsMLuACVNk2x21g3yaw+sG2qn16SnUd3IlBsQyvWxLMGHmCmXRMecPjGRSZ/1LmZB4rjQm68og==} engines: {node: '>=14.0.0'} @@ -707,6 +753,10 @@ packages: resolution: {integrity: sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g==} dev: true + /@types/long@4.0.2: + resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} + dev: false + /@types/node@18.15.11: resolution: {integrity: sha512-E5Kwq2n4SbMzQOn6wnmBjuK9ouqlURrcZDVfbo9ftDDTFt3nk7ZKK4GMOzoYgnpQJKcxwQw+lGaBvvlMo0qN/Q==} @@ -742,6 +792,15 @@ packages: resolution: {integrity: sha512-6hogE75Hl2Ov/jgp8ZhDaGmIF/q3J07GtXf8nCJCwKTHq7971po5+DId7grft09zG7plBwpF6ZU0yx9Du4/e1A==} dev: true + /@xenova/transformers@2.1.1: + resolution: {integrity: sha512-6DTz2ob46C/CqL8bmtFIUSvI9Nfi3d4HUF4fHf1hV1J6BFE2n8rNhqZkmTgo/rHhOmaliGAeRxHjZvRXeSqZRQ==} + dependencies: + onnxruntime-web: 1.14.0 + sharp: 0.32.1 + optionalDependencies: + onnxruntime-node: 1.14.0 + dev: false + /abab@2.0.6: resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==} dev: false @@ -838,6 +897,14 @@ packages: engines: {node: '>=8'} dev: true + /bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + dev: false + /body-parser@1.20.1: resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -911,6 +978,13 @@ packages: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} dev: true + /buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + dev: false + /buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} dependencies: @@ -959,9 +1033,34 @@ packages: fsevents: 2.3.2 dev: true + /chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + dev: false + + /color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + dependencies: + color-name: 1.1.4 + dev: false + /color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - dev: true + + /color-string@1.9.1: + resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==} + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.2 + dev: false + + /color@4.2.3: + resolution: {integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==} + engines: {node: '>=12.5.0'} + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + dev: false /combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} @@ -1095,6 +1194,18 @@ packages: resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==} dev: false + /decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + dependencies: + mimic-response: 3.1.0 + dev: false + + /deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + dev: false + /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} dev: false @@ -1131,6 +1242,11 @@ packages: engines: {node: '>=8'} dev: true + /detect-libc@2.0.1: + resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} + engines: {node: '>=8'} + dev: false + /didyoumean@1.2.2: resolution: {integrity: sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==} dev: true @@ -1190,6 +1306,12 @@ packages: engines: {node: '>= 0.8'} dev: false + /end-of-stream@1.4.4: + resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + dependencies: + once: 1.4.0 + dev: false + /entities@4.4.0: resolution: {integrity: sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==} engines: {node: '>=0.12'} @@ -1290,6 +1412,11 @@ packages: resolution: {integrity: sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw==} dev: false + /expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + dev: false + /express@4.18.2: resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} engines: {node: '>= 0.10.0'} @@ -1372,6 +1499,10 @@ packages: - supports-color dev: false + /flatbuffers@1.12.0: + resolution: {integrity: sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ==} + dev: false + /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -1395,6 +1526,10 @@ packages: engines: {node: '>= 0.6'} dev: false + /fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + dev: false + /fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true @@ -1423,6 +1558,10 @@ packages: engines: {node: '>=10'} dev: true + /github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + dev: false + /glob-parent@5.1.2: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} @@ -1474,6 +1613,10 @@ packages: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} dev: true + /guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + dev: false + /has-proto@1.0.1: resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} engines: {node: '>= 0.4'} @@ -1579,6 +1722,10 @@ packages: /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + /ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + dev: false + /internal-ip@7.0.0: resolution: {integrity: sha512-qE4TeD4brqC45Vq/+VASeMiS1KRyfBkR6HT2sh9pZVVCzSjPkaCEfKFU+dL0PRv7NHJtvoKN2r82G6wTfzorkw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -1604,6 +1751,10 @@ packages: engines: {node: '>= 10'} dev: true + /is-arrayish@0.3.2: + resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} + dev: false + /is-binary-path@2.1.0: resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} engines: {node: '>=8'} @@ -1763,6 +1914,17 @@ packages: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} dev: false + /long@4.0.0: + resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} + dev: false + + /lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + dependencies: + yallist: 4.0.0 + dev: false + /lucide-svelte@0.130.0(svelte@3.58.0): resolution: {integrity: sha512-KDxJsWucA8p6XnNR5JDLvy3opMyQHWBZd+0mUO+1NI573UHmt7GDETWBynmbAjqavLH/SLT2r6P587aM4IAmPw==} peerDependencies: @@ -1839,6 +2001,11 @@ packages: engines: {node: '>=6'} dev: true + /mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + dev: false + /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} @@ -1852,7 +2019,10 @@ packages: /minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: true + + /mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + dev: false /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -1895,11 +2065,26 @@ packages: hasBin: true dev: true + /napi-build-utils@1.0.2: + resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} + dev: false + /negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} dev: false + /node-abi@3.43.0: + resolution: {integrity: sha512-QB0MMv+tn9Ur2DtJrc8y09n0n6sw88CyDniWSX2cHW10goQXYPK9ZpFJOktDS4ron501edPX6h9i7Pg+RnH5nQ==} + engines: {node: '>=10'} + dependencies: + semver: 7.5.1 + dev: false + + /node-addon-api@6.1.0: + resolution: {integrity: sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==} + dev: false + /node-html-parser@6.1.5: resolution: {integrity: sha512-fAaM511feX++/Chnhe475a0NHD8M7AxDInsqQpz6x63GRF7xYNdS8Vo5dKsIVPgsOvG7eioRRTZQnWBrhDHBSg==} dependencies: @@ -1962,7 +2147,6 @@ packages: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} dependencies: wrappy: 1.0.2 - dev: true /onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} @@ -1971,6 +2155,36 @@ packages: mimic-fn: 2.1.0 dev: true + /onnx-proto@4.0.4: + resolution: {integrity: sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA==} + dependencies: + protobufjs: 6.11.3 + dev: false + + /onnxruntime-common@1.14.0: + resolution: {integrity: sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew==} + dev: false + + /onnxruntime-node@1.14.0: + resolution: {integrity: sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==} + os: [win32, darwin, linux] + requiresBuild: true + dependencies: + onnxruntime-common: 1.14.0 + dev: false + optional: true + + /onnxruntime-web@1.14.0: + resolution: {integrity: sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==} + dependencies: + flatbuffers: 1.12.0 + guid-typescript: 1.0.9 + long: 4.0.0 + onnx-proto: 4.0.4 + onnxruntime-common: 1.14.0 + platform: 1.3.6 + dev: false + /optionator@0.8.3: resolution: {integrity: sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==} engines: {node: '>= 0.8.0'} @@ -2060,6 +2274,10 @@ packages: engines: {node: '>= 6'} dev: true + /platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + dev: false + /png-chunk-text@1.0.0: resolution: {integrity: sha512-DEROKU3SkkLGWNMzru3xPVgxyd48UGuMSZvioErCure6yhOc/pRH2ZV+SEn7nmaf7WNf3NdIpH+UTrRdKyq9Lw==} dev: false @@ -2160,11 +2378,50 @@ packages: source-map-js: 1.0.2 dev: true + /prebuild-install@7.1.1: + resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} + engines: {node: '>=10'} + hasBin: true + dependencies: + detect-libc: 2.0.1 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 1.0.2 + node-abi: 3.43.0 + pump: 3.0.0 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.1 + tunnel-agent: 0.6.0 + dev: false + /prelude-ls@1.1.2: resolution: {integrity: sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==} engines: {node: '>= 0.8.0'} dev: false + /protobufjs@6.11.3: + resolution: {integrity: sha512-xL96WDdCZYdU7Slin569tFX712BxsxslWwAfAhCYjQKGTq7dAU91Lomy6nLLhh/dyGhk/YH4TwTSRxTzhuHyZg==} + hasBin: true + requiresBuild: true + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/long': 4.0.2 + '@types/node': 18.15.11 + long: 4.0.0 + dev: false + /proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -2177,6 +2434,13 @@ packages: resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} dev: false + /pump@3.0.0: + resolution: {integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==} + dependencies: + end-of-stream: 1.4.4 + once: 1.4.0 + dev: false + /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} @@ -2227,12 +2491,31 @@ packages: unpipe: 1.0.0 dev: false + /rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + dev: false + /read-cache@1.0.0: resolution: {integrity: sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==} dependencies: pify: 2.3.0 dev: true + /readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + dev: false + /readdirp@3.6.0: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} @@ -2318,6 +2601,14 @@ packages: xmlchars: 2.2.0 dev: false + /semver@7.5.1: + resolution: {integrity: sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==} + engines: {node: '>=10'} + hasBin: true + dependencies: + lru-cache: 6.0.0 + dev: false + /send@0.18.0: resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} engines: {node: '>= 0.8.0'} @@ -2355,6 +2646,21 @@ packages: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} dev: false + /sharp@0.32.1: + resolution: {integrity: sha512-kQTFtj7ldpUqSe8kDxoGLZc1rnMFU0AO2pqbX6pLy3b7Oj8ivJIdoKNwxHVQG2HN6XpHPJqCSM2nsma2gOXvOg==} + engines: {node: '>=14.15.0'} + requiresBuild: true + dependencies: + color: 4.2.3 + detect-libc: 2.0.1 + node-addon-api: 6.1.0 + prebuild-install: 7.1.1 + semver: 7.5.1 + simple-get: 4.0.1 + tar-fs: 2.1.1 + tunnel-agent: 0.6.0 + dev: false + /shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -2386,6 +2692,24 @@ packages: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} dev: true + /simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + dev: false + + /simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + dev: false + + /simple-swizzle@0.2.2: + resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==} + dependencies: + is-arrayish: 0.3.2 + dev: false + /sliced@1.0.1: resolution: {integrity: sha512-VZBmZP8WU3sMOZm1bdgTadsQbcscK0UM8oKxKVBs4XAhUo2Xxzm/OFMGBkPusxw9xL3Uy8LrzEqGqJhclsr0yA==} dev: false @@ -2417,6 +2741,12 @@ packages: engines: {node: '>= 0.8'} dev: false + /string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + dependencies: + safe-buffer: 5.2.1 + dev: false + /strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -2429,6 +2759,11 @@ packages: min-indent: 1.0.1 dev: true + /strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + dev: false + /sucrase@3.31.0: resolution: {integrity: sha512-6QsHnkqyVEzYcaiHsOKkzOtOgdJcb8i54x6AV2hDwyZcY9ZyykGZVw6L/YN98xC0evwTP6utsWWrKRaa8QlfEQ==} engines: {node: '>=8'} @@ -2626,6 +2961,26 @@ packages: - ts-node dev: true + /tar-fs@2.1.1: + resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==} + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.0 + tar-stream: 2.2.0 + dev: false + + /tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.4 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + dev: false + /thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -2676,6 +3031,12 @@ packages: resolution: {integrity: sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==} dev: true + /tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + dependencies: + safe-buffer: 5.2.1 + dev: false + /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -2733,7 +3094,6 @@ packages: /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - dev: true /utils-merge@1.0.1: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} @@ -2868,7 +3228,6 @@ packages: /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - dev: true /ws@8.13.0: resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==} @@ -2892,6 +3251,10 @@ packages: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} dev: false + /yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: false + /yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} diff --git a/src/lib/Setting/Pages/BotSettings.svelte b/src/lib/Setting/Pages/BotSettings.svelte index 33a1b07e..36aca3c7 100644 --- a/src/lib/Setting/Pages/BotSettings.svelte +++ b/src/lib/Setting/Pages/BotSettings.svelte @@ -101,6 +101,10 @@ Palm2 {language.apiKey} {/if} +{#if $DataBase.aiModel === 'novellist' || $DataBase.subModel === 'novellist'} + NovelList {language.apiKey} + +{/if} {#if $DataBase.aiModel.startsWith('claude') || $DataBase.subModel.startsWith('claude')} Claude {language.apiKey} diff --git a/src/lib/UI/ModelList.svelte b/src/lib/UI/ModelList.svelte index ddcd9d8a..0b07827f 100644 --- a/src/lib/UI/ModelList.svelte +++ b/src/lib/UI/ModelList.svelte @@ -76,6 +76,7 @@ {/each} {/await} + {#if $DataBase.plugins.length > 0} {/if} diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 7ebc7917..e3cd18c3 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -72,7 +72,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' : aiModel === 'gpt4' ? 'gpt-4' : 'gpt-4-32k', messages: formated, temperature: temperature, - max_tokens: arg.maxTokens ?? maxTokens, + max_tokens: maxTokens, presence_penalty: arg.PresensePenalty ?? (db.PresensePenalty / 100), frequency_penalty: arg.frequencyPenalty ?? (db.frequencyPenalty / 100), logit_bias: bias, @@ -460,6 +460,44 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' result: data.results[0].text } } + case "novellist":{ + const auth_key = db.novellistAPI; + const api_server_url = 'https://api.tringpt.com/'; + + const headers = { + 'Authorization': `Bearer ${auth_key}`, + 'Content-Type': 'application/json' + }; + + const send_body = { + text: stringlizeChat(formated, currentChar?.name ?? ''), + length: maxTokens, + temperature: temperature, + top_p: 0.7, + tailfree: 1.0, + rep_pen: arg.frequencyPenalty ?? (db.frequencyPenalty / 100), + }; + + const response = await globalFetch(api_server_url + '/api', { + method: 'POST', + headers: headers, + body: send_body, + }); + + if(!response.ok){ + return { + type: 'fail', + result: response.data + } + } + + const result = response.data.data[0]; + + return { + 'type': 'success', + 'result': result + } + } default:{ if(aiModel.startsWith('claude')){ for(let i=0;i= 200 && da.status < 300) return { - ok: da.ok, + ok: da.ok && da.status >= 200 && da.status < 300, data: new Uint8Array(await da.arrayBuffer()) } } else{ const dat = await da.json() - addFetchLog(dat, da.ok) + addFetchLog(dat, da.ok && da.status >= 200 && da.status < 300) return { - ok: da.ok, + ok: da.ok && da.status >= 200 && da.status < 300, data: dat } } @@ -455,17 +455,17 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri }) if(arg.rawResponse){ - addFetchLog("Uint8Array Response", da.ok) + addFetchLog("Uint8Array Response", da.ok && da.status >= 200 && da.status < 300) return { - ok: da.ok, + ok: da.ok && da.status >= 200 && da.status < 300, data: new Uint8Array(await da.arrayBuffer()) } } else{ const dat = await da.json() - addFetchLog(dat, da.ok) + addFetchLog(dat, da.ok && da.status >= 200 && da.status < 300) return { - ok: da.ok, + ok: da.ok && da.status >= 200 && da.status < 300, data: dat } } @@ -567,9 +567,9 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri ,signal: arg.abortSignal }) - addFetchLog("Uint8Array Response", da.ok) + addFetchLog("Uint8Array Response", da.ok && da.status >= 200 && da.status < 300) return { - ok: da.ok, + ok: da.ok && da.status >= 200 && da.status < 300, data: new Uint8Array(await da.arrayBuffer()) } } @@ -586,9 +586,9 @@ export async function globalFetch(url:string, arg:{body?:any,headers?:{[key:stri }) const dat = await da.json() - addFetchLog(dat, da.ok) + addFetchLog(dat, da.ok && da.status >= 200 && da.status < 300) return { - ok: da.ok, + ok: da.ok && da.status >= 200 && da.status < 300, data: dat } } diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index f8fd0870..32797555 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -1,7 +1,13 @@ import type { Tiktoken } from "@dqbd/tiktoken"; -import type { character } from "./storage/database"; +import { DataBase, type character } from "./storage/database"; +import { get } from "svelte/store"; +import { tokenizeTransformers } from "./transformers/transformer"; async function encode(data:string):Promise<(number[]|Uint32Array)>{ + let db = get(DataBase) + if(db.aiModel === 'novellist'){ + return await tokenizeTransformers('trin',data) + } return await tikJS(data) } diff --git a/src/ts/transformers/transformer.ts b/src/ts/transformers/transformer.ts new file mode 100644 index 00000000..b95e40b7 --- /dev/null +++ b/src/ts/transformers/transformer.ts @@ -0,0 +1,30 @@ +import type { PreTrainedTokenizer } from "@xenova/transformers" +type transformerLibType = typeof import("@xenova/transformers"); +let tokenizer:PreTrainedTokenizer = null +let transformerLib:transformerLibType + +const tokenizerDict = { + 'trin': 'naclbit/trin_tokenizer_v3', +} as const + +type tokenizerTypes = keyof(typeof tokenizerDict) + +let tokenizerType:tokenizerTypes|'' = '' + + +async function loadTransformers() { + if(!transformerLib){ + transformerLib = await import('@xenova/transformers') + } +} + +export async function tokenizeTransformers(type:tokenizerTypes, text:string) { + await loadTransformers() + if(tokenizerType !== type){ + const AutoTokenizer = transformerLib.AutoTokenizer + tokenizer = await AutoTokenizer.from_pretrained(tokenizerDict[type]) + tokenizerType = type + } + + return tokenizer.encode(text) +} From b631da7e724e7a0ebb737d3868d8b326c8bfb831 Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 09:00:21 +0900 Subject: [PATCH 2/7] [fix] unstringlize novellist --- src/ts/process/request.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index e3cd18c3..e4ae002d 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -495,7 +495,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' return { 'type': 'success', - 'result': result + 'result': unstringlizeChat(result, formated, currentChar?.name ?? '') } } default:{ From 9974ff0df10f873521ca1e1a34232035389652ac Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 09:23:29 +0900 Subject: [PATCH 3/7] [refactor] transformer tokenizer --- src/ts/tokenizer.ts | 2 +- src/ts/transformers/transformer.ts | 12 +++--------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/ts/tokenizer.ts b/src/ts/tokenizer.ts index 32797555..032a9460 100644 --- a/src/ts/tokenizer.ts +++ b/src/ts/tokenizer.ts @@ -6,7 +6,7 @@ import { tokenizeTransformers } from "./transformers/transformer"; async function encode(data:string):Promise<(number[]|Uint32Array)>{ let db = get(DataBase) if(db.aiModel === 'novellist'){ - return await tokenizeTransformers('trin',data) + return await tokenizeTransformers('naclbit/trin_tokenizer_v3',data) } return await tikJS(data) } diff --git a/src/ts/transformers/transformer.ts b/src/ts/transformers/transformer.ts index b95e40b7..893f2666 100644 --- a/src/ts/transformers/transformer.ts +++ b/src/ts/transformers/transformer.ts @@ -3,13 +3,7 @@ type transformerLibType = typeof import("@xenova/transformers"); let tokenizer:PreTrainedTokenizer = null let transformerLib:transformerLibType -const tokenizerDict = { - 'trin': 'naclbit/trin_tokenizer_v3', -} as const - -type tokenizerTypes = keyof(typeof tokenizerDict) - -let tokenizerType:tokenizerTypes|'' = '' +let tokenizerType:string = '' async function loadTransformers() { @@ -18,11 +12,11 @@ async function loadTransformers() { } } -export async function tokenizeTransformers(type:tokenizerTypes, text:string) { +export async function tokenizeTransformers(type:string, text:string) { await loadTransformers() if(tokenizerType !== type){ const AutoTokenizer = transformerLib.AutoTokenizer - tokenizer = await AutoTokenizer.from_pretrained(tokenizerDict[type]) + tokenizer = await AutoTokenizer.from_pretrained(type) tokenizerType = type } From a92e4235c9183cd1002287f2e7b067e3087a2781 Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 09:23:59 +0900 Subject: [PATCH 4/7] [rm] removed novellist from menu, dude to not working --- src/lib/UI/ModelList.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/UI/ModelList.svelte b/src/lib/UI/ModelList.svelte index 0b07827f..ddcd9d8a 100644 --- a/src/lib/UI/ModelList.svelte +++ b/src/lib/UI/ModelList.svelte @@ -76,7 +76,6 @@ {/each} {/await} - {#if $DataBase.plugins.length > 0} {/if} From b211b256c295bd898b14f1e22eea8db869e581dc Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 09:46:23 +0900 Subject: [PATCH 5/7] [feat] better supamemory --- src/ts/process/supaMemory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ts/process/supaMemory.ts b/src/ts/process/supaMemory.ts index 8cdf1892..7e2a96da 100644 --- a/src/ts/process/supaMemory.ts +++ b/src/ts/process/supaMemory.ts @@ -78,7 +78,7 @@ export async function supaMemory( async function summarize(stringlizedChat:string){ const supaPrompt = db.supaMemoryPrompt === '' ? - "[Summarize the ongoing role story, including as many events from the past as possible, using assistant as a narrative helper;do not analyze. include all of the characters' names, statuses, thoughts, relationships, and attire. Be sure to include dialogue exchanges and context by referencing previous statements and reactions. assistant's summary should provide an objective overview of the story while also considering relevant past conversations and events. It must also remove redundancy and unnecessary content from the prompt so that gpt3 and other sublanguage models]\n" + "[Summarize the ongoing role story, It must also remove redundancy and unnecessary content from the prompt so that gpt3 and other sublanguage models]\n" : db.supaMemoryPrompt let result = '' From 8829a9077476b1b954062ba57a59f7f8ce11fe9e Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 09:47:27 +0900 Subject: [PATCH 6/7] [feat] better supamemory 2 --- src/ts/process/supaMemory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ts/process/supaMemory.ts b/src/ts/process/supaMemory.ts index 7e2a96da..d3b5748a 100644 --- a/src/ts/process/supaMemory.ts +++ b/src/ts/process/supaMemory.ts @@ -78,7 +78,7 @@ export async function supaMemory( async function summarize(stringlizedChat:string){ const supaPrompt = db.supaMemoryPrompt === '' ? - "[Summarize the ongoing role story, It must also remove redundancy and unnecessary content from the prompt so that gpt3 and other sublanguage models]\n" + "[Summarize the ongoing role story, It must also remove redundancy and unnecessary text and content from the output to reduce tokens for gpt3 and other sublanguage models]\n" : db.supaMemoryPrompt let result = '' From 833310ba832c6cb393f49712ecbb5a9d2fc00b4e Mon Sep 17 00:00:00 2001 From: kwaroran Date: Wed, 7 Jun 2023 09:48:11 +0900 Subject: [PATCH 7/7] bump version to 1.24.1 --- src-tauri/tauri.conf.json | 2 +- src/ts/storage/database.ts | 2 +- version.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index a14d92b5..4e39e075 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -8,7 +8,7 @@ }, "package": { "productName": "RisuAI", - "version": "1.24.0" + "version": "1.24.1" }, "tauri": { "allowlist": { diff --git a/src/ts/storage/database.ts b/src/ts/storage/database.ts index dc8897e7..fb154fbd 100644 --- a/src/ts/storage/database.ts +++ b/src/ts/storage/database.ts @@ -8,7 +8,7 @@ import { defaultAutoSuggestPrompt, defaultJailbreak, defaultMainPrompt } from '. export const DataBase = writable({} as any as Database) export const loadedStore = writable(false) -export let appVer = '1.24.0' +export let appVer = '1.24.1' export function setDatabase(data:Database){ if(checkNullish(data.characters)){ diff --git a/version.json b/version.json index b55dafdf..5539bf1a 100644 --- a/version.json +++ b/version.json @@ -1 +1 @@ -{"version":"1.24.0"} \ No newline at end of file +{"version":"1.24.1"} \ No newline at end of file