From 93e5b7bdb4dad76ecc96213a34a0be6abf86e2ea Mon Sep 17 00:00:00 2001 From: gdosu Date: Sat, 13 Jan 2024 19:34:49 +0900 Subject: [PATCH 1/5] Fix aws claude --- src/ts/process/request.ts | 52 ++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index a8ce4839..21309e29 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1438,7 +1438,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' const bedrock = db.claudeAws - if(bedrock){ + if(bedrock && aiModel !== 'reverse_proxy'){ function getCredentialParts(key:string) { const [accessKeyId, secretAccessKey, region] = key.split(":"); @@ -1450,14 +1450,41 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' } const { accessKeyId, secretAccessKey, region } = getCredentialParts(apiKey); - const AMZ_HOST = "invoke-bedrock.%REGION%.amazonaws.com"; + const AMZ_HOST = "bedrock-runtime.%REGION%.amazonaws.com"; const host = AMZ_HOST.replace("%REGION%", region); const stream = false - const url = `https://${host}/model/${model}/invoke${stream ? "-with-response-stream" : ""}` + + const LATEST_AWS_V2_MINOR_VERSION = 1; + let awsModel = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`; + + const pattern = /^(claude-)?(instant-)?(v)?(\d+)(\.(\d+))?(-\d+k)?$/i; + const match = raiModel.match(pattern); + + if (match) { + const [, , instant, , major, , minor] = match; + + if (instant) { + awsModel = "anthropic.claude-instant-v1"; + } + + // There's only one v1 model + if (major === "1") { + awsModel = "anthropic.claude-v1"; + } + + // Try to map Anthropic API v2 models to AWS v2 models + if (major === "2") { + if (minor === "0") { + awsModel = "anthropic.claude-v2"; + } + } + } + + const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}` const params = { - prompt : "\n\nHuman: " + requestPrompt, - model: raiModel, + prompt : requestPrompt.startsWith("\n\nHuman: ") ? requestPrompt : "\n\nHuman: " + requestPrompt, + //model: raiModel, max_tokens_to_sample: maxTokens, stop_sequences: ["\n\nHuman:", "\n\nSystem:", "\n\nAssistant:"], temperature: temperature, @@ -1466,12 +1493,12 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' method: "POST", protocol: "https:", hostname: host, - path: `/model/${model}/invoke${stream ? "-with-response-stream" : ""}`, + path: `/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`, headers: { ["Host"]: host, - ["content-type"]: "application/json", + ["Content-Type"]: "application/json", ["accept"]: "application/json", - "anthropic-version": "2023-06-01", + //"anthropic-version": "2023-06-01", }, body: JSON.stringify(params), }); @@ -1486,15 +1513,10 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' const signed = await signer.sign(rq); - const da = await globalFetch(`${signed.protocol}//${signed.hostname}`, { + const da = await globalFetch(url, { method: "POST", body: params, - headers: { - ["Host"]: host, - ["content-type"]: "application/json", - ["accept"]: "application/json", - "anthropic-version": "2023-06-01", - } + headers: signed.headers, }) From 3b9a12899f187d0b77008ba1f45205950ae4dcb8 Mon Sep 17 00:00:00 2001 From: gdosu Date: Sat, 13 Jan 2024 19:56:24 +0900 Subject: [PATCH 2/5] Force plain fetch for aws --- src/ts/process/request.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 21309e29..b3a9ba43 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1501,6 +1501,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' //"anthropic-version": "2023-06-01", }, body: JSON.stringify(params), + plainFetchForce: true }); From 8f3011914af5fa1840204bf312a6620db8543bf2 Mon Sep 17 00:00:00 2001 From: gdosu Date: Sat, 13 Jan 2024 20:13:10 +0900 Subject: [PATCH 3/5] Fix mistake --- src/ts/process/request.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index b3a9ba43..6d98b608 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1501,7 +1501,6 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' //"anthropic-version": "2023-06-01", }, body: JSON.stringify(params), - plainFetchForce: true }); @@ -1518,6 +1517,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' method: "POST", body: params, headers: signed.headers, + plainFetchForce: true }) From ebe7a938ce002b1bce15a0bd325ba43e8b6430a0 Mon Sep 17 00:00:00 2001 From: gdosu Date: Sat, 13 Jan 2024 20:53:03 +0900 Subject: [PATCH 4/5] Fix versioning, applied top-p --- src/ts/process/request.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index 6d98b608..d99bd569 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1462,7 +1462,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' const match = raiModel.match(pattern); if (match) { - const [, , instant, , major, , minor] = match; + const [, , instant, v, major, dot, minor] = match; if (instant) { awsModel = "anthropic.claude-instant-v1"; @@ -1477,7 +1477,9 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' if (major === "2") { if (minor === "0") { awsModel = "anthropic.claude-v2"; - } + } else if (!v && !dot && !minor) { + awsModel = "anthropic.claude-v2"; + } } } @@ -1488,6 +1490,7 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' max_tokens_to_sample: maxTokens, stop_sequences: ["\n\nHuman:", "\n\nSystem:", "\n\nAssistant:"], temperature: temperature, + top_p: db.top_p, } const rq = new HttpRequest({ method: "POST", From 013cce23308b193bc088c2c56a37a2c5b45d6016 Mon Sep 17 00:00:00 2001 From: gdosu Date: Sat, 13 Jan 2024 21:26:38 +0900 Subject: [PATCH 5/5] fix versioning mistake --- src/ts/process/request.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ts/process/request.ts b/src/ts/process/request.ts index d99bd569..43907563 100644 --- a/src/ts/process/request.ts +++ b/src/ts/process/request.ts @@ -1469,12 +1469,12 @@ export async function requestChatDataMain(arg:requestDataArgument, model:'model' } // There's only one v1 model - if (major === "1") { + else if (major === "1") { awsModel = "anthropic.claude-v1"; } // Try to map Anthropic API v2 models to AWS v2 models - if (major === "2") { + else if (major === "2") { if (minor === "0") { awsModel = "anthropic.claude-v2"; } else if (!v && !dot && !minor) {