feat: aws sonnet 3.7 + legacy (#772)
# PR Checklist - [ ] Have you checked if it works normally in all models? *Ignore this if it doesn't use models.* - [ ] Have you checked if it works normally in all web, local, and node hosted versions? If it doesn't, have you blocked it in those versions? - [ ] Have you added type definitions? # Description This PR aims to enable the use of the newly added claude 3.7 sonnet model (and some legacys (¬‿¬)) on AWS.
This commit is contained in:
@@ -745,6 +745,20 @@ export const LLMModels: LLMModel[] = [
|
|||||||
parameters: ClaudeParameters,
|
parameters: ClaudeParameters,
|
||||||
tokenizer: LLMTokenizer.Claude
|
tokenizer: LLMTokenizer.Claude
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'Claude 3.7 Sonnet (20250219) v1',
|
||||||
|
id: 'anthropic.claude-3-7-sonnet-20250219-v1:0',
|
||||||
|
provider: LLMProvider.AWS,
|
||||||
|
format: LLMFormat.AWSBedrockClaude,
|
||||||
|
flags: [
|
||||||
|
LLMFlags.hasPrefill,
|
||||||
|
LLMFlags.hasImageInput,
|
||||||
|
LLMFlags.hasFirstSystemPrompt,
|
||||||
|
LLMFlags.claudeThinking
|
||||||
|
],
|
||||||
|
parameters: [...ClaudeParameters, 'thinking_tokens'],
|
||||||
|
tokenizer: LLMTokenizer.Claude
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'Claude 3.5 Sonnet (20241022) v2',
|
name: 'Claude 3.5 Sonnet (20241022) v2',
|
||||||
id: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
|
id: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
|
||||||
@@ -772,6 +786,43 @@ export const LLMModels: LLMModel[] = [
|
|||||||
parameters: ClaudeParameters,
|
parameters: ClaudeParameters,
|
||||||
tokenizer: LLMTokenizer.Claude
|
tokenizer: LLMTokenizer.Claude
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'Claude 3 Sonnet (20240229) v1',
|
||||||
|
id: 'anthropic.claude-3-sonnet-20240229-v1:0',
|
||||||
|
provider: LLMProvider.AWS,
|
||||||
|
format: LLMFormat.AWSBedrockClaude,
|
||||||
|
flags: [
|
||||||
|
LLMFlags.hasPrefill,
|
||||||
|
LLMFlags.hasImageInput,
|
||||||
|
LLMFlags.hasFirstSystemPrompt
|
||||||
|
],
|
||||||
|
parameters: ClaudeParameters,
|
||||||
|
tokenizer: LLMTokenizer.Claude
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Claude 2.1',
|
||||||
|
id: 'anthropic.claude-v2:1',
|
||||||
|
provider: LLMProvider.AWS,
|
||||||
|
format: LLMFormat.AWSBedrockClaude,
|
||||||
|
flags: [
|
||||||
|
LLMFlags.hasPrefill,
|
||||||
|
LLMFlags.hasFirstSystemPrompt
|
||||||
|
],
|
||||||
|
parameters: ClaudeParameters,
|
||||||
|
tokenizer: LLMTokenizer.Claude
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Claude 2',
|
||||||
|
id: 'anthropic.claude-v2',
|
||||||
|
provider: LLMProvider.AWS,
|
||||||
|
format: LLMFormat.AWSBedrockClaude,
|
||||||
|
flags: [
|
||||||
|
LLMFlags.hasPrefill,
|
||||||
|
LLMFlags.hasFirstSystemPrompt
|
||||||
|
],
|
||||||
|
parameters: ClaudeParameters,
|
||||||
|
tokenizer: LLMTokenizer.Claude
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'Ooba',
|
name: 'Ooba',
|
||||||
id: 'ooba',
|
id: 'ooba',
|
||||||
|
|||||||
@@ -2705,18 +2705,17 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
|
|||||||
const host = AMZ_HOST.replace("%REGION%", region);
|
const host = AMZ_HOST.replace("%REGION%", region);
|
||||||
const stream = false; // todo?
|
const stream = false; // todo?
|
||||||
|
|
||||||
const awsModel = "us." + arg.modelInfo.internalID;
|
const awsModel = !arg.modelInfo.internalID.includes("claude-v2") ? "us." + arg.modelInfo.internalID : arg.modelInfo.internalID;
|
||||||
const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`
|
const url = `https://${host}/model/${awsModel}/invoke${stream ? "-with-response-stream" : ""}`
|
||||||
|
|
||||||
const params = {
|
let params = {...body}
|
||||||
messages : claudeChat,
|
params.anthropic_version = "bedrock-2023-05-31"
|
||||||
system: systemPrompt.trim(),
|
delete params.model
|
||||||
max_tokens: maxTokens,
|
delete params.stream
|
||||||
// stop_sequences: null,
|
if (params.thinking?.type === "enabled"){
|
||||||
temperature: arg.temperature,
|
params.temperature = 1.0
|
||||||
top_p: db.top_p,
|
delete params.top_k
|
||||||
top_k: db.top_k,
|
delete params.top_p
|
||||||
anthropic_version: "bedrock-2023-05-31",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const rq = new HttpRequest({
|
const rq = new HttpRequest({
|
||||||
@@ -2761,10 +2760,49 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
|
|||||||
result: JSON.stringify(res.data.error)
|
result: JSON.stringify(res.data.error)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
const contents = res?.data?.content
|
||||||
|
if(!contents || contents.length === 0){
|
||||||
|
return {
|
||||||
|
type: 'fail',
|
||||||
|
result: JSON.stringify(res.data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let resText = ''
|
||||||
|
let thinking = false
|
||||||
|
for(const content of contents){
|
||||||
|
if(content.type === 'text'){
|
||||||
|
if(thinking){
|
||||||
|
resText += "</Thoughts>\n\n"
|
||||||
|
thinking = false
|
||||||
|
}
|
||||||
|
resText += content.text
|
||||||
|
}
|
||||||
|
if(content.type === 'thinking'){
|
||||||
|
if(!thinking){
|
||||||
|
resText += "<Thoughts>\n"
|
||||||
|
thinking = true
|
||||||
|
}
|
||||||
|
resText += content.thinking ?? ''
|
||||||
|
}
|
||||||
|
if(content.type === 'redacted_thinking'){
|
||||||
|
if(!thinking){
|
||||||
|
resText += "<Thoughts>\n"
|
||||||
|
thinking = true
|
||||||
|
}
|
||||||
|
resText += '\n{{redacted_thinking}}\n'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(arg.extractJson && db.jsonSchemaEnabled){
|
||||||
|
return {
|
||||||
|
type: 'success',
|
||||||
|
result: extractJSON(resText, db.jsonSchema)
|
||||||
|
}
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
type: 'success',
|
type: 'success',
|
||||||
result: res.data.content[0].text
|
result: resText
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user