Add cache point

src/lang/en.ts

@@ -1062,4 +1062,6 @@ export const languageEnglish = {
     thinkingTokens: "Thinking Tokens",
     antiServerOverload: "Anti-Server Overload",
     localActivationInGlobalLorebook: "Local Activation in Global Lorebook",
+    cachePoint: "Cache Point",
+    all: "All",
 }

src/lang/ko.ts

@@ -979,5 +979,7 @@ export const languageKorean = {
     "paste": "붙여넣기",
     "depth": "깊이",
     "alwaysActiveInChat": "언제나 활성화 (현재 챗)",
-    "childLoreDesc": "이것은 캐릭터 로어의 복사본이며, 삭제하거나 원본 로어에서 직접 비활성화하기 전에는 '언제나 활성화' 상태로 유지됩니다."
+    "childLoreDesc": "이것은 캐릭터 로어의 복사본이며, 삭제하거나 원본 로어에서 직접 비활성화하기 전에는 '언제나 활성화' 상태로 유지됩니다.",
+    "cachePoint": "캐시 포인트",
+    "all": "모두",
 }

src/lib/UI/PromptDataItem.svelte

@@ -183,6 +183,10 @@
                 promptItem.text = ""
                 promptItem.role = "system"
             }
+            if(promptItem.type === 'cache'){
+                promptItem.depth = 1
+                promptItem.role = 'all'
+            }
             if(promptItem.type === 'chat'){
                 promptItem.rangeStart = -1000
                 promptItem.rangeEnd = 'end'
@@ -198,6 +202,7 @@
             <OptionInput value="memory">{language.formating.memory}</OptionInput>
             <OptionInput value="postEverything">{language.formating.postEverything}</OptionInput>
             <OptionInput value="chatML">{"chatML"}</OptionInput>
+            <OptionInput value="cache">{language.cachePoint}</OptionInput>
 
             {#if DBState.db.promptSettings.customChainOfThought}
                 <OptionInput value="cot">{language.cot}</OptionInput>
@@ -224,6 +229,17 @@
             <span>{language.prompt}</span>
             <TextAreaInput highlight bind:value={promptItem.text} />
         {/if}
+        {#if promptItem.type === 'cache'}
+            <span>{language.depth}</span>
+            <NumberInput bind:value={promptItem.depth} />
+            <span>{language.role}</span>
+            <SelectInput bind:value={promptItem.role}>
+                <OptionInput value="all">{language.all}</OptionInput>
+                <OptionInput value="user">{language.user}</OptionInput>
+                <OptionInput value="bot">{language.character}</OptionInput>
+                <OptionInput value="system">{language.systemPrompt}</OptionInput>
+            </SelectInput>
+        {/if}
         {#if promptItem.type === 'chat'}
             {#if promptItem.rangeStart !== -1000}
                 <span>{language.rangeStart}</span>

src/ts/process/index.svelte.ts

@@ -40,6 +40,7 @@ export interface OpenAIChat{
     attr?:string[]
     multimodals?: MultiModal[]
     thoughts?: string[]
+    cachePoint?: boolean
 }
 
 export interface MultiModal{
@@ -635,6 +636,9 @@ export async function sendChat(chatProcessIndex = -1,arg:{
                     supaMemoryCardUsed = true
                     break
                 }
+                case 'cache':{
+                    break
+                }
             }
         }
     }
@@ -1142,6 +1146,22 @@ export async function sendChat(chatProcessIndex = -1,arg:{
                     }
 
                     pushPrompts(pmt)
+                    break
+                }
+                case 'cache':{
+                    let pointer = formated.length - 1
+                    let depthRemaining = card.depth
+                    while(pointer >= 0){
+                        if(depthRemaining === 0){
+                            break
+                        }
+                        if(formated[pointer].role === card.role || card.role === 'all'){
+                            formated[pointer].cachePoint = true
+                            depthRemaining--
+                        }
+                    }
+
+                    break
                 }
             }
         }

src/ts/process/prompt.ts

@@ -4,7 +4,7 @@ import { getDatabase, presetTemplate, setDatabase, type Database } from "../stor
 import { alertError, alertNormal } from "../alert";
 import type { OobaChatCompletionRequestParams } from "../model/ooba";
 
-export type PromptItem = PromptItemPlain|PromptItemTyped|PromptItemChat|PromptItemAuthorNote|PromptItemChatML
+export type PromptItem = PromptItemPlain|PromptItemTyped|PromptItemChat|PromptItemAuthorNote|PromptItemChatML|PromptItemCache
 export type PromptType = PromptItem['type'];
 export type PromptSettings = {
     assistantPrefill: string
@@ -52,6 +52,14 @@ export interface PromptItemChat {
     name?: string
 }
 
+export interface PromptItemCache {
+    type: 'cache';
+    name: string
+    depth: number
+    role: 'user'|'assistant'|'system'|'all'
+
+}
+
 export async function tokenizePreset(prompts:PromptItem[], consti:boolean = false){
     let total = 0
     for(const prompt of prompts){

src/ts/process/request.ts

@@ -370,6 +370,7 @@ export interface OpenAIChatExtra {
     thoughts?:string[]
     prefix?:boolean
     reasoning_content?:string
+    cachePoint?:boolean
 }
 
 function reformater(formated:OpenAIChat[],modelInfo:LLMModel){
@@ -570,6 +571,7 @@ async function requestOpenAI(arg:RequestDataArgumentExtended):Promise<requestDat
             delete formatedChat[i].attr
             delete formatedChat[i].multimodals
             delete formatedChat[i].thoughts
+            delete formatedChat[i].cachePoint
         }
         if(aiModel === 'reverse_proxy' && db.reverseProxyOobaMode && formatedChat[i].role === 'system'){
             const cont = formatedChat[i].content
@@ -2490,7 +2492,8 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
 
     const addClaudeChat = (chat:{
         role: 'user'|'assistant'
-        content: string
+        content: string,
+        cache: boolean
     }, multimodals?:MultiModal[]) => {
         if(claudeChat.length > 0 && claudeChat[claudeChat.length-1].role === chat.role){
             let content = claudeChat[claudeChat.length-1].content
@@ -2533,6 +2536,11 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
                     }
                 }
             }
+            if(chat.cache){
+                content[content.length-1].cache_control = {
+                    type: 'ephemeral'
+                }
+            }
             claudeChat[claudeChat.length-1].content = content
         }
         else{
@@ -2566,6 +2574,11 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
                 }
 
             }
+            if(chat.cache){
+                formatedChat.content[0].cache_control = {
+                    type: 'ephemeral'
+                }
+            }
             claudeChat.push(formatedChat)
         }
     }
@@ -2574,14 +2587,16 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
             case 'user':{
                 addClaudeChat({
                     role: 'user',
-                    content: chat.content
+                    content: chat.content,
+                    cache: chat.cachePoint
                 }, chat.multimodals)
                 break
             }
             case 'assistant':{
                 addClaudeChat({
                     role: 'assistant',
-                    content: chat.content
+                    content: chat.content,
+                    cache: chat.cachePoint
                 }, chat.multimodals)
                 break
             }
@@ -2592,7 +2607,8 @@ async function requestClaude(arg:RequestDataArgumentExtended):Promise<requestDat
                 else{
                     addClaudeChat({
                         role: 'user',
-                        content: "System: " + chat.content
+                        content: "System: " + chat.content,
+                        cache: chat.cachePoint
                     })
                 }
                 break