add claude caching

settings.example.js

@@ -68,7 +68,7 @@ export default {
         },
         claudeOptions: {
             modelOptions: {
-                model: 'claude-3-opus-20240229',
+                model: 'claude-3-opus-20240229', // 'claude-3-5-sonnet-20241022'
                 max_tokens: 4096,
                 temperature: 1,
                 stream: true,
@@ -77,6 +77,11 @@ export default {
                 systemMessage: '', // fs.readFileSync('./contexts/waluigiASCII.txt', 'utf8'),
                 n: 2,
             },
+            // Add cache options
+            cacheOptions: {
+                enabled: true, // users can set this to true to enable caching
+                minTokens: 1024, // minimum tokens required for caching (1024 for Sonnet/Opus, 2048 for Haiku)
+            },
         },
         infrastructOptions: {
             modelOptions: {

src/ClaudeClient.js

@@ -1,4 +1,5 @@
 import ChatClient from './ChatClient.js';
+import { getMessagesForConversation, getChildren, getParent } from './conversation.js';
 
 const CLAUDE_MODEL_INFO = {
     default: {
@@ -62,9 +63,11 @@ export default class ClaudeClient extends ChatClient {
     }
 
     getHeaders() {
-        let anthropicBeta
+        let anthropicBeta;
         if ('steering' in this.options && this.options.steering) {
             anthropicBeta = 'steering-2024-06-04';
+        } else if (this.options?.cacheOptions?.enabled || this.cache?.enabled) {
+            anthropicBeta = 'prompt-caching-2024-07-31';
         } else {
             anthropicBeta = 'messages-2023-12-15';
         }
@@ -114,22 +117,74 @@ export default class ClaudeClient extends ChatClient {
     }
 
     buildApiParams(userMessage = null, previousMessages = [], systemMessage = null) {
-        // const maxHistoryLength = 20;
-        const { messages: history, system } = super.buildApiParams(userMessage, previousMessages, systemMessage);
-        // merge all consecutive messages from the same author
-        const mergedMessageHistory = [];
-        let lastMessage = null;
-        for (const message of history) {
-            if (lastMessage && lastMessage.role === message.role) {
-                lastMessage.content += `${message.content}`;
-            } else {
-                lastMessage = message;
-                mergedMessageHistory.push(message);
-            }
-        }
-        return {
-            messages: mergedMessageHistory, //.slice(-maxHistoryLength),
-            ...(system ? { system } : {}),
-        };
+       const { messages: history, system } = super.buildApiParams(userMessage, previousMessages, systemMessage);
+       const mergedMessageHistory = [];
+       let lastMessage = null;
+       const cacheEnabled = this.options?.cacheOptions?.enabled || this.cache?.enabled || false;
+       const MAX_CACHE_BLOCKS = 4;
+
+       // Check if we're at a point where conversation branches into multiple paths
+       const currentBranch = getMessagesForConversation(previousMessages, userMessage?.parentMessageId);
+       const isBranchingPoint = previousMessages.length > 0 && 
+           getChildren(previousMessages, previousMessages[previousMessages.length - 1].id).length > 1;
+
+       // Find last two points where conversation branched into multiple paths
+       const branchPoints = [];
+       let messageId = userMessage?.parentMessageId;
+       while (messageId && branchPoints.length < 2) {
+           if (getChildren(previousMessages, messageId).length > 1) {
+               branchPoints.push(messageId);
+           }
+           messageId = getParent(previousMessages, messageId)?.id;
+       }
+
+       // Determine if current message should be a cache checkpoint
+       const shouldCacheMessage = (message, index) => {
+           if (!cacheEnabled) return false;
+
+           if (branchPoints.includes(message.id)) return true;
+           if (isBranchingPoint && index === mergedMessageHistory.length - 1) return true;
+
+           const usedCacheBlocks = branchPoints.length + (isBranchingPoint ? 1 : 0);
+           const remainingBlocks = MAX_CACHE_BLOCKS - usedCacheBlocks;
+           if (remainingBlocks > 0) {
+               return currentBranch.length >= 10 && currentBranch.length % 10 === 0;
+           }
+
+           return false;
+       };
+
+       for (const [index, message] of history.entries()) {
+           if (lastMessage && lastMessage.role === message.role) {
+               const lastContent = lastMessage.content[lastMessage.content.length - 1];
+               lastContent.text += message.content;
+           } else {
+               const messageContent = {
+                   type: 'text',
+                   text: message.content,
+                   ...(shouldCacheMessage(message, index) && { cache_control: { type: 'ephemeral' } })
+               };
+
+               const messageWithOptionalCache = {
+                   role: message.role,
+                   content: [messageContent]
+               };
+
+               lastMessage = messageWithOptionalCache;
+               mergedMessageHistory.push(messageWithOptionalCache);
+           }
+       }
+
+       // Add system message to cache if present
+       const systemWithCache = system ? [{
+           type: 'text',
+           text: system,
+           ...(cacheEnabled && { cache_control: { type: 'ephemeral' } })
+       }] : undefined;
+
+       return {
+           messages: mergedMessageHistory,
+           ...(systemWithCache ? { system: systemWithCache } : {})
+       };
     }
 }