feat(ai-bedrock): add prompt caching to Converse

.changeset/bedrock-converse-cache-point.md

@@ -0,0 +1,7 @@
+---
+'@tanstack/ai-bedrock': minor
+---
+
+Add prompt-cache checkpoints to the Bedrock Converse adapter.
+
+Set `metadata.cachePoint` on a system prompt, a text content part, or a tool. The adapter places a Bedrock `cachePoint` block right after that item. This block makes the preceding prompt eligible for caching. A later request can read matching tokens at the reduced cache rate. Bedrock bills tokens that miss the cache at the standard input rate. `{ type: 'default' }` uses the 5-minute TTL. Add `ttl: '1h'` for the 1-hour cache. A request may carry up to four checkpoints.

docs/adapters/bedrock.md

@@ -177,6 +177,50 @@ const adapter = createBedrockText(
 )
 ```
 
+### Prompt caching
+
+Add a `cachePoint` to make a prompt prefix eligible for caching. Later requests can read matching tokens at the reduced cache rate. Bedrock bills cache misses at the standard input rate.
+
+Explicit prompt caching is model-dependent. Use `cachePoint` only with a model that AWS lists as supporting it. The minimum checkpoint size and the TTL options also vary by model.
+
+Set `metadata.cachePoint` on the item that ends the stable part of your prompt. The adapter places a `cachePoint` block right after it:
+
+```typescript
+import { bedrockText } from '@tanstack/ai-bedrock'
+import { chat } from '@tanstack/ai'
+
+const stream = chat({
+  adapter: bedrockText('us.anthropic.claude-sonnet-4-5-20250929-v1:0', {
+    region: 'us-east-1',
+  }),
+  systemPrompts: [
+    {
+      content: 'Long, stable instructions...',
+      metadata: { cachePoint: { type: 'default' } },
+    },
+  ],
+  messages: [
+    {
+      role: 'user',
+      content: [
+        {
+          type: 'text',
+          content: 'What changed since yesterday?',
+          // Caches the conversation up to here for the next round.
+          metadata: { cachePoint: { type: 'default' } },
+        },
+      ],
+    },
+  ],
+})
+```
+
+Tools take the same metadata. Pass `metadata: { cachePoint: { type: 'default' } }` to `toolDefinition()` for the last tool in the list to cache the tool definitions.
+
+- Bedrock accepts up to four checkpoints per request.
+- Add `ttl: '1h'` to keep an entry for one hour. The default is 5 minutes, and a read refreshes the timer. Bedrock processes tool checkpoints first, then system prompt checkpoints, and then message checkpoints. If a request mixes both TTLs, place every `1h` checkpoint before the first `5m` checkpoint in that combined order.
+- A checkpoint below the model's minimum size is ignored. The request still succeeds, and nothing is cached.
+
 ### Token usage
 
 `onUsage` and `RUN_FINISHED.usage` report Bedrock's counts as `promptTokens`, `completionTokens`, and `totalTokens`. When a request hits or writes a prompt cache, the cache counts arrive on `promptTokensDetails.cachedTokens` and `promptTokensDetails.cacheWriteTokens`. Bedrock counts only the uncached part of the input in `promptTokens`, so add the two cache counts to it to get the full input size.

packages/ai-bedrock/src/adapters/converse-text.ts

@@ -38,7 +38,11 @@ import type {
   StructuredOutputResult,
 } from '@tanstack/ai/adapters'
 import type { BedrockClientConfig } from '../utils/client'
-import type { BedrockMessageMetadataByModality } from '../message-types'
+import type {
+  BedrockMessageMetadataByModality,
+  BedrockSystemPromptMetadata,
+  BedrockToolMetadata,
+} from '../message-types'
 import type {
   BedrockConverseModels,
   ResolveConverseProviderOptions,
@@ -78,7 +82,15 @@ export class BedrockConverseTextAdapter<
   TModel,
   TProviderOptions,
   TInputModalities,
-  BedrockMessageMetadataByModality
+  BedrockMessageMetadataByModality,
+  // TToolCapabilities — Converse has no per-model tool-capability table; the
+  // base default.
+  ReadonlyArray<string>,
+  // TToolCallMetadata — Converse has no tool-call metadata round-tripping.
+  unknown,
+  // TSystemPromptMetadata — narrows `systemPrompts[i].metadata` at the chat()
+  // call site so users get `cachePoint` autocomplete.
+  BedrockSystemPromptMetadata
 > {
   override readonly kind = 'text' as const
   override readonly name = 'bedrock-converse' as const
@@ -593,10 +605,12 @@ function convertTools(tools: Array<Tool>): Array<ConverseToolInput> {
     const inputSchema: JSONSchema = convertSchemaToJsonSchema(
       tool.inputSchema,
     ) ?? { type: 'object', properties: {}, required: [] }
+    const { cachePoint }: BedrockToolMetadata = tool.metadata ?? {}
     return {
       name: tool.name,
       description: tool.description,
       inputSchema,
+      ...(cachePoint && { cachePoint }),
     }
   })
 }

packages/ai-bedrock/src/converse/message-converter.ts

@@ -15,6 +15,10 @@ import type {
   ToolResultContentBlock,
 } from '@aws-sdk/client-bedrock-runtime'
 import type { DocumentType } from '@smithy/types'
+import type {
+  BedrockSystemPromptMetadata,
+  BedrockTextMetadata,
+} from '../message-types'
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -176,6 +180,8 @@ function messageToBlocks(
     for (const part of msg.content) {
       const docIndex = isDocumentPart(part) ? ++docCounter.value : 0
       blocks.push(contentPartToBlock(part, docIndex))
+      const { cachePoint } = (part.metadata ?? {}) as BedrockTextMetadata
+      if (cachePoint) blocks.push({ cachePoint })
     }
   }
   // null → no text blocks
@@ -225,7 +231,8 @@ function messageToBlocks(
 /**
  * Convert TanStack AI messages + system prompts into the Converse API format.
  *
- * - System prompts are lifted into `SystemContentBlock[]`.
+ * - System prompts are lifted into `SystemContentBlock[]`; a prompt whose
+ *   `metadata.cachePoint` is set is followed by a `cachePoint` block.
  * - `tool` role messages are remapped to `user` role `toolResult` blocks.
  * - Consecutive messages with the same Converse role are merged (Converse
  *   requires strict user/assistant alternation).
@@ -235,9 +242,13 @@ export function toConverseMessages(
   systemPrompts?: Array<SystemPrompt>,
 ): { system: Array<SystemContentBlock>; messages: Array<Message> } {
   // Build system blocks (uses normalizeSystemPrompts for runtime validation)
-  const system: Array<SystemContentBlock> = normalizeSystemPrompts(
-    systemPrompts,
-  ).map((p) => ({ text: p.content }))
+  const system: Array<SystemContentBlock> =
+    normalizeSystemPrompts<BedrockSystemPromptMetadata>(systemPrompts).flatMap(
+      (p) =>
+        p.metadata?.cachePoint
+          ? [{ text: p.content }, { cachePoint: p.metadata.cachePoint }]
+          : [{ text: p.content }],
+    )
 
   // Convert each ModelMessage to a Converse Message, merging same-role pairs
   const converseMessages: Array<Message> = []

packages/ai-bedrock/src/converse/tool-converter.ts

@@ -3,11 +3,14 @@ import type {
   ToolConfiguration,
 } from '@aws-sdk/client-bedrock-runtime'
 import type { DocumentType } from '@smithy/types'
+import type { BedrockCachePoint } from '../message-types'
 
 export interface ConverseToolInput {
   name: string
   description?: string
   inputSchema: unknown
+  /** Emit a `cachePoint` entry right after this tool. */
+  cachePoint?: BedrockCachePoint
 }
 
 export type ToolChoiceInput =
@@ -26,13 +29,16 @@ export function toToolConfig(
   if (choice === 'none') return undefined
   const toolChoice = mapChoice(choice)
   return {
-    tools: tools.map((t) => ({
-      toolSpec: {
-        name: t.name,
-        ...(t.description ? { description: t.description } : {}),
-        inputSchema: { json: t.inputSchema as DocumentType },
+    tools: tools.flatMap((t) => [
+      {
+        toolSpec: {
+          name: t.name,
+          ...(t.description ? { description: t.description } : {}),
+          inputSchema: { json: t.inputSchema as DocumentType },
+        },
       },
-    })),
+      ...(t.cachePoint ? [{ cachePoint: t.cachePoint }] : []),
+    ]),
     ...(toolChoice ? { toolChoice } : {}),
   }
 }

packages/ai-bedrock/src/index.ts

@@ -199,6 +199,9 @@ export {
   type BedrockModelInputModalitiesByName,
 } from './model-meta'
 export type {
+  BedrockCachePoint,
+  BedrockSystemPromptMetadata,
+  BedrockToolMetadata,
   BedrockMessageMetadataByModality,
   BedrockTextMetadata,
   BedrockImageMetadata,

packages/ai-bedrock/src/message-types.ts

@@ -1,10 +1,37 @@
+/**
+ * A Bedrock prompt-cache checkpoint. Bedrock caches everything in the request
+ * before this block and reads it back at the cache rate while the entry lives.
+ * A request may carry up to four. Omit `ttl` for the 5-minute default.
+ *
+ * @see https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html
+ */
+export interface BedrockCachePoint {
+  type: 'default'
+  ttl?: '5m' | '1h'
+}
+
+/** Metadata on a `systemPrompts` entry, read by the Converse adapter. */
+export interface BedrockSystemPromptMetadata {
+  /** Place a cache checkpoint right after this system prompt. */
+  cachePoint?: BedrockCachePoint
+}
+
+/** Metadata on a tool definition, read by the Converse adapter. */
+export interface BedrockToolMetadata {
+  /** Place a cache checkpoint right after this tool's definition. */
+  cachePoint?: BedrockCachePoint
+}
+
 /**
  * Bedrock content-part metadata by modality, used for type inference when
  * constructing multimodal messages. Bedrock's OpenAI-compatible Chat
- * Completions accepts the standard OpenAI image-detail hint; other modalities
- * carry no extra metadata today.
+ * Completions accepts the standard OpenAI image-detail hint; the Converse
+ * adapter reads `cachePoint` on text parts.
  */
-export interface BedrockTextMetadata {}
+export interface BedrockTextMetadata {
+  /** Place a cache checkpoint right after this text block (Converse only). */
+  cachePoint?: BedrockCachePoint
+}
 
 export interface BedrockImageMetadata {
   /** Image processing detail: 'auto' (default), 'low', or 'high'. */

packages/ai-bedrock/tests/converse/adapter.test.ts

@@ -8,7 +8,7 @@ import type {
   ConverseStreamCommandInput,
   ConverseStreamOutput,
 } from '@aws-sdk/client-bedrock-runtime'
-import type { AdapterYieldChunk, TextOptions } from '@tanstack/ai'
+import type { AdapterYieldChunk, TextOptions, Tool } from '@tanstack/ai'
 
 /**
  * Subclass that overrides the protected SDK seams so no real AWS call happens.
@@ -80,6 +80,34 @@ describe('BedrockConverseTextAdapter', () => {
     expect(types).toContain(EventType.RUN_FINISHED)
   })
 
+  it('writes system and tool cachePoints into the Converse request', async () => {
+    const a = new StubAdapter({ apiKey: 'k' }, 'us.amazon.nova-pro-v1:0')
+    a.streamEvents = [{ messageStop: { stopReason: 'end_turn' } }]
+    const tool: Tool = {
+      name: 'lookup',
+      description: 'd',
+      inputSchema: { type: 'object', properties: {} },
+      metadata: { cachePoint: { type: 'default' } },
+    }
+    for await (const _c of a.chatStream(
+      textOptions({
+        systemPrompts: [
+          { content: 'stable', metadata: { cachePoint: { type: 'default' } } },
+        ],
+        tools: [tool],
+      }),
+    )) {
+      // drain
+    }
+    expect(a.capturedStreamInput?.system).toEqual([
+      { text: 'stable' },
+      { cachePoint: { type: 'default' } },
+    ])
+    expect(a.capturedStreamInput?.toolConfig?.tools?.[1]).toEqual({
+      cachePoint: { type: 'default' },
+    })
+  })
+
   it('maps modelOptions sampling + stop into Converse inferenceConfig', async () => {
     const a = new StubAdapter({ apiKey: 'k' }, 'us.amazon.nova-pro-v1:0')
     a.streamEvents = [{ messageStop: { stopReason: 'end_turn' } }]

packages/ai-bedrock/tests/converse/message-converter.test.ts

@@ -20,6 +20,40 @@ describe('toConverseMessages', () => {
     expect(system).toEqual([{ text: 'a' }, { text: 'b' }])
   })
 
+  it('follows a system prompt with a cachePoint when its metadata asks for one', () => {
+    const { system } = toConverseMessages(
+      [{ role: 'user', content: 'hi' }],
+      [
+        { content: 'stable', metadata: { cachePoint: { type: 'default' } } },
+        'volatile',
+      ],
+    )
+    expect(system).toEqual([
+      { text: 'stable' },
+      { cachePoint: { type: 'default' } },
+      { text: 'volatile' },
+    ])
+  })
+
+  it('follows a text part with a cachePoint when its metadata asks for one', () => {
+    const { messages } = toConverseMessages([
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            content: 'a',
+            metadata: { cachePoint: { type: 'default', ttl: '1h' } },
+          },
+        ],
+      },
+    ])
+    expect(messages[0]!.content).toEqual([
+      { text: 'a' },
+      { cachePoint: { type: 'default', ttl: '1h' } },
+    ])
+  })
+
   it('merges consecutive same-role messages (Converse requires alternation)', () => {
     const { messages } = toConverseMessages([
       { role: 'user', content: 'a' },

packages/ai-bedrock/tests/converse/tool-converter.test.ts

@@ -23,6 +23,22 @@ describe('toToolConfig', () => {
     expect(cfg?.toolChoice).toEqual({ auto: {} })
   })
 
+  it('follows a tool with a cachePoint when the tool asks for one', () => {
+    const cfg = toToolConfig(
+      [
+        { name: 'a', inputSchema: {}, cachePoint: { type: 'default' } },
+        { name: 'b', inputSchema: {} },
+      ],
+      'auto',
+    )
+    expect(cfg?.tools?.map((t) => Object.keys(t)[0])).toEqual([
+      'toolSpec',
+      'cachePoint',
+      'toolSpec',
+    ])
+    expect(cfg?.tools?.[1]).toEqual({ cachePoint: { type: 'default' } })
+  })
+
   it('maps required -> any and a named tool -> tool', () => {
     expect(
       toToolConfig([{ name: 'a', inputSchema: {} }], 'required')?.toolChoice,

testing/e2e/README.md

@@ -297,6 +297,8 @@ The default `bedrock-converse` adapter (introduced later) uses `@aws-sdk/client-
 
 **Coverage today:** the Converse translation layer (message converter, tool converter, stream processor, structured output, adapter) is covered by unit tests in `packages/ai-bedrock/tests/converse/` (64 tests). The OpenAI-compatible `bedrock` and `bedrock-responses` entries remain in the E2E matrix as-is.
 
+**Partial coverage:** `bedrock-converse-cache.spec.ts` drives the real Converse adapter through the AWS SDK against a hand-crafted mount (`/bedrock-converse-cache` in `global-setup.ts`) that encodes `vnd.amazon.eventstream` frames with `@smithy/eventstream-codec`. It covers `metadata.cachePoint` placement and the cache usage counters. The mount answers one fixed stream, so it is not a general Converse replay.
+
 **Follow-up:** a Bedrock/Converse provider will be added to aimock to close this gap and enable full E2E coverage of the Converse path.
 
 ### BytePlus (Ark) path handling and record-mode gap