diff --git a/CHANGELOG.md b/CHANGELOG.md index 115285860..1e7197fbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - 2026-09-28 ### Added +- Summarizing compaction (LOU-W3): `summarizeStrategy({ model, protectedTokens?, prompt?, maxSummaryTokens? })` replaces the turns before the protected tail with one `[Conversation summary]` user message written by `model` (an `LLMProvider` or a `"provider/model"` spec), keeping system and pinned messages and each assistant turn together with its tool results; if the summary call fails it falls back to pruning and reports `error`. `twoPhaseStrategy()` (the recommended strategy) prunes first and summarizes only when still above the threshold. `pinMessage()` / `isPinned()` and the new optional `Message.metadata` (never sent to providers) mark messages no built-in strategy compacts. `CompactionStrategy.compact()` may now be async, so `compactMessages()` returns a Promise; `CompactionResult` / `CompactionInfo` gain `summary` and `error`, and `CompactionInput` gains `thresholdTokens` and `signal`. A strategy that throws no longer fails the run: the hook reports the `error` to `onCompaction`. See docs/compaction.md. - `createAgent({ retry, fallbackModels })` (LOU-V7.2): `retry` (`withRetry()` options, or `false`) retries failed model calls and defaults to `{ maxRetries: 2 }` for models given as `provider/model` strings (or picked from the environment); a `provider` instance is wrapped only when `retry` is set. `fallbackModels` are `provider/model` strings tried in order once the primary model's retries are used up (`withFallback([withRetry(primary), withRetry(fallback), ...])`). `agent.stream()`, `session.stream()` and `AgentExecutor.stream()` report them as two new typed events, `provider.retry` (`attempt`, `maxRetries`, `delayMs`, `error: { message, category? }`, `provider`) and `provider.fallback` (`from`, `to`, `error: { message }`), for any `withRetry()` / `withFallback()` provider. Additive: the event schema version stays 1. - Context compaction (LOU-W2): `createCompactionHook()` returns an `AgentHook` that, before a model call estimated above `thresholdPercent` (default 0.9) of the model's context window, replaces tool results older than the newest `protectedTokens` (default 40,000) with a `[pruned: result, N chars]` marker, in the run's transcript itself. `compactMessages()` compacts a conversation by hand; `CompactionStrategy` / `pruneToolResultsStrategy()` make the strategy pluggable. See docs/compaction.md. - `AgentSpec.mcpServers` (LOU-D20): a validated map of MCP servers (stdio `command`/`args`/`env` or HTTP `url`/`headers`) in agent spec files. `loadSpec()` reports a bad entry with its name, `loushy doctor` reads the validated field instead of the raw file, and `specToAgent()` exposes the parsed servers as `agent.mcpServers` (connecting them is TODO(LOU-D20.2)). Existing specs are unaffected. diff --git a/docs/api-overview.md b/docs/api-overview.md index fb732a3f1..1cc49e3e4 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -481,9 +481,12 @@ returns an `AgentHook` that, before each model call above 90% (by default) of the model's context window, replaces tool results older than the newest 40,000 tokens with a `[pruned: result, N chars]` marker. It edits the run's transcript in place, so pruning persists in checkpoints and -`result.messages`. `compactMessages(messages, options)` does the same once, -by hand, and `pruneToolResultsStrategy()` / `CompactionStrategy` are the -built-in and pluggable strategies. See [Context compaction](./compaction.md). +`result.messages`. `twoPhaseStrategy({ model })` (recommended) prunes first +and, if the run is still too big, replaces old turns with a summary written +by `model`; `summarizeStrategy()` only summarizes. `pinMessage(message)` marks +a message that is never pruned or summarized. `compactMessages(messages, options)` +does the same once, by hand (async), and `CompactionStrategy` is the +pluggable interface (`compact()` may be async). See [Context compaction](./compaction.md). ## Flows, evals, observability and security diff --git a/docs/compaction.md b/docs/compaction.md index 8f239878a..16605af6d 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -2,24 +2,31 @@ A long agent run fills its context window mostly with tool results: file contents, search hits, API responses the model read once and no longer needs -in full. Compaction keeps such a run under the model's limit by replacing old -tool results with a short marker before the request is sent. +in full. Compaction keeps such a run under the model's limit in two phases: +first it replaces old tool results with a short marker, and if that is not +enough it replaces the oldest turns with a summary written by a (cheaper) +model. -## Pruning old tool results in a run +## Compacting a run `createCompactionHook()` returns an `AgentHook` named `compaction`. Register it -and pass the registry to `AgentExecutor.execute()`: +and pass the registry to `AgentExecutor.execute()`. The recommended setup is +the two-phase strategy with a cheap summarizer model: ```ts -import { AgentExecutor, HookRegistry, createCompactionHook } from '@loushy/build-ai-agent'; +import { AgentExecutor, HookRegistry, createCompactionHook, twoPhaseStrategy } from '@loushy/build-ai-agent'; const hooks = new HookRegistry(); hooks.register( createCompactionHook({ + strategy: twoPhaseStrategy({ model: 'openai/gpt-4o-mini' }), // prune, then summarize if still too big thresholdPercent: 0.9, // compact above 90% of the context window (the default) - protectedTokens: 40_000, // never prune the newest 40K tokens (the default) - onCompaction: ({ tokensBefore, tokensAfter, prunedToolCallIds }) => - console.log(`compacted ${tokensBefore} -> ${tokensAfter} tokens (${prunedToolCallIds.length} results)`), + protectedTokens: 40_000, // never change the newest 40K tokens (the default) + onCompaction: ({ tokensBefore, tokensAfter, prunedToolCallIds, summary, error }) => { + console.log(`compacted ${tokensBefore} -> ${tokensAfter} tokens (${prunedToolCallIds.length} results pruned)`); + if (summary) console.log('summary:', summary); + if (error) console.warn('summarizer failed, kept the pruned conversation:', error.message); + }, }) ); @@ -29,18 +36,39 @@ const result = await AgentExecutor.execute({ agent, input, provider, toolRegistr Before every model call the hook estimates the request's size with `estimateTokens` (see [Models, tokens and cost](./api-overview.md#models-tokens-and-cost)). When it is above `thresholdPercent` of the context window, the hook runs its -strategy and calls `onCompaction` with the token counts, the pruned -`toolCallId`s and the strategy name. +strategy (and waits for it, if it is async) and calls `onCompaction` with the +token counts, the pruned `toolCallId`s, the summary if there is one and the +strategy name. | Option | Default | Meaning | | --- | --- | --- | | `thresholdPercent` | `0.9` | Compact when the estimated request is above this share of the context window. Must be in (0, 1]. | | `contextWindow` | registry, else `128_000` | Context window in tokens. By default it is looked up for `request.model` with `getModelInfo()`; register your own models with `registerModel()`. | | `protectedTokens` | `40_000` | The newest messages that fit in this many tokens are never changed. | -| `strategy` | `pruneToolResultsStrategy()` | How to compact (see below). | -| `onCompaction` | none | Called after each compaction that changed the conversation. | +| `strategy` | `pruneToolResultsStrategy()` | How to compact (see below). `twoPhaseStrategy()` is recommended; it needs a summarizer model, so it is not the default. | +| `onCompaction` | none | Called after each compaction that changed the conversation or reported an `error`. | -## What pruning changes +Compaction never fails a run. If the strategy throws, or the summarizer call +fails, the run continues (with the pruned conversation, or unchanged) and +`onCompaction` receives the `error`. + +The hook rewrites the run's transcript itself, not a copy: the request's +`messages` array is the run's message list, and the hook changes it in place. +A pruned result or a summary therefore stays in later steps, in checkpoints +(a resumed session does not bring the full history back) and in +`result.messages`, and it is not compacted again on the next step. +Compaction is lossy: if the model needs a pruned result again, it has to call +the tool again. + +## Strategies + +| Strategy | What it does | +| --- | --- | +| `pruneToolResultsStrategy()` | Replaces old tool results with a marker. No model call. | +| `summarizeStrategy({ model, ... })` | Replaces old turns with one summary message written by `model`. | +| `twoPhaseStrategy({ model, ... })` | Prunes first; summarizes only if the conversation is still above the threshold. Recommended. | + +### Pruning tool results `pruneToolResultsStrategy()` replaces the `content` of each tool result older than the protected tail with a marker such as @@ -51,53 +79,112 @@ than the protected tail with a marker such as for every provider; - the results of the latest assistant turn, which the model has not read yet, even when they are bigger than `protectedTokens`; -- results that are already a marker, or shorter than one. +- pinned results, results that are already a marker, or results shorter than + a marker. -The hook rewrites the run's transcript itself, not a copy: the request's -`messages` array is the run's message list, and the hook changes it in place. -A pruned result therefore stays pruned in later steps, in checkpoints (a -resumed session does not bring the full result back) and in -`result.messages`, and it is not pruned again on the next step. Pruning is -lossy: if the model needs a pruned result again, it has to call the tool -again. +### Summarizing old turns + +`summarizeStrategy()` sends the messages before the protected tail to the +summarizer with a focused instruction and replaces them with one `user` +message: + +```text +[Conversation summary] + +``` + +| Option | Default | Meaning | +| --- | --- | --- | +| `model` | required | The summarizer: an `LLMProvider`, or a `"provider/model"` spec resolved with `resolveProvider()`. A small, cheap model is usually enough. | +| `protectedTokens` | the hook's | Recent tokens kept as they are. | +| `prompt` | `DEFAULT_SUMMARY_PROMPT` | The instruction sent as the summarizer's system message. The conversation to summarize is the user message. | +| `maxSummaryTokens` | provider default | `maxTokens` of the summary call. | + +The summary keeps the transcript valid for every provider: + +- system messages and pinned messages stay where they are, and the summary + goes where the first summarized message was; +- an assistant turn is summarized or kept together with all its tool results, + so no tool call loses its result (and no result loses its call); +- the last message and the latest assistant turn's results are always kept. + +If the summary call fails (or returns nothing), the strategy falls back to +pruning and returns the failure as `error`. + +### Two phases + +`twoPhaseStrategy()` takes the same options. It prunes tool results first, +which is free, and calls the summarizer only when the pruned conversation is +still above `thresholdPercent` of the context window. The summarizer then +reads the pruned conversation. If it fails, the pruned conversation is kept and +`error` is reported. + +## Pinning messages + +`pinMessage(message)` returns a copy of the message marked as pinned (it sets +`metadata.pinned: true`; `Message.metadata` is application data that is never +sent to the model). The built-in strategies never prune or summarize a pinned +message; a pinned tool result keeps its whole assistant turn. Pin what the +agent must always see in full, such as the task statement or a key document: + +```ts +import { pinMessage, isPinned, type Message } from '@loushy/build-ai-agent'; + +const messages: Message[] = [ + { role: 'system', content: 'You are a research assistant.' }, + pinMessage({ role: 'user', content: 'Goal: compare the three vendors on price and support.' }), +]; +console.log(isPinned(messages[1])); // true +``` + +Custom strategies should honor `isPinned()` too. ## Compacting by hand `compactMessages(messages, options)` runs a strategy once, whatever the size -of the conversation, and returns a new array (its input is not modified). Use -it to shrink a stored transcript before you continue it: +of the conversation, and resolves to a new array (its input is not modified). +Use it to shrink a stored transcript before you continue it: ```ts -import { compactMessages, type Message } from '@loushy/build-ai-agent'; +import { compactMessages, summarizeStrategy, type Message } from '@loushy/build-ai-agent'; declare const history: Message[]; -const { messages, tokensBefore, tokensAfter, prunedToolCallIds } = compactMessages(history, { +const { messages, tokensBefore, tokensAfter, prunedToolCallIds } = await compactMessages(history, { protectedTokens: 8_000, model: 'gpt-4o-mini', // for the context-window lookup and the token estimator }); console.log(`${tokensBefore} -> ${tokensAfter} tokens, pruned ${prunedToolCallIds.length} results`); + +const summarized = await compactMessages(history, { + protectedTokens: 8_000, + strategy: summarizeStrategy({ model: 'openai/gpt-4o-mini' }), +}); +console.log(summarized.summary ?? summarized.error?.message); ``` ## Writing a strategy -A strategy is a name and a `compact()` function. It receives the messages, a -token counter for the request's model, the context window and -`protectedTokens`, and returns the new messages with before/after token counts. -Return the input array unchanged when there is nothing to do: the hook then -leaves the run alone and does not call `onCompaction`. +A strategy is a name and a `compact()` function, which may be async. It +receives the messages, a token counter for the request's model, the context +window, `protectedTokens`, `thresholdTokens` (the size to get under) and the +run's abort `signal`, and returns the new messages with before/after token +counts (and optionally a `summary` or an `error`). Return the input array +unchanged when there is nothing to do: the hook then leaves the run alone and +does not call `onCompaction`. ```ts -import { createCompactionHook, type CompactionStrategy } from '@loushy/build-ai-agent'; +import { createCompactionHook, isPinned, type CompactionStrategy } from '@loushy/build-ai-agent'; -// Keep the system prompt, the first user message and the last 20 messages. +// Keep the system prompt, pinned messages and the last 20 messages. const keepRecent: CompactionStrategy = { name: 'keep-recent', compact({ messages, estimateTokens }) { const tokensBefore = estimateTokens(messages); if (messages.length <= 22) return { messages, tokensBefore, tokensAfter: tokensBefore, prunedToolCallIds: [] }; // A real strategy must not split a tool call from its result. - const kept = [...messages.slice(0, 2), ...messages.slice(-20)]; + const older = messages.slice(0, -20).filter((m) => m.role === 'system' || isPinned(m)); + const kept = [...older, ...messages.slice(-20)]; return { messages: kept, tokensBefore, tokensAfter: estimateTokens(kept), prunedToolCallIds: [] }; }, }; @@ -105,6 +192,6 @@ const keepRecent: CompactionStrategy = { const hook = createCompactionHook({ strategy: keepRecent }); ``` -A summarizing strategy (replace old turns with a model-written summary) and -typed `compaction.*` stream events are planned; `onCompaction` is where they -will plug in. +Typed `compaction.*` stream events and a `createAgent({ compaction })` option +are planned (LOU-W3.2); until then `onCompaction` is where to observe +compactions. diff --git a/llms-full.txt b/llms-full.txt index 6a077e06d..431fe94cc 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1138,9 +1138,12 @@ returns an `AgentHook` that, before each model call above 90% (by default) of the model's context window, replaces tool results older than the newest 40,000 tokens with a `[pruned: result, N chars]` marker. It edits the run's transcript in place, so pruning persists in checkpoints and -`result.messages`. `compactMessages(messages, options)` does the same once, -by hand, and `pruneToolResultsStrategy()` / `CompactionStrategy` are the -built-in and pluggable strategies. See [Context compaction](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/compaction.md). +`result.messages`. `twoPhaseStrategy({ model })` (recommended) prunes first +and, if the run is still too big, replaces old turns with a summary written +by `model`; `summarizeStrategy()` only summarizes. `pinMessage(message)` marks +a message that is never pruned or summarized. `compactMessages(messages, options)` +does the same once, by hand (async), and `CompactionStrategy` is the +pluggable interface (`compact()` may be async). See [Context compaction](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/compaction.md). ## Flows, evals, observability and security @@ -1986,24 +1989,31 @@ Source: docs/compaction.md A long agent run fills its context window mostly with tool results: file contents, search hits, API responses the model read once and no longer needs -in full. Compaction keeps such a run under the model's limit by replacing old -tool results with a short marker before the request is sent. +in full. Compaction keeps such a run under the model's limit in two phases: +first it replaces old tool results with a short marker, and if that is not +enough it replaces the oldest turns with a summary written by a (cheaper) +model. -## Pruning old tool results in a run +## Compacting a run `createCompactionHook()` returns an `AgentHook` named `compaction`. Register it -and pass the registry to `AgentExecutor.execute()`: +and pass the registry to `AgentExecutor.execute()`. The recommended setup is +the two-phase strategy with a cheap summarizer model: ```ts -import { AgentExecutor, HookRegistry, createCompactionHook } from '@loushy/build-ai-agent'; +import { AgentExecutor, HookRegistry, createCompactionHook, twoPhaseStrategy } from '@loushy/build-ai-agent'; const hooks = new HookRegistry(); hooks.register( createCompactionHook({ + strategy: twoPhaseStrategy({ model: 'openai/gpt-4o-mini' }), // prune, then summarize if still too big thresholdPercent: 0.9, // compact above 90% of the context window (the default) - protectedTokens: 40_000, // never prune the newest 40K tokens (the default) - onCompaction: ({ tokensBefore, tokensAfter, prunedToolCallIds }) => - console.log(`compacted ${tokensBefore} -> ${tokensAfter} tokens (${prunedToolCallIds.length} results)`), + protectedTokens: 40_000, // never change the newest 40K tokens (the default) + onCompaction: ({ tokensBefore, tokensAfter, prunedToolCallIds, summary, error }) => { + console.log(`compacted ${tokensBefore} -> ${tokensAfter} tokens (${prunedToolCallIds.length} results pruned)`); + if (summary) console.log('summary:', summary); + if (error) console.warn('summarizer failed, kept the pruned conversation:', error.message); + }, }) ); @@ -2013,18 +2023,39 @@ const result = await AgentExecutor.execute({ agent, input, provider, toolRegistr Before every model call the hook estimates the request's size with `estimateTokens` (see [Models, tokens and cost](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/api-overview.md#models-tokens-and-cost)). When it is above `thresholdPercent` of the context window, the hook runs its -strategy and calls `onCompaction` with the token counts, the pruned -`toolCallId`s and the strategy name. +strategy (and waits for it, if it is async) and calls `onCompaction` with the +token counts, the pruned `toolCallId`s, the summary if there is one and the +strategy name. | Option | Default | Meaning | | --- | --- | --- | | `thresholdPercent` | `0.9` | Compact when the estimated request is above this share of the context window. Must be in (0, 1]. | | `contextWindow` | registry, else `128_000` | Context window in tokens. By default it is looked up for `request.model` with `getModelInfo()`; register your own models with `registerModel()`. | | `protectedTokens` | `40_000` | The newest messages that fit in this many tokens are never changed. | -| `strategy` | `pruneToolResultsStrategy()` | How to compact (see below). | -| `onCompaction` | none | Called after each compaction that changed the conversation. | +| `strategy` | `pruneToolResultsStrategy()` | How to compact (see below). `twoPhaseStrategy()` is recommended; it needs a summarizer model, so it is not the default. | +| `onCompaction` | none | Called after each compaction that changed the conversation or reported an `error`. | + +Compaction never fails a run. If the strategy throws, or the summarizer call +fails, the run continues (with the pruned conversation, or unchanged) and +`onCompaction` receives the `error`. + +The hook rewrites the run's transcript itself, not a copy: the request's +`messages` array is the run's message list, and the hook changes it in place. +A pruned result or a summary therefore stays in later steps, in checkpoints +(a resumed session does not bring the full history back) and in +`result.messages`, and it is not compacted again on the next step. +Compaction is lossy: if the model needs a pruned result again, it has to call +the tool again. + +## Strategies + +| Strategy | What it does | +| --- | --- | +| `pruneToolResultsStrategy()` | Replaces old tool results with a marker. No model call. | +| `summarizeStrategy({ model, ... })` | Replaces old turns with one summary message written by `model`. | +| `twoPhaseStrategy({ model, ... })` | Prunes first; summarizes only if the conversation is still above the threshold. Recommended. | -## What pruning changes +### Pruning tool results `pruneToolResultsStrategy()` replaces the `content` of each tool result older than the protected tail with a marker such as @@ -2035,53 +2066,112 @@ than the protected tail with a marker such as for every provider; - the results of the latest assistant turn, which the model has not read yet, even when they are bigger than `protectedTokens`; -- results that are already a marker, or shorter than one. +- pinned results, results that are already a marker, or results shorter than + a marker. -The hook rewrites the run's transcript itself, not a copy: the request's -`messages` array is the run's message list, and the hook changes it in place. -A pruned result therefore stays pruned in later steps, in checkpoints (a -resumed session does not bring the full result back) and in -`result.messages`, and it is not pruned again on the next step. Pruning is -lossy: if the model needs a pruned result again, it has to call the tool -again. +### Summarizing old turns + +`summarizeStrategy()` sends the messages before the protected tail to the +summarizer with a focused instruction and replaces them with one `user` +message: + +```text +[Conversation summary] + +``` + +| Option | Default | Meaning | +| --- | --- | --- | +| `model` | required | The summarizer: an `LLMProvider`, or a `"provider/model"` spec resolved with `resolveProvider()`. A small, cheap model is usually enough. | +| `protectedTokens` | the hook's | Recent tokens kept as they are. | +| `prompt` | `DEFAULT_SUMMARY_PROMPT` | The instruction sent as the summarizer's system message. The conversation to summarize is the user message. | +| `maxSummaryTokens` | provider default | `maxTokens` of the summary call. | + +The summary keeps the transcript valid for every provider: + +- system messages and pinned messages stay where they are, and the summary + goes where the first summarized message was; +- an assistant turn is summarized or kept together with all its tool results, + so no tool call loses its result (and no result loses its call); +- the last message and the latest assistant turn's results are always kept. + +If the summary call fails (or returns nothing), the strategy falls back to +pruning and returns the failure as `error`. + +### Two phases + +`twoPhaseStrategy()` takes the same options. It prunes tool results first, +which is free, and calls the summarizer only when the pruned conversation is +still above `thresholdPercent` of the context window. The summarizer then +reads the pruned conversation. If it fails, the pruned conversation is kept and +`error` is reported. + +## Pinning messages + +`pinMessage(message)` returns a copy of the message marked as pinned (it sets +`metadata.pinned: true`; `Message.metadata` is application data that is never +sent to the model). The built-in strategies never prune or summarize a pinned +message; a pinned tool result keeps its whole assistant turn. Pin what the +agent must always see in full, such as the task statement or a key document: + +```ts +import { pinMessage, isPinned, type Message } from '@loushy/build-ai-agent'; + +const messages: Message[] = [ + { role: 'system', content: 'You are a research assistant.' }, + pinMessage({ role: 'user', content: 'Goal: compare the three vendors on price and support.' }), +]; +console.log(isPinned(messages[1])); // true +``` + +Custom strategies should honor `isPinned()` too. ## Compacting by hand `compactMessages(messages, options)` runs a strategy once, whatever the size -of the conversation, and returns a new array (its input is not modified). Use -it to shrink a stored transcript before you continue it: +of the conversation, and resolves to a new array (its input is not modified). +Use it to shrink a stored transcript before you continue it: ```ts -import { compactMessages, type Message } from '@loushy/build-ai-agent'; +import { compactMessages, summarizeStrategy, type Message } from '@loushy/build-ai-agent'; declare const history: Message[]; -const { messages, tokensBefore, tokensAfter, prunedToolCallIds } = compactMessages(history, { +const { messages, tokensBefore, tokensAfter, prunedToolCallIds } = await compactMessages(history, { protectedTokens: 8_000, model: 'gpt-4o-mini', // for the context-window lookup and the token estimator }); console.log(`${tokensBefore} -> ${tokensAfter} tokens, pruned ${prunedToolCallIds.length} results`); + +const summarized = await compactMessages(history, { + protectedTokens: 8_000, + strategy: summarizeStrategy({ model: 'openai/gpt-4o-mini' }), +}); +console.log(summarized.summary ?? summarized.error?.message); ``` ## Writing a strategy -A strategy is a name and a `compact()` function. It receives the messages, a -token counter for the request's model, the context window and -`protectedTokens`, and returns the new messages with before/after token counts. -Return the input array unchanged when there is nothing to do: the hook then -leaves the run alone and does not call `onCompaction`. +A strategy is a name and a `compact()` function, which may be async. It +receives the messages, a token counter for the request's model, the context +window, `protectedTokens`, `thresholdTokens` (the size to get under) and the +run's abort `signal`, and returns the new messages with before/after token +counts (and optionally a `summary` or an `error`). Return the input array +unchanged when there is nothing to do: the hook then leaves the run alone and +does not call `onCompaction`. ```ts -import { createCompactionHook, type CompactionStrategy } from '@loushy/build-ai-agent'; +import { createCompactionHook, isPinned, type CompactionStrategy } from '@loushy/build-ai-agent'; -// Keep the system prompt, the first user message and the last 20 messages. +// Keep the system prompt, pinned messages and the last 20 messages. const keepRecent: CompactionStrategy = { name: 'keep-recent', compact({ messages, estimateTokens }) { const tokensBefore = estimateTokens(messages); if (messages.length <= 22) return { messages, tokensBefore, tokensAfter: tokensBefore, prunedToolCallIds: [] }; // A real strategy must not split a tool call from its result. - const kept = [...messages.slice(0, 2), ...messages.slice(-20)]; + const older = messages.slice(0, -20).filter((m) => m.role === 'system' || isPinned(m)); + const kept = [...older, ...messages.slice(-20)]; return { messages: kept, tokensBefore, tokensAfter: estimateTokens(kept), prunedToolCallIds: [] }; }, }; @@ -2089,9 +2179,9 @@ const keepRecent: CompactionStrategy = { const hook = createCompactionHook({ strategy: keepRecent }); ``` -A summarizing strategy (replace old turns with a model-written summary) and -typed `compaction.*` stream events are planned; `onCompaction` is where they -will plug in. +Typed `compaction.*` stream events and a `createAgent({ compaction })` option +are planned (LOU-W3.2); until then `onCompaction` is where to observe +compactions. # Configuration diff --git a/src/context/compaction.summarize.test.ts b/src/context/compaction.summarize.test.ts new file mode 100644 index 000000000..c923c6526 --- /dev/null +++ b/src/context/compaction.summarize.test.ts @@ -0,0 +1,321 @@ +import { describe, it, expect, vi } from 'vitest'; +import { z } from 'zod'; +import { + DEFAULT_SUMMARY_PROMPT, + compactMessages, + createCompactionHook, + isPinned, + pinMessage, + pruneToolResultsStrategy, + summarizeStrategy, + twoPhaseStrategy, + type CompactionInfo, + type CompactionStrategy, +} from './compaction'; +import type { Message } from '../providers'; +import { estimateTokens } from '../models'; +import { HookRegistry } from '../execution/hooks'; +import { AgentExecutor } from '../execution/AgentExecutor'; +import { defineTool } from '../tools/defineTool'; +import { ToolRegistry } from '../tools'; +import { mockModel } from '../testing'; + +const BIG = 'x'.repeat(4_000); // about 1,000 tokens + +const call = (id: string, name = 'search') => ({ id, type: 'function' as const, function: { name, arguments: '{}' } }); + +/** system, user, then `rounds` of assistant tool call + its (large) result. */ +function transcript(rounds: number): Message[] { + const messages: Message[] = [ + { role: 'system', content: 'You are helpful.' }, + { role: 'user', content: 'start' }, + ]; + for (let i = 1; i <= rounds; i++) { + messages.push({ role: 'assistant', content: '', toolCalls: [call(`call_${i}`)] }); + messages.push({ role: 'tool', toolCallId: `call_${i}`, toolName: 'search', content: `${i}:${BIG}` }); + } + return messages; +} + +/** Every tool call has a later result, and every result answers an earlier call. */ +function expectValidTranscript(messages: Message[]): void { + messages.forEach((message, index) => { + for (const c of message.toolCalls ?? []) { + const answer = messages.slice(index + 1).find((m) => m.role === 'tool' && m.toolCallId === c.id); + expect(answer, `result for ${c.id}`).toBeDefined(); + } + if (message.role === 'tool') { + const asked = messages.slice(0, index).some((m) => m.toolCalls?.some((c) => c.id === message.toolCallId)); + expect(asked, `call for ${message.toolCallId}`).toBe(true); + } + }); +} + +const summaryOf = (text: string): Message => ({ role: 'user', content: `[Conversation summary]\n${text}` }); + +describe('summarizeStrategy', () => { + it('replaces the prefix with one summary message and keeps the system prompt and the tail intact', async () => { + const summarizer = mockModel(['The user asked for searches; results 1-8 were read.']); + const original = transcript(10); + const result = await compactMessages(original, { + protectedTokens: 2_100, + strategy: summarizeStrategy({ model: summarizer, maxSummaryTokens: 500 }), + }); + + // The tail (the last two assistant turns with their results) is the original, untouched. + expect(result.messages).toEqual([original[0], summaryOf('The user asked for searches; results 1-8 were read.'), ...original.slice(-4)]); + expect(result.messages[0]).toBe(original[0]); + result.messages.slice(2).forEach((m, i) => expect(m).toBe(original[original.length - 4 + i])); + expect(result.summary).toBe('The user asked for searches; results 1-8 were read.'); + expect(result.tokensBefore).toBe(estimateTokens(original)); + expect(result.tokensAfter).toBe(estimateTokens(result.messages)); + expectValidTranscript(result.messages); + + // One focused summary call over the folded messages only. + expect(summarizer.calls).toHaveLength(1); + const [request] = summarizer.calls; + expect(request.maxTokens).toBe(500); + expect(request.messages[0]).toEqual({ role: 'system', content: DEFAULT_SUMMARY_PROMPT }); + expect(request.messages[1].content).toContain('user: start'); + expect(request.messages[1].content).toContain('[called search({})]'); + expect(request.messages[1].content).toContain('tool search: 8:'); + expect(request.messages[1].content).not.toContain('9:'); + expect(request.messages[1].content).not.toContain('You are helpful.'); + }); + + it('keeps an assistant turn and all its tool results together', async () => { + const original: Message[] = [ + { role: 'system', content: 'sys' }, + { role: 'user', content: 'go' }, + { role: 'assistant', content: '', toolCalls: [call('a'), call('b')] }, + { role: 'tool', toolCallId: 'a', toolName: 'search', content: BIG }, + { role: 'tool', toolCallId: 'b', toolName: 'search', content: BIG }, + { role: 'assistant', content: '', toolCalls: [call('c')] }, + { role: 'tool', toolCallId: 'c', toolName: 'search', content: 'small' }, + ]; + // 1,100 tokens reach into the middle of the first turn's results; the whole turn is kept. + const result = await compactMessages(original, { + protectedTokens: 1_100, + strategy: summarizeStrategy({ model: mockModel(['S']), prompt: 'Summarize briefly.' }), + }); + expect(result.messages).toEqual([original[0], summaryOf('S'), ...original.slice(2)]); + expectValidTranscript(result.messages); + }); + + it('changes nothing when there is nothing before the tail to summarize', async () => { + const summarizer = mockModel([]); + const original = transcript(1); + const result = await compactMessages(original, { protectedTokens: 100_000, strategy: summarizeStrategy({ model: summarizer }) }); + expect(result.messages).toBe(original); + expect(summarizer.calls).toHaveLength(0); + }); + + it('falls back to pruning and reports the error when the summarizer fails, never throwing', async () => { + const original = transcript(6); + const pruned = await compactMessages(original, { protectedTokens: 0 }); + + const failing = await compactMessages(original, { + protectedTokens: 0, + strategy: summarizeStrategy({ model: mockModel([{ error: new Error('rate limited') }]) }), + }); + expect(failing.error?.message).toBe('rate limited'); + expect(failing.messages).toEqual(pruned.messages); + expect(failing.prunedToolCallIds).toEqual(pruned.prunedToolCallIds); + expect(failing.summary).toBeUndefined(); + + const empty = await compactMessages(original, { protectedTokens: 0, strategy: summarizeStrategy({ model: mockModel([' ']) }) }); + expect(empty.error?.message).toMatch(/empty summary/); + expect(empty.messages).toEqual(pruned.messages); + + // A model spec that cannot be resolved is a failure too, not a crash. + const unresolved = await compactMessages(original, { protectedTokens: 0, strategy: summarizeStrategy({ model: 'nosuch/model' }) }); + expect(unresolved.error).toBeInstanceOf(Error); + expect(unresolved.messages).toEqual(pruned.messages); + }); +}); + +describe('pinned messages', () => { + it('pinMessage marks a copy and keeps existing metadata', () => { + const message: Message = { role: 'user', content: 'hi', metadata: { source: 'ui' } }; + const pinned = pinMessage(message); + expect(pinned).toEqual({ role: 'user', content: 'hi', metadata: { source: 'ui', pinned: true } }); + expect(isPinned(pinned)).toBe(true); + expect(isPinned(message)).toBe(false); + }); + + function withPins(): Message[] { + const messages = transcript(8); + messages[1] = pinMessage({ role: 'user', content: 'PINNED-GOAL' }); + messages[5] = pinMessage(messages[5]); // call_2's result + return messages; + } + + it('are never pruned', async () => { + const original = withPins(); + const result = await compactMessages(original, { protectedTokens: 0, strategy: pruneToolResultsStrategy() }); + expect(result.messages[5]).toBe(original[5]); + expect(result.prunedToolCallIds).toEqual(['call_1', 'call_3', 'call_4', 'call_5', 'call_6', 'call_7']); + }); + + it('are never summarized: they stay in place with their tool call, and are not sent to the summarizer', async () => { + const original = withPins(); + const summarizer = mockModel(['S']); + const result = await compactMessages(original, { protectedTokens: 0, strategy: summarizeStrategy({ model: summarizer }) }); + + // system, pinned user, summary (where call_1 was), call_2's turn with its pinned result, then the tail. + expect(result.messages).toEqual([original[0], original[1], summaryOf('S'), original[4], original[5], ...original.slice(-2)]); + expectValidTranscript(result.messages); + const sent = summarizer.calls[0].messages[1].content; + expect(sent).not.toContain('PINNED-GOAL'); + expect(sent).not.toContain('2:'); + expect(sent).toContain('1:'); + expect(sent).toContain('7:'); + }); + + it('survive the two-phase strategy', async () => { + const original = withPins(); + const result = await compactMessages(original, { + protectedTokens: 0, + contextWindow: 1_000, + strategy: twoPhaseStrategy({ model: mockModel(['S']) }), + }); + expect(result.summary).toBe('S'); + expect(result.messages.filter(isPinned)).toEqual([original[1], original[5]]); + expectValidTranscript(result.messages); + }); +}); + +describe('twoPhaseStrategy', () => { + it('only prunes when pruning gets under the threshold', async () => { + const summarizer = mockModel([]); + const original = transcript(10); + // ~10,100 tokens; pruning all but the last result leaves ~1,200, under 90% of 2,000. + const result = await compactMessages(original, { + protectedTokens: 0, + contextWindow: 2_000, + strategy: twoPhaseStrategy({ model: summarizer }), + }); + expect(summarizer.calls).toHaveLength(0); + expect(result.summary).toBeUndefined(); + expect(result.prunedToolCallIds).toHaveLength(9); + expect(result.tokensAfter).toBeLessThan(1_800); + }); + + it('summarizes the pruned conversation when pruning is not enough', async () => { + const summarizer = mockModel(['S']); + const original = transcript(10); + const result = await compactMessages(original, { + protectedTokens: 0, + contextWindow: 1_000, + strategy: twoPhaseStrategy({ model: summarizer }), + }); + expect(result.prunedToolCallIds).toHaveLength(9); + expect(result.summary).toBe('S'); + expect(result.tokensBefore).toBe(estimateTokens(original)); + expect(result.messages).toEqual([original[0], summaryOf('S'), ...original.slice(-2)]); + // The summarizer read the pruned results, not the full ones. + expect(summarizer.calls[0].messages[1].content).toContain('[pruned: search result'); + expect(summarizer.calls[0].messages[1].content).not.toContain(BIG); + }); + + it('keeps the pruned result and reports the error when the summarizer fails', async () => { + const original = transcript(10); + const result = await compactMessages(original, { + protectedTokens: 0, + contextWindow: 1_000, + strategy: twoPhaseStrategy({ model: mockModel([{ error: new Error('down') }]) }), + }); + expect(result.error?.message).toBe('down'); + expect(result.prunedToolCallIds).toHaveLength(9); + expect(result.tokensBefore).toBe(estimateTokens(original)); + expect(result.messages).toHaveLength(original.length); + expectValidTranscript(result.messages); + }); +}); + +describe('async strategies in the hook', () => { + async function runHook(strategy: CompactionStrategy, onCompaction: (info: CompactionInfo) => void) { + const hooks = new HookRegistry(); + hooks.register(createCompactionHook({ strategy, contextWindow: 1_000, protectedTokens: 0, onCompaction })); + const messages = transcript(4); + await hooks.runPreGenerate({ messages, request: { messages } }); + return messages; + } + + it('reports the summary, and reports a fallback error without throwing', async () => { + const infos: CompactionInfo[] = []; + const summarized = await runHook(summarizeStrategy({ model: mockModel(['S']) }), (info) => infos.push(info)); + expect(summarized[1]).toEqual(summaryOf('S')); + expect(infos[0]).toMatchObject({ strategy: 'summarize', summary: 'S' }); + expect(infos[0].error).toBeUndefined(); + + const fellBack = await runHook(summarizeStrategy({ model: mockModel([{ error: new Error('boom') }]) }), (info) => infos.push(info)); + expect(fellBack.some((m) => m.content.startsWith('[pruned: '))).toBe(true); + expect(infos[1]).toMatchObject({ strategy: 'summarize', prunedToolCallIds: ['call_1', 'call_2', 'call_3'] }); + expect(infos[1].error?.message).toBe('boom'); + }); + + it('leaves the run alone and reports the error when a strategy throws', async () => { + const onCompaction = vi.fn<(info: CompactionInfo) => void>(); + const throwing: CompactionStrategy = { name: 'broken', compact: async () => Promise.reject(new Error('bad strategy')) }; + const messages = await runHook(throwing, onCompaction); + expect(messages).toEqual(transcript(4)); + expect(onCompaction).toHaveBeenCalledTimes(1); + expect(onCompaction.mock.calls[0][0]).toMatchObject({ strategy: 'broken', prunedToolCallIds: [] }); + expect(onCompaction.mock.calls[0][0].error?.message).toBe('bad strategy'); + }); +}); + +describe('summarize compaction in a run', () => { + it('summarizes old turns before a model call and keeps the summary in the transcript', async () => { + const fetchPage = defineTool({ + name: 'fetch_page', + description: 'Fetch a page', + input: z.object({ n: z.number() }), + execute: async ({ n }) => `${n}:${BIG}`, + }); + const toolRegistry = new ToolRegistry(); + toolRegistry.register(fetchPage); + const fetchCall = (n: number) => ({ toolCalls: [{ name: 'fetch_page', args: { n }, id: `call_${n}` }] }); + const model = mockModel([fetchCall(1), fetchCall(2), fetchCall(3), fetchCall(4), 'done']); + const summarizer = mockModel(['Read pages 1 and 2.'], { onExhausted: 'repeat-last' }); + + const compactions: CompactionInfo[] = []; + const hooks = new HookRegistry(); + hooks.register( + createCompactionHook({ + contextWindow: 3_000, + protectedTokens: 1_500, + strategy: summarizeStrategy({ model: summarizer }), + onCompaction: (info) => compactions.push(info), + }) + ); + + const result = await AgentExecutor.execute({ + agent: { id: 'a', name: 'Agent', prompt: 'Read pages.', tools: { fetch_page: { tool: 'fetch_page' } } }, + input: 'read four pages', + provider: model, + toolRegistry, + hooks, + maxSteps: 10, + }); + + expect(result.text).toBe('done'); + expect(compactions).toHaveLength(1); + expect(compactions[0]).toMatchObject({ strategy: 'summarize', summary: 'Read pages 1 and 2.' }); + expect(summarizer.calls).toHaveLength(1); + expect(summarizer.calls[0].messages[1].content).toContain('user: read four pages'); + + // The 4th model call is the first to see the summary; every request stays valid and under the threshold. + const summary = summaryOf('Read pages 1 and 2.'); + expect(model.calls[3].messages[1]).toEqual(summary); + for (const req of model.calls) { + expectValidTranscript(req.messages as Message[]); + expect(estimateTokens(req.messages as Message[])).toBeLessThan(3_000 * 0.9); + } + // The summary persists in the run's transcript, after the system prompt. + expect(result.messages[1]).toEqual(summary); + expect(result.messages.filter((m) => m.role === 'tool').map((m) => m.toolCallId)).toEqual(['call_3', 'call_4']); + expectValidTranscript(result.messages); + }); +}); diff --git a/src/context/compaction.test.ts b/src/context/compaction.test.ts index 7392f6195..922ee5144 100644 --- a/src/context/compaction.test.ts +++ b/src/context/compaction.test.ts @@ -49,9 +49,9 @@ function expectValidTranscript(messages: Message[]): void { } describe('pruneToolResultsStrategy / compactMessages', () => { - it('replaces old tool results with a marker and keeps the protected tail intact and in order', () => { + it('replaces old tool results with a marker and keeps the protected tail intact and in order', async () => { const original = transcript(10); - const result = compactMessages(original, { protectedTokens: 3_000, strategy: pruneToolResultsStrategy() }); + const result = await compactMessages(original, { protectedTokens: 3_000, strategy: pruneToolResultsStrategy() }); expect(result.messages).toHaveLength(original.length); expect(result.messages.map((m) => m.role)).toEqual(original.map((m) => m.role)); @@ -74,10 +74,10 @@ describe('pruneToolResultsStrategy / compactMessages', () => { expectValidTranscript(result.messages); }); - it('never touches system messages, user messages or assistant turns', () => { + it('never touches system messages, user messages or assistant turns', async () => { const original = transcript(4, BIG); original.splice(4, 0, { role: 'user', content: BIG }); - const result = compactMessages(original, { protectedTokens: 0 }); + const result = await compactMessages(original, { protectedTokens: 0 }); result.messages.forEach((m, i) => { if (m.role !== 'tool') expect(m).toBe(original[i]); @@ -85,31 +85,31 @@ describe('pruneToolResultsStrategy / compactMessages', () => { expect(result.prunedToolCallIds).toEqual(['call_1', 'call_2', 'call_3']); }); - it('never prunes results of the latest assistant turn, even with no protected tokens', () => { + it('never prunes results of the latest assistant turn, even with no protected tokens', async () => { const original = transcript(3); - const result = compactMessages(original, { protectedTokens: 0 }); + const result = await compactMessages(original, { protectedTokens: 0 }); expect(result.messages.at(-1)).toBe(original.at(-1)); expect(result.prunedToolCallIds).toEqual(['call_1', 'call_2']); }); - it('is pure and idempotent: markers and short results are left alone', () => { + it('is pure and idempotent: markers and short results are left alone', async () => { const original = transcript(5); original.push({ role: 'assistant', content: '', toolCalls: [{ id: 'tiny', type: 'function', function: { name: 'n', arguments: '{}' } }] }); original.push({ role: 'tool', toolCallId: 'tiny', toolName: 'n', content: '"ok"' }); original.push({ role: 'assistant', content: 'done' }); const snapshot = structuredClone(original); - const first = compactMessages(original, { protectedTokens: 0 }); + const first = await compactMessages(original, { protectedTokens: 0 }); expect(original).toEqual(snapshot); expect(first.prunedToolCallIds).toEqual(['call_1', 'call_2', 'call_3', 'call_4', 'call_5']); - const second = compactMessages(first.messages, { protectedTokens: 0 }); + const second = await compactMessages(first.messages, { protectedTokens: 0 }); expect(second.messages).toBe(first.messages); expect(second.prunedToolCallIds).toEqual([]); expect(second.tokensAfter).toBe(second.tokensBefore); }); - it('uses the given strategy', () => { + it('uses the given strategy', async () => { const strategy: CompactionStrategy = { name: 'drop-all', compact: ({ messages, estimateTokens: count, contextWindow }) => ({ @@ -119,7 +119,7 @@ describe('pruneToolResultsStrategy / compactMessages', () => { prunedToolCallIds: [], }), }; - const result = compactMessages(transcript(2), { strategy, model: 'gpt-4o-mini' }); + const result = await compactMessages(transcript(2), { strategy, model: 'gpt-4o-mini' }); expect(result.messages).toHaveLength(2); expect(result.tokensAfter).toBe(128_000); // gpt-4o-mini's window from the registry }); diff --git a/src/context/compaction.ts b/src/context/compaction.ts index 96e427700..ac7fee2b2 100644 --- a/src/context/compaction.ts +++ b/src/context/compaction.ts @@ -1,6 +1,7 @@ /** - * Context compaction (LOU-W2): keep a long run under its model's context - * window by pruning old tool results. + * Context compaction (LOU-W2, LOU-W3): keep a long run under its model's + * context window by pruning old tool results and, when that is not enough, + * folding old turns into a model-written summary. * * Ships as an `AgentHook` (`createCompactionHook()`), so it needs no changes * to the execution loop: `preGenerate` runs before every model call, and @@ -11,7 +12,8 @@ * `ExecutionResult.messages`, and is not pruned again on the next step. */ -import type { Message } from '../providers/llm'; +import type { LLMProvider, Message } from '../providers/llm'; +import { resolveProvider } from '../providers/resolveProvider'; import type { AgentHook, GenerateHookContext } from '../execution/hooks'; import { estimateTokens } from '../models/estimateTokens'; import { getModelInfo } from '../models/registry'; @@ -29,6 +31,10 @@ export interface CompactionInput { contextWindow: number; /** How many of the most recent tokens to leave untouched. */ protectedTokens: number; + /** The size (`thresholdPercent` of the window) a compacted conversation should end up under. */ + thresholdTokens: number; + /** The run's cancellation signal, for strategies that call a model. */ + signal?: AbortSignal; } /** What a {@link CompactionStrategy} returns. */ @@ -39,12 +45,32 @@ export interface CompactionResult { tokensAfter: number; /** `toolCallId`s whose results were replaced by a marker. */ prunedToolCallIds: string[]; + /** The summary that replaced old turns, when the strategy summarized. */ + summary?: string; + /** Why the strategy fell back (e.g. the summarizer failed); `messages` is then the fallback result. */ + error?: Error; } -/** A way of shrinking a conversation. */ +/** A way of shrinking a conversation. `compact()` may be async (e.g. to call a model). */ export interface CompactionStrategy { name: string; - compact(input: CompactionInput): CompactionResult; + compact(input: CompactionInput): CompactionResult | Promise; +} + +/** + * Returns a copy of `message` marked as pinned (`metadata.pinned: true`). + * The built-in strategies never prune or summarize a pinned message. + * + * @example + * messages.push(pinMessage({ role: 'user', content: 'Always answer in French.' })); + */ +export function pinMessage(message: Message): Message { + return { ...message, metadata: { ...message.metadata, pinned: true } }; +} + +/** Whether `message` was marked with {@link pinMessage}. */ +export function isPinned(message: Message): boolean { + return message.metadata?.pinned === true; } const DEFAULT_PROTECTED_TOKENS = 40_000; @@ -54,9 +80,11 @@ const DEFAULT_THRESHOLD_PERCENT = 0.9; const PRUNED_MARKER = /^\[pruned: .* result, \d+ chars\]$/; +const toError = (cause: unknown): Error => (cause instanceof Error ? cause : new Error(String(cause))); + /** The message with its result replaced by a marker, or `undefined` when it is not a tool result worth pruning. */ function prunedToolResult(message: Message): Message | undefined { - if (message.role !== 'tool' || PRUNED_MARKER.test(message.content)) return undefined; + if (message.role !== 'tool' || isPinned(message) || PRUNED_MARKER.test(message.content)) return undefined; const marker = `[pruned: ${message.toolName ?? 'tool'} result, ${message.content.length} chars]`; return marker.length < message.content.length ? { ...message, content: marker } : undefined; } @@ -83,20 +111,150 @@ function protectedTailStart(messages: Message[], count: CompactionTokenCounter, * short marker such as `[pruned: search result, 18234 chars]`. Only `tool` * messages change (system, user and assistant messages, tool calls included, * are kept as they are), so every tool call still has its result and the - * transcript stays valid. A result that is already a marker, or shorter than - * one, is left alone, so running it again changes nothing. + * transcript stays valid. A pinned result, a result that is already a + * marker, or one shorter than a marker is left alone, so running it again + * changes nothing. */ export function pruneToolResultsStrategy(): CompactionStrategy { return { name: 'prune-tool-results', - compact({ messages, estimateTokens: count, protectedTokens }) { + compact: pruneToolResults, + }; +} + +function pruneToolResults({ messages, estimateTokens: count, protectedTokens }: CompactionInput): CompactionResult { + const tokensBefore = count(messages); + const tailStart = protectedTailStart(messages, count, protectedTokens); + const compacted = messages.map((message, index) => (index < tailStart && prunedToolResult(message)) || message); + const pruned = compacted.filter((message, index) => message !== messages[index]); + const prunedToolCallIds = pruned.flatMap((message) => (message.toolCallId ? [message.toolCallId] : [])); + if (pruned.length === 0) return { messages, tokensBefore, tokensAfter: tokensBefore, prunedToolCallIds }; + return { messages: compacted, tokensBefore, tokensAfter: count(compacted), prunedToolCallIds }; +} + +/** Options for {@link summarizeStrategy} and {@link twoPhaseStrategy}. */ +export interface SummarizeStrategyOptions { + /** The summarizer: a provider, or a `"provider/model"` spec for `resolveProvider()` (a cheap model is fine). */ + model: LLMProvider | string; + /** Recent tokens kept as they are. Defaults to the hook's `protectedTokens`. */ + protectedTokens?: number; + /** Instruction sent to the summarizer as its system message. Defaults to {@link DEFAULT_SUMMARY_PROMPT}. */ + prompt?: string; + /** `maxTokens` for the summary call. Defaults to the provider's default. */ + maxSummaryTokens?: number; +} + +/** The summarizer's default instruction. */ +export const DEFAULT_SUMMARY_PROMPT = `You compact the history of an AI agent's conversation. Summarize it so the agent can continue without it: +1. Goal: what the user wants. +2. Instructions: directives and constraints the user gave. +3. Progress: what has been done, which tools were called and what they returned. +4. State: where things stand and what is still pending. +5. Details: names, ids, paths, numbers and values needed to continue. +Write only the summary.`; + +/** The first line of the message that replaces summarized turns. */ +export const SUMMARY_HEADER = '[Conversation summary]'; + +function renderForSummary(messages: Message[]): string { + return messages + .map((m) => { + const calls = (m.toolCalls ?? []).map((c) => `\n[called ${c.function.name}(${c.function.arguments})]`).join(''); + return `${m.role === 'tool' ? `tool ${m.toolName ?? ''}`.trim() : m.role}: ${m.content}${calls}`; + }) + .join('\n\n'); +} + +/** + * Splits the messages before the protected tail into the ones to summarize + * and the ones to keep. An assistant turn and the tool results that follow it + * form one group, kept or folded whole, so no tool call loses its result; a + * group is kept if it holds a system or pinned message. The tail always holds + * the last message and starts on a group boundary. `summaryAt` is where the + * summary goes in `kept`: the position of the first folded group. + */ +function splitForSummary(messages: Message[], count: CompactionTokenCounter, protectedTokens: number) { + let tailStart = Math.min(protectedTailStart(messages, count, protectedTokens), messages.length - 1); + while (tailStart > 0 && messages[tailStart].role === 'tool') tailStart--; + const kept: Message[] = []; + const folded: Message[] = []; + let summaryAt = -1; + for (let i = 0; i < tailStart; ) { + let end = i + 1; + if (messages[i].role === 'assistant') while (end < tailStart && messages[end].role === 'tool') end++; + const group = messages.slice(i, end); + if (group.some((m) => m.role === 'system' || isPinned(m))) { + kept.push(...group); + } else { + if (summaryAt < 0) summaryAt = kept.length; + folded.push(...group); + } + i = end; + } + return { kept, folded, summaryAt, tail: messages.slice(tailStart) }; +} + +/** + * Replaces the turns before the protected tail with one `user` message, + * `"[Conversation summary]\n"`, written by `model`. System and + * pinned messages are kept, and an assistant turn is summarized together with + * its tool results, so the transcript stays valid for every provider. If the + * summary call fails, it falls back to {@link pruneToolResultsStrategy} and + * reports the failure as `error`; it never throws. + * + * @example + * createCompactionHook({ strategy: summarizeStrategy({ model: 'openai/gpt-4o-mini' }) }); + */ +export function summarizeStrategy(options: SummarizeStrategyOptions): CompactionStrategy { + let provider: LLMProvider | undefined; + return { + name: 'summarize', + async compact(input) { + const { messages, estimateTokens: count, signal } = input; + const protectedTokens = options.protectedTokens ?? input.protectedTokens; + const { kept, folded, summaryAt, tail } = splitForSummary(messages, count, protectedTokens); const tokensBefore = count(messages); - const tailStart = protectedTailStart(messages, count, protectedTokens); - const compacted = messages.map((message, index) => (index < tailStart && prunedToolResult(message)) || message); - const pruned = compacted.filter((message, index) => message !== messages[index]); - const prunedToolCallIds = pruned.flatMap((message) => (message.toolCallId ? [message.toolCallId] : [])); - if (pruned.length === 0) return { messages, tokensBefore, tokensAfter: tokensBefore, prunedToolCallIds }; - return { messages: compacted, tokensBefore, tokensAfter: count(compacted), prunedToolCallIds }; + if (folded.length === 0) return { messages, tokensBefore, tokensAfter: tokensBefore, prunedToolCallIds: [] }; + try { + provider ??= typeof options.model === 'string' ? resolveProvider(options.model) : options.model; + const { text } = await provider.generate({ + messages: [ + { role: 'system', content: options.prompt ?? DEFAULT_SUMMARY_PROMPT }, + { role: 'user', content: renderForSummary(folded) }, + ], + maxTokens: options.maxSummaryTokens, + signal, + }); + const summary = text.trim(); + if (!summary) throw new Error('the summarizer returned an empty summary'); + kept.splice(summaryAt, 0, { role: 'user', content: `${SUMMARY_HEADER}\n${summary}` }); + const compacted = [...kept, ...tail]; + return { messages: compacted, tokensBefore, tokensAfter: count(compacted), prunedToolCallIds: [], summary }; + } catch (cause) { + return { ...pruneToolResults({ ...input, protectedTokens }), error: toError(cause) }; + } + }, + }; +} + +/** + * The recommended strategy: prunes old tool results first and, only if the + * conversation is still above the threshold, summarizes what is left with + * {@link summarizeStrategy} (which falls back to the pruned result on failure). + * + * @example + * createCompactionHook({ strategy: twoPhaseStrategy({ model: 'openai/gpt-4o-mini' }) }); + */ +export function twoPhaseStrategy(options: SummarizeStrategyOptions): CompactionStrategy { + const summarize = summarizeStrategy(options); + return { + name: 'two-phase', + async compact(input) { + const pruned = pruneToolResults(input); + if (pruned.tokensAfter <= input.thresholdTokens) return pruned; + const result = await summarize.compact({ ...input, messages: pruned.messages }); + const prunedToolCallIds = [...pruned.prunedToolCallIds, ...result.prunedToolCallIds]; + return { ...result, tokensBefore: pruned.tokensBefore, prunedToolCallIds }; }, }; } @@ -109,17 +267,21 @@ export interface CompactMessagesOptions { contextWindow?: number; /** How many recent tokens to keep intact. Defaults to 40,000. */ protectedTokens?: number; + /** Share of the context window to compact below (`CompactionInput.thresholdTokens`). Defaults to 0.9. */ + thresholdPercent?: number; /** Model id, used for the context-window lookup and passed to the token estimator. */ model?: string; } -function prepare(messages: Message[], options: CompactMessagesOptions) { - const { model, protectedTokens = DEFAULT_PROTECTED_TOKENS } = options; +function prepare(messages: Message[], options: CompactMessagesOptions, signal?: AbortSignal) { + const { model, protectedTokens = DEFAULT_PROTECTED_TOKENS, thresholdPercent = DEFAULT_THRESHOLD_PERCENT } = options; const strategy = options.strategy ?? pruneToolResultsStrategy(); const contextWindow = options.contextWindow ?? (model ? getModelInfo(model)?.contextWindow : undefined) ?? FALLBACK_CONTEXT_WINDOW; const count: CompactionTokenCounter = (input) => estimateTokens(input, { model }); - return { strategy, contextWindow, input: { messages, estimateTokens: count, contextWindow, protectedTokens } }; + const thresholdTokens = thresholdPercent * contextWindow; + const input: CompactionInput = { messages, estimateTokens: count, contextWindow, protectedTokens, thresholdTokens, signal }; + return { strategy, input }; } /** @@ -128,9 +290,9 @@ function prepare(messages: Message[], options: CompactMessagesOptions) { * {@link createCompactionHook}. * * @example - * const { messages: smaller, tokensBefore, tokensAfter } = compactMessages(history, { protectedTokens: 8_000 }); + * const { messages: smaller, tokensBefore, tokensAfter } = await compactMessages(history, { protectedTokens: 8_000 }); */ -export function compactMessages(messages: Message[], options: CompactMessagesOptions = {}): CompactionResult { +export async function compactMessages(messages: Message[], options: CompactMessagesOptions = {}): Promise { const { strategy, input } = prepare(messages, options); return strategy.compact(input); } @@ -142,13 +304,15 @@ export interface CompactionInfo { prunedToolCallIds: string[]; /** The strategy's `name`. */ strategy: string; + /** The summary that replaced old turns, when the strategy summarized. */ + summary?: string; + /** Set when the strategy failed or fell back; the run continues either way. */ + error?: Error; } /** Options for {@link createCompactionHook}. */ export interface CompactionHookOptions extends Omit { - /** Compact once the request is estimated above this share of the context window. Defaults to 0.9. */ - thresholdPercent?: number; - /** Called after each compaction that changed the conversation. */ + /** Called after each compaction that changed the conversation or reported an `error`. */ onCompaction?: (info: CompactionInfo) => void; } @@ -157,7 +321,8 @@ export interface CompactionHookOptions extends Omit; } /**