diff --git a/CHANGELOG.md b/CHANGELOG.md index 67c4889bd..115285860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - 2026-09-28 ### Added +- `createAgent({ retry, fallbackModels })` (LOU-V7.2): `retry` (`withRetry()` options, or `false`) retries failed model calls and defaults to `{ maxRetries: 2 }` for models given as `provider/model` strings (or picked from the environment); a `provider` instance is wrapped only when `retry` is set. `fallbackModels` are `provider/model` strings tried in order once the primary model's retries are used up (`withFallback([withRetry(primary), withRetry(fallback), ...])`). `agent.stream()`, `session.stream()` and `AgentExecutor.stream()` report them as two new typed events, `provider.retry` (`attempt`, `maxRetries`, `delayMs`, `error: { message, category? }`, `provider`) and `provider.fallback` (`from`, `to`, `error: { message }`), for any `withRetry()` / `withFallback()` provider. Additive: the event schema version stays 1. - Context compaction (LOU-W2): `createCompactionHook()` returns an `AgentHook` that, before a model call estimated above `thresholdPercent` (default 0.9) of the model's context window, replaces tool results older than the newest `protectedTokens` (default 40,000) with a `[pruned: result, N chars]` marker, in the run's transcript itself. `compactMessages()` compacts a conversation by hand; `CompactionStrategy` / `pruneToolResultsStrategy()` make the strategy pluggable. See docs/compaction.md. - `AgentSpec.mcpServers` (LOU-D20): a validated map of MCP servers (stdio `command`/`args`/`env` or HTTP `url`/`headers`) in agent spec files. `loadSpec()` reports a bad entry with its name, `loushy doctor` reads the validated field instead of the raw file, and `specToAgent()` exposes the parsed servers as `agent.mcpServers` (connecting them is TODO(LOU-D20.2)). Existing specs are unaffected. - `createAgent()` agents can pause for approval (LOU-D21): a `needsApproval` tool no longer fails the run with "requires approval but no approvalStore". The run pauses (`finishReason: 'awaiting-approval'`) in a per-agent `InMemoryApprovalStore` (or the new `approvalStore` option), and `agent.approvals.list()` / `agent.approvals.resolve({ id, approved, note? })` continue it, in its session if it paused in one. The `approve` option decides calls in code without pausing. @@ -19,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - README revamped into a short front page (LOU-D52); the details it dropped moved to `docs/` (new pages: tools, approvals, providers, cli, flows, guardrails, utilities). ### Changed +- Retries no longer stack (LOU-V7.2): the built-in providers (`OpenAIProvider`, `AnthropicProvider`, `OllamaProvider`, `OpenRouterProvider`) now pass `maxRetries` to the `ai` SDK calls: `config.maxRetries`, or 2 (the `ai` SDK's default, so behaviour is unchanged) when unset. `createAgent()` builds the providers it resolves from strings with `maxRetries: 0` and retries in its `withRetry()` wrapper instead, so retries happen in one place (at most 3 calls by default, as before) and each one is observable. If you wrap a built-in provider in `withRetry()` / `resilientProvider()` yourself, build it with `maxRetries: 0`; note that a `maxRetries` you already set in a provider config is now honoured by the `ai` SDK. - `AgentType` is optional and deprecated (LOU-D34, not breaking): `AgentBuilder.build()` no longer requires `setType()` and `AgentConfig.agentType` is now optional (`createAgent()` and `specToAgent()` agents carry no type). `AgentType`, `setType()` and the `agent-types` registry/validators are marked `@deprecated`: they have no runtime effect and will be removed in the next minor; they stay exported for now. Drop your `setType(...)` calls. - BREAKING: `encrypt()` now generates a random salt per call instead of a hardcoded one. Ciphertext produced before this change cannot be decrypted with the new code and must be re-encrypted. - BREAKING: The mock repositories (`MockAgentRepository`, `MockSessionRepository`, `MockResultRepository`, `MockMemoryRepository`, `MockAttachmentRepository` — previously re-exported from `src/data/mocks.ts` via the package root/`./data` subpath; this package has no `createMockRepositories` factory) are no longer exported from the package root. Import them from `@loushy/build-ai-agent/testing` instead. diff --git a/docs/configuration.md b/docs/configuration.md index 84a20f7ea..97d8ea5a1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -180,6 +180,40 @@ what the examples and the Quick Start use by default. ## Provider retries and fallback +`createAgent()` retries failed model calls on its own, and can fall back to +other models: + +```ts +import { createAgent } from '@loushy/build-ai-agent'; + +const agent = createAgent({ + model: 'openai/gpt-4o-mini', + retry: { maxRetries: 3, backoff: { initialMs: 1000 } }, // default { maxRetries: 2 }; false turns it off + fallbackModels: ['anthropic/claude-3-5-haiku-latest'], // tried in order once the retries are used up +}); + +for await (const event of agent.stream('Hello!')) { + if (event.type === 'provider.retry') console.warn(`retry ${event.attempt} in ${event.delayMs}ms: ${event.error.message}`); + if (event.type === 'provider.fallback') console.warn(`falling back from ${event.from} to ${event.to}`); +} +``` + +- `retry` takes the `withRetry()` options below. It applies to the `model` + string (or the model picked from the environment) and to every + `fallbackModels` entry. createAgent builds those providers with the `ai` + SDK's own retries off (`maxRetries: 0`), so retries happen in one place and + the default `{ maxRetries: 2 }` makes as many calls as before. +- A `provider` instance you pass keeps its own retry behaviour; it is wrapped + in `withRetry()` only when you set `retry`. `fallbackModels` work with it too. +- `fallbackModels` are `provider/model` strings, resolved like `model` when the + agent is created. The agent runs `withFallback([withRetry(primary), withRetry(fallback1), ...])`: + every call starts with the primary model. +- `stream()` and `session.stream()` report each retry as a `provider.retry` + event and each switch as `provider.fallback` (see + [Streaming](./streaming.md#event-schema-version-1)). `send()` returns the + final result as before. + +To build the same thing by hand, or for providers you construct yourself, `withRetry(provider, options)` and `withFallback(providers, options)` wrap any `LLMProvider` and return another one, so they compose and can be passed anywhere a provider is accepted: @@ -217,8 +251,11 @@ const agent = createAgent({ prompt: 'You are helpful.', provider }); `name` and `defaultModel` report the provider that served the latest call. - `resilientProvider(provider, { maxRetries, timeout })` applies the `LLMProviderConfig` fields of the same names. -- The built-in providers also run the `ai` SDK's own internal retries (2 by - default) inside each attempt. +- The built-in providers pass their config's `maxRetries` (default 2) to the + `ai` SDK's own internal retries, which run inside each `withRetry` attempt. + Build a provider you wrap with `maxRetries: 0` + (`new OpenAIProvider({ apiKey, maxRetries: 0 })`) to retry in one place. + `createAgent()` does this for the models it resolves. ## `createAgent()` options @@ -238,6 +275,8 @@ const agent = createAgent({ prompt: 'You are helpful.', provider }); | `approvalStore` | Where a `needsApproval` pause is saved (default: a per-agent `InMemoryApprovalStore`); see [Approvals](./approvals.md). | | `approve` | `(call) => boolean`: decide approvals in code instead of pausing. | | `projectInstructions` | `true` or `{ cwd?, files? }`: append the nearest `AGENTS.md` / `CLAUDE.md` to the instructions (off by default; see [Project instructions](#project-instructions)). | +| `retry` | `withRetry()` options for failed model calls, or `false`. Default `{ maxRetries: 2 }` for `model` strings; a `provider` instance is wrapped only when set. See [Provider retries and fallback](#provider-retries-and-fallback). | +| `fallbackModels` | `provider/model` strings tried in order when the model still fails after its retries. | With neither `model` nor `provider`, `createAgent()` resolves from the environment: `LOUSHY_MODEL` (a `'provider/model'` string) if set, otherwise diff --git a/docs/providers.md b/docs/providers.md index c24d4a6bf..b42b83814 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -49,6 +49,28 @@ Misconfiguration errors say how to fix themselves: a missing key names the variable, an unknown prefix lists the supported ones and suggests the closest, and a missing peer package prints the exact `npm install` command. +### Retries and fallback models + +`createAgent()` retries a failed model call (rate limit, timeout, network +error, 5xx) twice by default, and with `fallbackModels` moves on to the next +model when the call still fails: + +```ts +import { createAgent } from '@loushy/build-ai-agent'; + +const agent = createAgent({ + model: 'openai/gpt-4o-mini', + retry: { maxRetries: 3 }, // or false + fallbackModels: ['anthropic/claude-3-5-haiku-latest', 'openrouter/meta-llama/llama-3.1-70b-instruct'], +}); +``` + +The `model` string and each fallback are resolved with the `ai` SDK's own +retries off, so `retry` is the only retry layer. A `provider` instance is +wrapped only when you set `retry`. `agent.stream()` reports `provider.retry` +and `provider.fallback` events. Details in +[Provider retries and fallback](./configuration.md#provider-retries-and-fallback). + ## Which model runs? In order: the agent's own `settings.model` (set with diff --git a/docs/streaming.md b/docs/streaming.md index 5d0b6ea87..863eb1fe1 100644 --- a/docs/streaming.md +++ b/docs/streaming.md @@ -109,6 +109,8 @@ The event types and their extra fields: | `approval.requested` | `approvalId: string`, `toolCallId`, `toolName`, `args: Record` | A tool call needs a human decision. The run then stops; resume it with `agent.approvals.resolve()` or `resumeAfterApproval()` (see [Approvals](./approvals.md)). | | `step.done` | `step: number`, `finishReason: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | A step ends. `finishReason` is the model's (`'stop'`, `'tool_calls'`, `'length'`, ...), or `'awaiting-approval'`, `'aborted'` or `'error'` when the step ended that way (a run that runs out of `maxSteps` still wanting to continue ends with `run.done` `'max-steps'`). `usage` is this step's model call, absent when the call produced no response. | | `error` | `error: { name: string, message: string }` | An error. If it ends the run, `run.done` with `finishReason: 'error'` follows. A provider error retried under `surfaceRetryableProviderErrors` is followed by further steps instead. | +| `provider.retry` | `attempt: number`, `maxRetries: number`, `delayMs: number`, `error: { message: string, category?: string }`, `provider: string` | A model call failed and is retried after `delayMs` (`createAgent({ retry })` or any `withRetry()` provider). `attempt` is the attempt that failed (1 = first); `category` is `'rate-limit'`, `'timeout'`, ... and absent when unknown (a 5xx, for example). | +| `provider.fallback` | `from: string`, `to: string`, `error: { message: string }` | A model call still failed after its retries and the next provider takes over (`createAgent({ fallbackModels })` or any `withFallback()` provider). `from` and `to` are provider names. | | `run.done` | `finishReason: string`, `text: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | Last event of every run, exactly once, including aborted, failed and awaiting-approval runs. `finishReason` and `text` match `run.result` (`'max-steps'` when the `maxSteps` budget ran out while the model still wanted to continue); a failed run has `finishReason: 'error'`, `text: ''` and no `usage`. | Optional fields are left out when they have no value. They are never @@ -120,7 +122,8 @@ Optional fields are left out when they have no value. They are never - Each `step.start` is followed by exactly one `step.done` with the same `step`, before the next `step.start`. Everything a step does happens between the two. -- Inside a step: `text.delta` events, then `text.done`, then the tool events. +- Inside a step: `provider.retry` / `provider.fallback` events (if the model + call fails), then `text.delta` events, then `text.done`, then the tool events. - Tool calls of one step run in parallel (see `toolConcurrency`): `tool.start` events come in the model's call order and `tool.done` / `tool.error` events in completion order. Match them by `toolCallId`. diff --git a/llms-full.txt b/llms-full.txt index dbaf02135..6a077e06d 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2277,6 +2277,40 @@ what the examples and the Quick Start use by default. ## Provider retries and fallback +`createAgent()` retries failed model calls on its own, and can fall back to +other models: + +```ts +import { createAgent } from '@loushy/build-ai-agent'; + +const agent = createAgent({ + model: 'openai/gpt-4o-mini', + retry: { maxRetries: 3, backoff: { initialMs: 1000 } }, // default { maxRetries: 2 }; false turns it off + fallbackModels: ['anthropic/claude-3-5-haiku-latest'], // tried in order once the retries are used up +}); + +for await (const event of agent.stream('Hello!')) { + if (event.type === 'provider.retry') console.warn(`retry ${event.attempt} in ${event.delayMs}ms: ${event.error.message}`); + if (event.type === 'provider.fallback') console.warn(`falling back from ${event.from} to ${event.to}`); +} +``` + +- `retry` takes the `withRetry()` options below. It applies to the `model` + string (or the model picked from the environment) and to every + `fallbackModels` entry. createAgent builds those providers with the `ai` + SDK's own retries off (`maxRetries: 0`), so retries happen in one place and + the default `{ maxRetries: 2 }` makes as many calls as before. +- A `provider` instance you pass keeps its own retry behaviour; it is wrapped + in `withRetry()` only when you set `retry`. `fallbackModels` work with it too. +- `fallbackModels` are `provider/model` strings, resolved like `model` when the + agent is created. The agent runs `withFallback([withRetry(primary), withRetry(fallback1), ...])`: + every call starts with the primary model. +- `stream()` and `session.stream()` report each retry as a `provider.retry` + event and each switch as `provider.fallback` (see + [Streaming](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/streaming.md#event-schema-version-1)). `send()` returns the + final result as before. + +To build the same thing by hand, or for providers you construct yourself, `withRetry(provider, options)` and `withFallback(providers, options)` wrap any `LLMProvider` and return another one, so they compose and can be passed anywhere a provider is accepted: @@ -2314,8 +2348,11 @@ const agent = createAgent({ prompt: 'You are helpful.', provider }); `name` and `defaultModel` report the provider that served the latest call. - `resilientProvider(provider, { maxRetries, timeout })` applies the `LLMProviderConfig` fields of the same names. -- The built-in providers also run the `ai` SDK's own internal retries (2 by - default) inside each attempt. +- The built-in providers pass their config's `maxRetries` (default 2) to the + `ai` SDK's own internal retries, which run inside each `withRetry` attempt. + Build a provider you wrap with `maxRetries: 0` + (`new OpenAIProvider({ apiKey, maxRetries: 0 })`) to retry in one place. + `createAgent()` does this for the models it resolves. ## `createAgent()` options @@ -2335,6 +2372,8 @@ const agent = createAgent({ prompt: 'You are helpful.', provider }); | `approvalStore` | Where a `needsApproval` pause is saved (default: a per-agent `InMemoryApprovalStore`); see [Approvals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/approvals.md). | | `approve` | `(call) => boolean`: decide approvals in code instead of pausing. | | `projectInstructions` | `true` or `{ cwd?, files? }`: append the nearest `AGENTS.md` / `CLAUDE.md` to the instructions (off by default; see [Project instructions](#project-instructions)). | +| `retry` | `withRetry()` options for failed model calls, or `false`. Default `{ maxRetries: 2 }` for `model` strings; a `provider` instance is wrapped only when set. See [Provider retries and fallback](#provider-retries-and-fallback). | +| `fallbackModels` | `provider/model` strings tried in order when the model still fails after its retries. | With neither `model` nor `provider`, `createAgent()` resolves from the environment: `LOUSHY_MODEL` (a `'provider/model'` string) if set, otherwise @@ -3378,6 +3417,28 @@ Misconfiguration errors say how to fix themselves: a missing key names the variable, an unknown prefix lists the supported ones and suggests the closest, and a missing peer package prints the exact `npm install` command. +### Retries and fallback models + +`createAgent()` retries a failed model call (rate limit, timeout, network +error, 5xx) twice by default, and with `fallbackModels` moves on to the next +model when the call still fails: + +```ts +import { createAgent } from '@loushy/build-ai-agent'; + +const agent = createAgent({ + model: 'openai/gpt-4o-mini', + retry: { maxRetries: 3 }, // or false + fallbackModels: ['anthropic/claude-3-5-haiku-latest', 'openrouter/meta-llama/llama-3.1-70b-instruct'], +}); +``` + +The `model` string and each fallback are resolved with the `ai` SDK's own +retries off, so `retry` is the only retry layer. A `provider` instance is +wrapped only when you set `retry`. `agent.stream()` reports `provider.retry` +and `provider.fallback` events. Details in +[Provider retries and fallback](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/configuration.md#provider-retries-and-fallback). + ## Which model runs? In order: the agent's own `settings.model` (set with @@ -3809,6 +3870,8 @@ The event types and their extra fields: | `approval.requested` | `approvalId: string`, `toolCallId`, `toolName`, `args: Record` | A tool call needs a human decision. The run then stops; resume it with `agent.approvals.resolve()` or `resumeAfterApproval()` (see [Approvals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/approvals.md)). | | `step.done` | `step: number`, `finishReason: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | A step ends. `finishReason` is the model's (`'stop'`, `'tool_calls'`, `'length'`, ...), or `'awaiting-approval'`, `'aborted'` or `'error'` when the step ended that way (a run that runs out of `maxSteps` still wanting to continue ends with `run.done` `'max-steps'`). `usage` is this step's model call, absent when the call produced no response. | | `error` | `error: { name: string, message: string }` | An error. If it ends the run, `run.done` with `finishReason: 'error'` follows. A provider error retried under `surfaceRetryableProviderErrors` is followed by further steps instead. | +| `provider.retry` | `attempt: number`, `maxRetries: number`, `delayMs: number`, `error: { message: string, category?: string }`, `provider: string` | A model call failed and is retried after `delayMs` (`createAgent({ retry })` or any `withRetry()` provider). `attempt` is the attempt that failed (1 = first); `category` is `'rate-limit'`, `'timeout'`, ... and absent when unknown (a 5xx, for example). | +| `provider.fallback` | `from: string`, `to: string`, `error: { message: string }` | A model call still failed after its retries and the next provider takes over (`createAgent({ fallbackModels })` or any `withFallback()` provider). `from` and `to` are provider names. | | `run.done` | `finishReason: string`, `text: string`, `usage?: { promptTokens, completionTokens, totalTokens }` | Last event of every run, exactly once, including aborted, failed and awaiting-approval runs. `finishReason` and `text` match `run.result` (`'max-steps'` when the `maxSteps` budget ran out while the model still wanted to continue); a failed run has `finishReason: 'error'`, `text: ''` and no `usage`. | Optional fields are left out when they have no value. They are never @@ -3820,7 +3883,8 @@ Optional fields are left out when they have no value. They are never - Each `step.start` is followed by exactly one `step.done` with the same `step`, before the next `step.start`. Everything a step does happens between the two. -- Inside a step: `text.delta` events, then `text.done`, then the tool events. +- Inside a step: `provider.retry` / `provider.fallback` events (if the model + call fails), then `text.delta` events, then `text.done`, then the tool events. - Tool calls of one step run in parallel (see `toolConcurrency`): `tool.start` events come in the model's call order and `tool.done` / `tool.error` events in completion order. Match them by `toolCallId`. diff --git a/src/createAgent.test.ts b/src/createAgent.test.ts index 409969b95..50c4a5564 100644 --- a/src/createAgent.test.ts +++ b/src/createAgent.test.ts @@ -48,7 +48,8 @@ describe('createAgent', () => { const agent = createAgent({ model: 'openai/gpt-4o-mini', instructions: 'Be brief.' }); const { text } = await agent.send('Hello!'); - expect(createSpy).toHaveBeenCalledWith('openai', { defaultModel: 'gpt-4o-mini', apiKey: 'sk-test' }); + // maxRetries: 0 - createAgent() retries in its own withRetry() wrapper (LOU-V7.2). + expect(createSpy).toHaveBeenCalledWith('openai', { maxRetries: 0, defaultModel: 'gpt-4o-mini', apiKey: 'sk-test' }); expect(text).toBe('Hi from the model'); }); @@ -83,6 +84,7 @@ describe('createAgent', () => { createAgent({ instructions: 'x' }); expect(createSpy).toHaveBeenCalledWith('anthropic', { + maxRetries: 0, defaultModel: 'claude-3-5-haiku-latest', apiKey: 'sk-ant', }); diff --git a/src/createAgent.ts b/src/createAgent.ts index 0da8a34e6..c4da18509 100644 --- a/src/createAgent.ts +++ b/src/createAgent.ts @@ -22,6 +22,7 @@ import { LLMProvider } from './providers/llm'; import { ToolRegistry } from './tools/ToolRegistry'; import { ToolDescriptor } from './types'; import { modelFromEnv, resolveProviderSpec } from './providers/providerSpec'; +import { withFallback, withRetry, type WithRetryOptions } from './providers/resilience'; import type { DefinedTool } from './tools/defineTool'; import { ToolConcurrency, assertToolConcurrency } from './execution/toolBatch'; import type { Skill } from './skills/defineSkill'; @@ -140,6 +141,33 @@ export interface CreateAgentBase { * ``` */ approve?: ApproveToolCall; + /** + * Retries of a failed model call (LOU-V7.2), with `withRetry()`: rate + * limits, timeouts, network errors and 5xx responses, with exponential + * backoff. Defaults to `{ maxRetries: 2 }` for every model given as a + * `provider/model` string (resolved with the `ai` SDK's own retries off, so + * this is the only retry layer); `false` turns retries off. A `provider` + * instance you pass is wrapped only when you set `retry`. `stream()` + * reports each retry as a `provider.retry` event. + * + * @example + * ```ts + * createAgent({ model: 'openai/gpt-4o-mini', retry: { maxRetries: 4, backoff: { initialMs: 1000 } } }); + * ``` + */ + retry?: WithRetryOptions | false; + /** + * `provider/model` strings tried in order when the primary model's call + * still fails after its retries (LOU-V7.2), each resolved like `model` and + * retried with `retry`. Every call starts with the primary. `stream()` + * reports each switch as a `provider.fallback` event. + * + * @example + * ```ts + * createAgent({ model: 'openai/gpt-4o-mini', fallbackModels: ['anthropic/claude-3-5-haiku-latest'] }); + * ``` + */ + fallbackModels?: readonly string[]; } /** @@ -367,10 +395,28 @@ function resolveInstructions(config: CreateAgentConfig): string { return config.instructions ?? config.prompt ?? DEFAULT_INSTRUCTIONS; } -/** The provider to run with: the given instance, else `model` resolved from env, else the env's choice. */ +/** The default `retry` for models given as strings: as many retries as the `ai` SDK makes on its own. */ +const DEFAULT_RETRY: WithRetryOptions = { maxRetries: 2 }; + +/** + * The provider to run with: the given instance, else `model` resolved (else + * the env's choice), wrapped in `withRetry()` and, with `fallbackModels`, + * `withFallback()` (LOU-V7.2). + */ function resolveModelSource(config: CreateAgentConfig): LLMProvider { - if (config.provider) return config.provider; - return resolveProviderSpec(config.model ?? modelFromEnv('createAgent'), 'createAgent'); + const { retry, fallbackModels = [] } = config; + const resolve = (spec: string) => { + // maxRetries: 0 turns off the 'ai' SDK's own retries: withRetry() is the only layer. + const provider = resolveProviderSpec(spec, 'createAgent', { maxRetries: 0 }); + return retry === false ? provider : withRetry(provider, retry ?? DEFAULT_RETRY); + }; + let primary: LLMProvider; + if (config.provider) { + primary = retry ? withRetry(config.provider, retry) : config.provider; + } else { + primary = resolve(config.model ?? modelFromEnv('createAgent')); + } + return fallbackModels.length > 0 ? withFallback([primary, ...fallbackModels.map(resolve)]) : primary; } /** diff --git a/src/createAgentResilience.test.ts b/src/createAgentResilience.test.ts new file mode 100644 index 000000000..38144c73a --- /dev/null +++ b/src/createAgentResilience.test.ts @@ -0,0 +1,185 @@ +/** + * LOU-V7.2: createAgent({ retry, fallbackModels }) and the typed + * `provider.retry` / `provider.fallback` stream events. No network: the + * registry's create() is stubbed, as in createAgent.test.ts. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { APICallError } from 'ai'; +import { createAgent } from './createAgent'; +import { LLMProviderRegistry, type LLMProvider, type LLMProviderConfig } from './providers/llm'; +import { mockModel, type MockModel } from './testing'; +import type { AgentEvent, AgentRun } from './execution'; + +const ENV_VARS = ['LOUSHY_MODEL', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'OPENROUTER_API_KEY', 'OLLAMA_BASE_URL']; + +function apiError(statusCode: number, headers?: Record): APICallError { + return new APICallError({ + message: `HTTP ${statusCode}`, + url: 'https://api.example.com/v1/chat', + requestBodyValues: {}, + statusCode, + responseHeaders: headers, + isRetryable: statusCode >= 500 || statusCode === 429, + }); +} + +/** `model` under another provider name, so a fallback is visible in events. */ +function named(model: MockModel, name: string): LLMProvider { + return { + name, + defaultModel: model.defaultModel, + generate: (call) => model.generate(call), + stream: (call) => model.stream(call), + supportsTools: (id) => model.supportsTools(id), + supportsStreaming: (id) => model.supportsStreaming(id), + getModels: () => model.getModels(), + }; +} + +/** Stubs the registry: each provider name resolves to the given provider; returns the configs it was built with. */ +function stubRegistry(providers: Record): Array<[string, LLMProviderConfig]> { + const created: Array<[string, LLMProviderConfig]> = []; + vi.spyOn(LLMProviderRegistry, 'create').mockImplementation((name, config) => { + created.push([name, config]); + return providers[name]; + }); + return created; +} + +async function collect(run: AgentRun): Promise { + const events: AgentEvent[] = []; + for await (const event of run) events.push(event); + return events; +} + +const fast = { backoff: { initialMs: 1, jitter: false } }; + +describe('createAgent retry and fallbackModels (LOU-V7.2)', () => { + beforeEach(() => { + for (const name of ENV_VARS) vi.stubEnv(name, ''); + vi.stubEnv('OPENAI_API_KEY', 'sk-openai'); + vi.stubEnv('ANTHROPIC_API_KEY', 'sk-ant'); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it('retries by default (maxRetries 2) with the ai SDK retries off, and streams provider.retry', async () => { + const model = mockModel([{ error: apiError(429, { 'retry-after': '0' }) }, 'recovered']); + const created = stubRegistry({ openai: named(model, 'openai') }); + const agent = createAgent({ model: 'openai/gpt-4o-mini' }); + + const run = agent.stream('hi'); + const events = await collect(run); + + expect(created).toEqual([['openai', { maxRetries: 0, defaultModel: 'gpt-4o-mini', apiKey: 'sk-openai' }]]); + expect((await run.result).text).toBe('recovered'); + expect(model.calls).toHaveLength(2); + expect(events.map((e) => e.type)).toEqual([ + 'run.start', + 'step.start', + 'provider.retry', + 'text.delta', + 'text.done', + 'step.done', + 'run.done', + ]); + expect(events[2]).toMatchObject({ + type: 'provider.retry', + attempt: 1, + maxRetries: 2, + delayMs: 0, + error: { message: 'HTTP 429', category: 'rate-limit' }, + provider: 'openai', + }); + expect(JSON.parse(JSON.stringify(events[2]))).toEqual(events[2]); + }); + + it('send() retries too and returns the final result', async () => { + const model = mockModel([{ error: apiError(503) }, { error: apiError(502) }, 'third time']); + stubRegistry({ openai: named(model, 'openai') }); + + const result = await createAgent({ model: 'openai/gpt-4o-mini', retry: fast }).send('hi'); + + expect(result.text).toBe('third time'); + expect(model.calls).toHaveLength(3); + }); + + it('retry: false fails fast', async () => { + const model = mockModel([{ error: apiError(503) }, 'never reached']); + const created = stubRegistry({ openai: named(model, 'openai') }); + + const agent = createAgent({ model: 'openai/gpt-4o-mini', retry: false }); + + await expect(agent.send('hi')).rejects.toThrow(); + expect(model.calls).toHaveLength(1); + expect(created[0][1].maxRetries).toBe(0); + }); + + it('fallbackModels: switches to the next model after the retries and streams provider.fallback', async () => { + const primary = mockModel([{ error: apiError(503) }, { error: apiError(503) }]); + const fallback = mockModel(['from the fallback'], { defaultModel: 'claude-3-5-haiku-latest' }); + const created = stubRegistry({ openai: named(primary, 'openai'), anthropic: named(fallback, 'anthropic') }); + const agent = createAgent({ + model: 'openai/gpt-4o-mini', + fallbackModels: ['anthropic/claude-3-5-haiku-latest'], + retry: { ...fast, maxRetries: 1 }, + }); + + const run = agent.stream('hi'); + const events = await collect(run); + + expect((await run.result).text).toBe('from the fallback'); + expect(created.map(([name, config]) => [name, config.maxRetries])).toEqual([ + ['openai', 0], + ['anthropic', 0], + ]); + expect(events.filter((e) => e.type.startsWith('provider.'))).toMatchObject([ + { type: 'provider.retry', attempt: 1, maxRetries: 1, provider: 'openai', error: { message: 'HTTP 503' } }, + { type: 'provider.fallback', from: 'openai', to: 'anthropic', error: { message: 'HTTP 503' } }, + ]); + expect(events.find((e) => e.type === 'provider.retry')).not.toHaveProperty('error.category'); // a 5xx is 'unknown' + expect(fallback.lastCall?.model).toBeUndefined(); // the fallback runs on its own default model + }); + + it('fallbackModels also work with a provider instance', async () => { + const primary = mockModel([{ error: apiError(401) }]); + const fallback = mockModel(['fallback answer']); + stubRegistry({ anthropic: named(fallback, 'anthropic') }); + + const result = await createAgent({ provider: primary, fallbackModels: ['anthropic/claude-3-5-haiku-latest'] }).send('hi'); + + expect(result.text).toBe('fallback answer'); + expect(primary.calls).toHaveLength(1); + }); + + it('a provider instance is retried only when retry is set', async () => { + const plain = mockModel([{ error: apiError(503) }, 'unused']); + await expect(createAgent({ provider: plain }).send('hi')).rejects.toThrow(); + expect(plain.calls).toHaveLength(1); + + const retried = mockModel([{ error: apiError(503) }, 'retried']); + const run = createAgent({ provider: retried, retry: fast }).stream('hi'); + const events = await collect(run); + expect((await run.result).text).toBe('retried'); + expect(events.some((e) => e.type === 'provider.retry' && e.provider === 'mock')).toBe(true); + }); + + it('session.stream() carries the provider events', async () => { + const model = mockModel([{ error: apiError(503) }, 'session answer']); + stubRegistry({ openai: named(model, 'openai') }); + const session = createAgent({ model: 'openai/gpt-4o-mini', retry: fast }).session(); + + const events = await collect(session.stream('hi')); + + expect(events.find((e) => e.type === 'provider.retry')).toMatchObject({ attempt: 1, provider: 'openai' }); + expect(events.at(-1)).toMatchObject({ type: 'run.done', text: 'session answer' }); + }); + + it('throws at creation for a bad fallback model string', () => { + stubRegistry({ openai: mockModel([]) }); + expect(() => createAgent({ model: 'openai/gpt-4o-mini', fallbackModels: ['nope'] })).toThrow(/createAgent: expected/); + }); +}); diff --git a/src/execution/agentEvents.test-d.ts b/src/execution/agentEvents.test-d.ts index 6d4f63e35..eb0d8c249 100644 --- a/src/execution/agentEvents.test-d.ts +++ b/src/execution/agentEvents.test-d.ts @@ -3,6 +3,7 @@ import { createAgent } from '../createAgent'; import { createMockProvider } from '../providers/mock'; import type { ExecutionResult } from './AgentExecutor'; import type { AgentRun } from './agentRun'; +import type { CompactedProviderErrorCategory } from './errors'; import { AGENT_EVENT_SCHEMA_VERSION, isToolEvent, @@ -57,6 +58,14 @@ describe('AgentEvent types', () => { } }); + it('narrows provider retry/fallback events on event.type (LOU-V7.2)', () => { + if (event.type === 'provider.retry') { + expectTypeOf(event.attempt).toBeNumber(); + expectTypeOf(event.error.category).toEqualTypeOf(); + } + if (event.type === 'provider.fallback') expectTypeOf(event.error).toEqualTypeOf<{ message: string }>(); + }); + it('covers every event type', () => { expectTypeOf().toEqualTypeOf< | 'run.start' @@ -69,6 +78,8 @@ describe('AgentEvent types', () => { | 'approval.requested' | 'step.done' | 'error' + | 'provider.retry' + | 'provider.fallback' | 'run.done' >(); }); diff --git a/src/execution/agentEvents.ts b/src/execution/agentEvents.ts index 4f9890284..372fbfd5f 100644 --- a/src/execution/agentEvents.ts +++ b/src/execution/agentEvents.ts @@ -9,6 +9,7 @@ import type { ExecutionFinishReason } from './AgentExecutor'; import type { SubagentInfo } from './hooks'; +import type { CompactedProviderErrorCategory } from './errors'; /** * Version of the {@link AgentEvent} schema, carried on every event as `v`. @@ -149,6 +150,37 @@ export interface AgentErrorEvent extends AgentEventBase<'error'> { error: AgentEventError; } +/** + * A model call failed and a `withRetry()` wrapper (e.g. `createAgent({ retry })`) + * retries it after `delayMs` (LOU-V7.2). Emitted inside the step, before + * the retried call. + */ +export interface ProviderRetryEvent extends AgentEventBase<'provider.retry'> { + /** The attempt that failed (1 = the first call). */ + attempt: number; + /** Retries the wrapper allows after the first attempt. */ + maxRetries: number; + /** How long the wrapper waits before the next attempt. */ + delayMs: number; + /** The failure, compacted like `compactProviderError()`; `category` (e.g. `'rate-limit'`) is absent when it is `'unknown'`. */ + error: { message: string; category?: CompactedProviderErrorCategory }; + /** Name of the provider that failed. */ + provider: string; +} + +/** + * A model call failed (after its retries) and the next provider of a + * `withFallback()` wrapper (e.g. `createAgent({ fallbackModels })`) takes + * over the call (LOU-V7.2). Emitted inside the step. + */ +export interface ProviderFallbackEvent extends AgentEventBase<'provider.fallback'> { + /** Name of the provider that failed. */ + from: string; + /** Name of the provider that takes over. */ + to: string; + error: { message: string }; +} + /** * Last event of every run, emitted exactly once - also for aborted, failed * and awaiting-approval runs. @@ -184,6 +216,8 @@ export type AgentEvent = | ApprovalRequestedEvent | StepDoneEvent | AgentErrorEvent + | ProviderRetryEvent + | ProviderFallbackEvent | RunDoneEvent; /** The `type` of an {@link AgentEvent}. */ @@ -215,6 +249,8 @@ const EVENT_TYPES: ReadonlySet = new Set([ 'approval.requested', 'step.done', 'error', + 'provider.retry', + 'provider.fallback', 'run.done', ]); diff --git a/src/execution/agentRun.ts b/src/execution/agentRun.ts index 2581298c2..61b7289ee 100644 --- a/src/execution/agentRun.ts +++ b/src/execution/agentRun.ts @@ -29,6 +29,8 @@ import { import { parseToolArguments } from './toolCallExecution'; import { canStream, generateViaStream } from './streamStep'; import { measureUsage } from './runUsage'; +import { compactProviderError } from './errors'; +import { withProviderEvents, type ProviderEventListener } from '../providers/providerEvents'; import type { Usage } from '../models/usage'; /** @@ -332,7 +334,8 @@ class AgentRunImpl implements AgentRun { }); }, generate: async (provider, request) => { - const generated = await this.generateStep(provider, request, subagent); + const call = withProviderEvents(request, this.providerEvents(subagent)); + const generated = await this.generateStep(provider, call, subagent); const measured = measureUsage(request.model ?? provider.name, request.messages, generated); stepResult = { finishReason: generated.finishReason, ...measured, usage: measured.usage }; return generated; @@ -341,6 +344,19 @@ class AgentRunImpl implements AgentRun { }; } + /** Reports what `withRetry()` / `withFallback()` do with this run's model calls (LOU-V7.2). */ + private providerEvents(subagent: SubagentInfo | undefined): ProviderEventListener { + return { + retry: ({ attempt, maxRetries, delayMs, error, provider }) => { + const { error: message, category } = compactProviderError(error); + const eventError = { message, ...(category !== 'unknown' && { category }) }; + this.emit({ type: 'provider.retry', attempt, maxRetries, delayMs, error: eventError, provider }, subagent); + }, + fallback: ({ from, to, error }) => + this.emit({ type: 'provider.fallback', from, to, error: { message: compactProviderError(error).error } }, subagent), + }; + } + /** Streams the step when the provider can; otherwise one `generate()` and a single `text.delta`. */ private async generateStep( provider: LLMProvider, diff --git a/src/execution/index.ts b/src/execution/index.ts index cc9f7630f..dd3cdeb23 100644 --- a/src/execution/index.ts +++ b/src/execution/index.ts @@ -45,6 +45,8 @@ export type { ApprovalRequestedEvent, StepDoneEvent, AgentErrorEvent, + ProviderRetryEvent, + ProviderFallbackEvent, RunDoneEvent, } from './agentEvents'; export type { AgentRun } from './agentRun'; diff --git a/src/providers/aiSdkProvider.test.ts b/src/providers/aiSdkProvider.test.ts index c4e210ed9..7184e7c78 100644 --- a/src/providers/aiSdkProvider.test.ts +++ b/src/providers/aiSdkProvider.test.ts @@ -57,6 +57,19 @@ describe('AiSdkProvider', () => { expect(settings.messages).toEqual([{ role: 'user', content: 'hi' }]); }); + it('LOU-V7.2: passes config.maxRetries (default 2) to the ai SDK calls', async () => { + generateTextMock.mockResolvedValue(textResult('stop')); + streamTextMock.mockResolvedValue({ textStream: (async function* () {})(), text: '', usage, finishReason: 'stop', toolCalls: [] }); + const call = { model: '', messages: [] }; + + await new OpenAIProvider({ apiKey: 'k' }).generate(call); + await new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }).generate(call); + await new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }).stream(call); + + expect(generateTextMock.mock.calls.map(([settings]) => settings.maxRetries)).toEqual([2, 0]); + expect(streamTextMock.mock.calls[0][0].maxRetries).toBe(0); + }); + it('LOU-V5: reports usage as-is, reads cache/reasoning tokens from provider metadata, and no usage when counts are NaN', async () => { const provider = new OpenAIProvider({ name: 'openai', apiKey: 'k' }); diff --git a/src/providers/aiSdkProvider.ts b/src/providers/aiSdkProvider.ts index 2d05d5da1..ba0342011 100644 --- a/src/providers/aiSdkProvider.ts +++ b/src/providers/aiSdkProvider.ts @@ -306,6 +306,9 @@ export abstract class AiSdkProvider impleme seed: options.seed, tools: convertTools(options.tools), maxSteps: 1, // Single step - tool execution happens in AgentExecutor + // The 'ai' SDK's own retries (its default is 2). createAgent() resolves + // providers with 0 and retries in its withRetry() wrapper instead. + maxRetries: this.config.maxRetries ?? 2, abortSignal: options.signal, }; } diff --git a/src/providers/providerEvents.ts b/src/providers/providerEvents.ts new file mode 100644 index 000000000..1e8b9198a --- /dev/null +++ b/src/providers/providerEvents.ts @@ -0,0 +1,46 @@ +/** + * Internal (LOU-V7.2): how a streaming run hears about the retries and + * fallbacks of the model calls it makes. The run attaches a listener to each + * request (`withProviderEvents()`); `withRetry()` and `withFallback()` report + * to the listener of the call they handle, besides their own `onRetry` / + * `onFallback`. Keyed by a symbol, so it never reaches a provider's wire + * format, and not re-exported from the providers barrel. + */ + +import type { GenerateOptions } from './llm'; + +/** A retry of one call: `RetryInfo` plus the wrapper's limit and the provider's name. */ +export interface ProviderRetryReport { + attempt: number; + maxRetries: number; + delayMs: number; + error: unknown; + provider: string; +} + +/** The next provider of a `withFallback()` takes over the call. */ +export interface ProviderFallbackReport { + from: string; + to: string; + error: unknown; +} + +export interface ProviderEventListener { + retry(report: ProviderRetryReport): void; + fallback(report: ProviderFallbackReport): void; +} + +const PROVIDER_EVENTS: unique symbol = Symbol('loushy.providerEvents'); + +type ListenedCall = GenerateOptions & { [PROVIDER_EVENTS]?: ProviderEventListener }; + +/** `call` with `listener` attached; the wrappers keep it when they copy the call. */ +export function withProviderEvents(call: GenerateOptions, listener: ProviderEventListener): GenerateOptions { + const listened: ListenedCall = { ...call, [PROVIDER_EVENTS]: listener }; + return listened; +} + +/** The listener attached to `call`, if any. */ +export function providerEventsOf(call: GenerateOptions): ProviderEventListener | undefined { + return (call as ListenedCall)[PROVIDER_EVENTS]; +} diff --git a/src/providers/providerSpec.ts b/src/providers/providerSpec.ts index 53269b9e4..c9e052b8b 100644 --- a/src/providers/providerSpec.ts +++ b/src/providers/providerSpec.ts @@ -178,9 +178,10 @@ function createProvider(caller: string, providerName: string, entry: ProviderEnt /** * Resolve a "/" spec into a configured LLMProvider, with - * errors reported in `caller`'s name. See `resolveProvider()`. + * errors reported in `caller`'s name. See `resolveProvider()`. `extra` is + * merged into the provider config (createAgent() sets `maxRetries: 0`). */ -export function resolveProviderSpec(spec: string, caller: string): LLMProvider { +export function resolveProviderSpec(spec: string, caller: string, extra: LLMProviderConfig = {}): LLMProvider { const separatorIndex = spec.indexOf('/'); if (separatorIndex <= 0 || separatorIndex === spec.length - 1) { throw new Error( @@ -199,6 +200,7 @@ export function resolveProviderSpec(spec: string, caller: string): LLMProvider { if (!envValue && entry.envRequired) throw missingKeyError(caller, entry.envKey); return createProvider(caller, providerName, entry, { + ...extra, defaultModel: model, [entry.configField]: envValue, }); diff --git a/src/providers/resilience.ts b/src/providers/resilience.ts index 2ddd43c88..b610d63b5 100644 --- a/src/providers/resilience.ts +++ b/src/providers/resilience.ts @@ -14,6 +14,7 @@ import type { GenerateOptions, LLMProvider, LLMProviderConfig } from './llm'; import { abortableDelay } from './abortableDelay'; +import { providerEventsOf } from './providerEvents'; import { CompactedLLMProviderError, ConfigurationError, @@ -86,6 +87,7 @@ function combineSignals(...signals: Array): AbortSignal } async function callWithRetry( + provider: LLMProvider, call: GenerateOptions, attempt: (call: GenerateOptions) => Promise, options: WithRetryOptions @@ -103,6 +105,7 @@ async function callWithRetry( } const delayMs = backoffDelay(error, n, options.backoff); onRetry?.({ attempt: n, error, delayMs }); + providerEventsOf(call)?.retry({ attempt: n, maxRetries, delayMs, error, provider: provider.name }); await abortableDelay(delayMs, signal); } } @@ -127,8 +130,8 @@ export function withRetry(provider: LLMProvider, options: WithRetryOptions = {}) get defaultModel() { return provider.defaultModel; }, - generate: (call) => callWithRetry(call, (attempt) => provider.generate(attempt), options), - stream: (call) => callWithRetry(call, (attempt) => provider.stream(attempt), options), + generate: (call) => callWithRetry(provider, call, (attempt) => provider.generate(attempt), options), + stream: (call) => callWithRetry(provider, call, (attempt) => provider.stream(attempt), options), supportsTools: (model) => provider.supportsTools(model), supportsStreaming: (model) => provider.supportsStreaming(model), getModels: () => provider.getModels(), @@ -137,8 +140,9 @@ export function withRetry(provider: LLMProvider, options: WithRetryOptions = {}) /** * `withRetry()` configured from a provider config's `maxRetries` and - * `timeout` (ms per attempt), the two `LLMProviderConfig` fields the - * built-in providers do not read themselves. + * `timeout` (ms per attempt). The built-in providers also pass + * `maxRetries` to the `ai` SDK's own retries (LOU-V7.2), so build the + * wrapped provider with `maxRetries: 0` to retry in one place. */ export function resilientProvider( provider: LLMProvider, @@ -194,7 +198,9 @@ export function withFallback(providers: LLMProvider[], options: WithFallbackOpti if (call.signal?.aborted || !fallbackOn(lastError)) { throw lastError; } - onFallback?.({ from: active.name, to: provider.name, error: lastError }); + const info: FallbackInfo = { from: active.name, to: provider.name, error: lastError }; + onFallback?.(info); + providerEventsOf(call)?.fallback(info); } active = provider; try {