From ea86a80b1db38867a516969b14f6ed1620e389d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 11:33:35 +0000 Subject: [PATCH] Add provider retry policy and model fallback wrappers (LOU-V7.1) New src/providers/resilience.ts: - withRetry(provider, options) retries generate()/stream() on transient failures with exponential backoff, classifying errors with the existing compactProviderError() (rate limit, timeout, network, 5xx retryable; auth, invalid request, context length not), honoring retryAfterMs, never retrying a cancellation, and stopping when the signal aborts. - withFallback(providers, options) tries providers in order, reports onFallback, rethrows the last error; name/defaultModel follow the active provider. - resilientProvider(provider, config) applies LLMProviderConfig.maxRetries and timeout. The mock provider's abortable delay moves to a shared abortableDelay module so the retry backoff reuses it. execution/retry.ts is left untouched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016bY29ibQgv3MDdjAbUT4YR --- docs/api-overview.md | 1 + docs/configuration.md | 42 +++++ llms-full.txt | 43 ++++++ src/providers/abortableDelay.ts | 19 +++ src/providers/index.ts | 1 + src/providers/mock.ts | 19 +-- src/providers/resilience.test.ts | 258 +++++++++++++++++++++++++++++++ src/providers/resilience.ts | 222 ++++++++++++++++++++++++++ 8 files changed, 587 insertions(+), 18 deletions(-) create mode 100644 src/providers/abortableDelay.ts create mode 100644 src/providers/resilience.test.ts create mode 100644 src/providers/resilience.ts diff --git a/docs/api-overview.md b/docs/api-overview.md index a7b4272b2..766725f68 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -153,6 +153,7 @@ Guarantees, whatever the limit: | `LLMProviderRegistry` | Registry of provider factories (`create`, `register`, `has`). | | `OpenAIProvider`, `AnthropicProvider`, `OllamaProvider`, `OpenRouterProvider` | Provider classes. | | `createMockProvider()`, `MockLLMProvider` | Deterministic mock provider for tests and demos. | +| `withRetry(provider, opts?)`, `withFallback(providers, opts?)` | Retry transient provider failures with backoff; fall back to the next provider. See [Configuration](configuration.md#provider-retries-and-fallback). | ## Testing diff --git a/docs/configuration.md b/docs/configuration.md index 51e212d7e..f82f45142 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -134,6 +134,48 @@ Real providers are resolved by `resolveProvider('/')` The `mock` provider needs no credentials and returns canned responses; it is what the examples and the Quick Start use by default. +## Provider retries and fallback + +`withRetry(provider, options)` and `withFallback(providers, options)` wrap any +`LLMProvider` and return another one, so they compose and can be passed +anywhere a provider is accepted: + +```ts +import { createAgent, resolveProvider, withFallback, withRetry } from '@loushy/build-ai-agent'; + +const provider = withFallback( + [ + withRetry(resolveProvider('openai/gpt-4o-mini'), { + maxRetries: 3, + backoff: { initialMs: 500, maxMs: 10_000 }, + onRetry: ({ attempt, delayMs }) => console.warn(`retry ${attempt} in ${delayMs}ms`), + }), + withRetry(resolveProvider('anthropic/claude-3-5-haiku-latest')), + ], + { onFallback: ({ from, to }) => console.warn(`falling back from ${from} to ${to}`) } +); + +const agent = createAgent({ prompt: 'You are helpful.', provider }); +``` + +- `withRetry` retries `generate()` and `stream()` (default `maxRetries: 2`) + on rate limits, timeouts, network errors and 5xx responses, using the same + classification as `compactProviderError()`. Auth failures, invalid requests + and context-length errors are not retried; neither is a cancellation. A + provider's `retryAfterMs` hint (a `Retry-After` header) replaces the backoff + delay. Pass `retryOn(error, attempt)` to change the rule, `timeoutMs` for a + per-attempt time limit, and `signal` to stop retrying. +- A `stream()` call is retried only when it rejects. An error inside a stream + that was already returned is not retried. +- `withFallback` tries each provider in order and rethrows the last error + when all fail. By default it falls back on any error except a cancellation + (`fallbackOn` changes that). Each fallback runs on its own `defaultModel`. + `name` and `defaultModel` report the provider that served the latest call. +- `resilientProvider(provider, { maxRetries, timeout })` applies the + `LLMProviderConfig` fields of the same names. +- The built-in providers also run the `ai` SDK's own internal retries (2 by + default) inside each attempt. + ## `createAgent()` options | Option | Description | diff --git a/llms-full.txt b/llms-full.txt index 88d5d9d41..af4e4f1db 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1317,6 +1317,7 @@ Guarantees, whatever the limit: | `LLMProviderRegistry` | Registry of provider factories (`create`, `register`, `has`). | | `OpenAIProvider`, `AnthropicProvider`, `OllamaProvider`, `OpenRouterProvider` | Provider classes. | | `createMockProvider()`, `MockLLMProvider` | Deterministic mock provider for tests and demos. | +| `withRetry(provider, opts?)`, `withFallback(providers, opts?)` | Retry transient provider failures with backoff; fall back to the next provider. See [Configuration](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/configuration.md#provider-retries-and-fallback). | ## Testing @@ -2289,6 +2290,48 @@ Real providers are resolved by `resolveProvider('/')` The `mock` provider needs no credentials and returns canned responses; it is what the examples and the Quick Start use by default. +## Provider retries and fallback + +`withRetry(provider, options)` and `withFallback(providers, options)` wrap any +`LLMProvider` and return another one, so they compose and can be passed +anywhere a provider is accepted: + +```ts +import { createAgent, resolveProvider, withFallback, withRetry } from '@loushy/build-ai-agent'; + +const provider = withFallback( + [ + withRetry(resolveProvider('openai/gpt-4o-mini'), { + maxRetries: 3, + backoff: { initialMs: 500, maxMs: 10_000 }, + onRetry: ({ attempt, delayMs }) => console.warn(`retry ${attempt} in ${delayMs}ms`), + }), + withRetry(resolveProvider('anthropic/claude-3-5-haiku-latest')), + ], + { onFallback: ({ from, to }) => console.warn(`falling back from ${from} to ${to}`) } +); + +const agent = createAgent({ prompt: 'You are helpful.', provider }); +``` + +- `withRetry` retries `generate()` and `stream()` (default `maxRetries: 2`) + on rate limits, timeouts, network errors and 5xx responses, using the same + classification as `compactProviderError()`. Auth failures, invalid requests + and context-length errors are not retried; neither is a cancellation. A + provider's `retryAfterMs` hint (a `Retry-After` header) replaces the backoff + delay. Pass `retryOn(error, attempt)` to change the rule, `timeoutMs` for a + per-attempt time limit, and `signal` to stop retrying. +- A `stream()` call is retried only when it rejects. An error inside a stream + that was already returned is not retried. +- `withFallback` tries each provider in order and rethrows the last error + when all fail. By default it falls back on any error except a cancellation + (`fallbackOn` changes that). Each fallback runs on its own `defaultModel`. + `name` and `defaultModel` report the provider that served the latest call. +- `resilientProvider(provider, { maxRetries, timeout })` applies the + `LLMProviderConfig` fields of the same names. +- The built-in providers also run the `ai` SDK's own internal retries (2 by + default) inside each attempt. + ## `createAgent()` options | Option | Description | diff --git a/src/providers/abortableDelay.ts b/src/providers/abortableDelay.ts new file mode 100644 index 000000000..15b638873 --- /dev/null +++ b/src/providers/abortableDelay.ts @@ -0,0 +1,19 @@ +/** + * Waits `ms`, rejecting with the signal's reason as soon as it is aborted + * (LOU-V1), or at once when it already is. Shared by the mock provider's + * simulated latency and `withRetry()`'s backoff (LOU-V7.1). + */ +export function abortableDelay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve, reject) => { + signal?.throwIfAborted(); // rejects this promise + const onAbort = () => { + clearTimeout(timer); + reject(signal?.reason); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} diff --git a/src/providers/index.ts b/src/providers/index.ts index 362978b4e..636dd0c3b 100644 --- a/src/providers/index.ts +++ b/src/providers/index.ts @@ -10,6 +10,7 @@ export * from './OllamaProvider'; export * from './OpenRouterProvider'; export * from './AnthropicProvider'; export * from './resolveProvider'; +export * from './resilience'; export { MissingPeerDependencyError } from './optionalPeer'; // Auto-register built-in providers. The factories only construct provider diff --git a/src/providers/mock.ts b/src/providers/mock.ts index 0adad3c3d..bf61e4204 100644 --- a/src/providers/mock.ts +++ b/src/providers/mock.ts @@ -13,6 +13,7 @@ import { StreamChunk, ToolCall, } from './llm'; +import { abortableDelay } from './abortableDelay'; /** * Mock response configuration @@ -24,24 +25,6 @@ export interface MockProviderConfig extends LLMProviderConfig { errorMessage?: string; } -/** - * Waits `ms`, rejecting with the signal's reason as soon as it is aborted - * (LOU-V1), so a simulated slow call can be cancelled mid-delay. - */ -function abortableDelay(ms: number, signal?: AbortSignal): Promise { - return new Promise((resolve, reject) => { - const onAbort = () => { - clearTimeout(timer); - reject(signal?.reason); - }; - const timer = setTimeout(() => { - signal?.removeEventListener('abort', onAbort); - resolve(); - }, ms); - signal?.addEventListener('abort', onAbort, { once: true }); - }); -} - /** * Mock LLM Provider */ diff --git a/src/providers/resilience.test.ts b/src/providers/resilience.test.ts new file mode 100644 index 000000000..33213c344 --- /dev/null +++ b/src/providers/resilience.test.ts @@ -0,0 +1,258 @@ +/** + * Provider resilience wrappers (LOU-V7.1): withRetry, withFallback, + * resilientProvider. + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { APICallError } from 'ai'; +import type { GenerateOptions, LLMProvider } from './llm'; +import { MockLLMProvider } from './mock'; +import { isRetryableProviderError, resilientProvider, withFallback, withRetry } from './resilience'; +import { CompactedLLMProviderError, compactProviderError } from '../execution/errors'; + +function apiError(statusCode: number, headers?: Record): APICallError { + return new APICallError({ + message: `HTTP ${statusCode}`, + url: 'https://api.example.com/v1/chat', + requestBodyValues: {}, + statusCode, + responseHeaders: headers, + isRetryable: statusCode >= 500 || statusCode === 429, + }); +} + +/** A provider that throws `failures` in order (one per call, generate or stream), then answers `reply`. */ +function flaky(failures: unknown[], name = 'flaky', reply = `${name} ok`) { + const inner = new MockLLMProvider({ responses: [reply], defaultModel: `${name}-model` }); + const calls: GenerateOptions[] = []; + const next = (call: GenerateOptions) => { + calls.push(call); + if (failures.length > 0) { + throw failures.shift(); + } + }; + const provider: LLMProvider = { + name, + defaultModel: inner.defaultModel, + generate: async (call) => (next(call), inner.generate(call)), + stream: async (call) => (next(call), inner.stream(call)), + supportsTools: () => true, + supportsStreaming: () => true, + getModels: async () => [`${name}-model`], + }; + return { provider, calls }; +} + +const fast = { backoff: { initialMs: 1, jitter: false } }; +const request: GenerateOptions = { messages: [{ role: 'user', content: 'hi' }] }; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('withRetry', () => { + it('retries retryable failures until the provider succeeds', async () => { + const { provider, calls } = flaky([apiError(503), apiError(429)]); + const onRetry = vi.fn(); + + const result = await withRetry(provider, { ...fast, onRetry }).generate(request); + + expect(result.text).toBe('flaky ok'); + expect(calls).toHaveLength(3); + expect(onRetry.mock.calls.map(([info]) => [info.attempt, info.delayMs])).toEqual([ + [1, 1], + [2, 2], + ]); + }); + + it('rethrows the last error once maxRetries is exhausted', async () => { + const last = apiError(502); + const { provider, calls } = flaky([apiError(503), last]); + + await expect(withRetry(provider, { ...fast, maxRetries: 1 }).generate(request)).rejects.toBe(last); + expect(calls).toHaveLength(2); + }); + + it('does not retry a non-retryable error', async () => { + const auth = apiError(401); + const { provider, calls } = flaky([auth]); + + await expect(withRetry(provider, fast).generate(request)).rejects.toBe(auth); + expect(calls).toHaveLength(1); + }); + + it('classifies errors like the executor does, including compacted ones', () => { + expect(isRetryableProviderError(apiError(500))).toBe(true); + expect(isRetryableProviderError(new Error('fetch failed'))).toBe(true); + expect(isRetryableProviderError(apiError(400))).toBe(false); + expect(isRetryableProviderError(new Error('boom'))).toBe(false); + const contextLength = new CompactedLLMProviderError( + compactProviderError(new Error('maximum context length exceeded')) + ); + expect(isRetryableProviderError(contextLength)).toBe(false); + }); + + it('honors retryOn', async () => { + const { provider, calls } = flaky([new Error('custom transient')]); + const retryOn = vi.fn(() => true); + + await expect(withRetry(provider, { ...fast, retryOn }).generate(request)).resolves.toMatchObject({ text: 'flaky ok' }); + expect(retryOn).toHaveBeenCalledWith(expect.any(Error), 1); + expect(calls).toHaveLength(2); + }); + + it("waits the provider's retryAfterMs instead of the backoff", async () => { + vi.useFakeTimers(); + const { provider, calls } = flaky([apiError(429, { 'retry-after': '5' })]); + const onRetry = vi.fn(); + + const pending = withRetry(provider, { ...fast, onRetry }).generate(request); + await vi.advanceTimersByTimeAsync(4_999); + expect(calls).toHaveLength(1); + expect(onRetry).toHaveBeenCalledWith(expect.objectContaining({ attempt: 1, delayMs: 5_000 })); + + await vi.advanceTimersByTimeAsync(1); + await expect(pending).resolves.toMatchObject({ text: 'flaky ok' }); + expect(calls).toHaveLength(2); + }); + + it('stops retrying when the signal aborts during the backoff wait', async () => { + const { provider, calls } = flaky([apiError(503), apiError(503)]); + const controller = new AbortController(); + const wrapped = withRetry(provider, { + backoff: { initialMs: 60_000 }, + onRetry: () => controller.abort(), + }); + + await expect(wrapped.generate({ ...request, signal: controller.signal })).rejects.toMatchObject({ + name: 'AbortError', + }); + expect(calls).toHaveLength(1); + }); + + it('never retries an abort and never calls the provider with an aborted signal', async () => { + const aborted = new DOMException('stopped', 'AbortError'); + const { provider, calls } = flaky([aborted]); + const retryOn = vi.fn(() => true); + + await expect(withRetry(provider, { ...fast, retryOn }).generate(request)).rejects.toBe(aborted); + expect(retryOn).not.toHaveBeenCalled(); + + const controller = new AbortController(); + controller.abort(); + await expect(withRetry(provider, { ...fast, signal: controller.signal }).generate(request)).rejects.toMatchObject({ + name: 'AbortError', + }); + expect(calls).toHaveLength(1); + }); + + it('retries a stream() call that rejects', async () => { + const { provider, calls } = flaky([apiError(503)]); + + const stream = await withRetry(provider, fast).stream(request); + + await expect(stream.text).resolves.toBe('flaky ok'); + expect(calls).toHaveLength(2); + }); + + it('delegates name, defaultModel and capabilities to the wrapped provider', async () => { + const { provider } = flaky([]); + const wrapped = withRetry(provider); + + expect(wrapped.name).toBe('flaky'); + expect(wrapped.defaultModel).toBe('flaky-model'); + expect(wrapped.supportsTools('x')).toBe(true); + expect(wrapped.supportsStreaming('x')).toBe(true); + await expect(wrapped.getModels()).resolves.toEqual(['flaky-model']); + }); +}); + +describe('resilientProvider', () => { + it("applies the config's maxRetries and per-attempt timeout", async () => { + const slow = new MockLLMProvider({ delay: 1_000 }); + const generate = vi.spyOn(slow, 'generate'); + + await expect(resilientProvider(slow, { maxRetries: 1, timeout: 5 }).generate(request)).rejects.toMatchObject({ + name: 'TimeoutError', + }); + expect(generate).toHaveBeenCalledTimes(2); + }); +}); + +describe('withFallback', () => { + it('switches to the next provider on failure and reports it', async () => { + const primaryError = apiError(401); + const primary = flaky([primaryError], 'primary'); + const backup = flaky([], 'backup'); + const onFallback = vi.fn(); + const provider = withFallback([primary.provider, backup.provider], { onFallback }); + + expect(provider.name).toBe('primary'); + const result = await provider.generate({ ...request, model: provider.defaultModel }); + + expect(result.text).toBe('backup ok'); + expect(onFallback).toHaveBeenCalledWith({ from: 'primary', to: 'backup', error: primaryError }); + expect(provider.name).toBe('backup'); + expect(provider.defaultModel).toBe('backup-model'); + // each provider ran on its own default model + expect(primary.calls[0].model).toBeUndefined(); + expect(backup.calls[0].model).toBeUndefined(); + }); + + it('starts every call with the first provider and forwards an explicit model to it only', async () => { + const primary = flaky([apiError(500)], 'primary'); + const backup = flaky([], 'backup'); + const provider = withFallback([primary.provider, backup.provider]); + await provider.generate(request); + + const result = await provider.generate({ ...request, model: 'gpt-x' }); + + expect(result.text).toBe('primary ok'); + expect(primary.calls[1].model).toBe('gpt-x'); + expect(provider.name).toBe('primary'); + }); + + it('rethrows the last error when every provider fails', async () => { + const last = apiError(503); + const provider = withFallback([flaky([apiError(500)], 'a').provider, flaky([last], 'b').provider]); + + await expect(provider.generate(request)).rejects.toBe(last); + }); + + it('does not fall back when fallbackOn says no or the call was aborted', async () => { + const first = flaky([apiError(500), apiError(500)], 'a'); + const second = flaky([], 'b'); + const noFallback = withFallback([first.provider, second.provider], { fallbackOn: () => false }); + await expect(noFallback.generate(request)).rejects.toMatchObject({ statusCode: 500 }); + + const controller = new AbortController(); + const aborting = withFallback([ + { ...first.provider, generate: async () => (controller.abort(), Promise.reject(apiError(500))) }, + second.provider, + ]); + await expect(aborting.generate({ ...request, signal: controller.signal })).rejects.toMatchObject({ + statusCode: 500, + }); + expect(second.calls).toHaveLength(0); + }); + + it('composes with withRetry and covers stream()', async () => { + const a = flaky([apiError(503), apiError(503), apiError(503)], 'a'); + const b = flaky([apiError(429)], 'b'); + const onFallback = vi.fn(); + const provider = withFallback([withRetry(a.provider, fast), withRetry(b.provider, fast)], { onFallback }); + + const stream = await provider.stream(request); + + await expect(stream.text).resolves.toBe('b ok'); + expect(a.calls).toHaveLength(3); + expect(b.calls).toHaveLength(2); + expect(onFallback).toHaveBeenCalledWith(expect.objectContaining({ from: 'a', to: 'b' })); + expect(provider.supportsTools('x')).toBe(true); + expect(provider.supportsStreaming('x')).toBe(true); + await expect(provider.getModels()).resolves.toEqual(['a-model']); + }); + + it('needs at least one provider', () => { + expect(() => withFallback([])).toThrow('at least one provider'); + }); +}); diff --git a/src/providers/resilience.ts b/src/providers/resilience.ts new file mode 100644 index 000000000..2ddd43c88 --- /dev/null +++ b/src/providers/resilience.ts @@ -0,0 +1,222 @@ +/** + * Provider resilience (LOU-V7.1): retry and model-fallback wrappers that + * turn any `LLMProvider` into another `LLMProvider`, so they compose with + * each other and work anywhere a provider is accepted. + * + * Retryability comes from the SDK's provider-error classification + * (`compactProviderError()` in execution/errors.ts), not a second rule set. + * The older generic `retry()` helper (execution/retry.ts) is deliberately + * not reused: its backoff sleep ignores the abort signal, its delay hint + * reads only the legacy `RateLimitError`, and its default classification + * retries any `LLMProviderError` without a status code (auth failures + * included). + */ + +import type { GenerateOptions, LLMProvider, LLMProviderConfig } from './llm'; +import { abortableDelay } from './abortableDelay'; +import { + CompactedLLMProviderError, + ConfigurationError, + compactProviderError, + isAbortError, + type CompactedProviderError, +} from '../execution/errors'; + +/** Exponential backoff between retries. */ +export interface BackoffOptions { + /** Delay before the first retry, in ms. Default 500. */ + initialMs?: number; + /** Upper bound for the computed delay, in ms. Default 30_000. */ + maxMs?: number; + /** Multiplier applied per retry. Default 2. */ + factor?: number; + /** Randomize each delay to 50-100% of its value. Default true. */ + jitter?: boolean; +} + +/** Passed to `WithRetryOptions.onRetry` before each backoff wait. */ +export interface RetryInfo { + /** The attempt that just failed (1 = the first call). */ + attempt: number; + error: unknown; + /** How long the wrapper waits before the next attempt. */ + delayMs: number; +} + +export interface WithRetryOptions { + /** Retries after the first attempt. Default 2 (so at most 3 calls). */ + maxRetries?: number; + backoff?: BackoffOptions; + /** Whether a failure is retried. Default `isRetryableProviderError`. */ + retryOn?: (error: unknown, attempt: number) => boolean; + onRetry?: (info: RetryInfo) => void; + /** Stops retrying (and aborts the in-flight call) once aborted; the call's own `signal` does the same. */ + signal?: AbortSignal; + /** Per-attempt time limit in ms (for `stream()` it also bounds reading the stream); a timed-out attempt is retryable. */ + timeoutMs?: number; +} + +/** The compacted classification of a provider failure. */ +function classify(error: unknown): CompactedProviderError { + return error instanceof CompactedLLMProviderError ? error.compacted : compactProviderError(error); +} + +/** + * Default retry rule: rate limits, timeouts, network failures and 5xx + * responses are retryable; auth failures, invalid requests and + * context-length errors are not, and neither is a cancellation. + */ +export function isRetryableProviderError(error: unknown): boolean { + return !isAbortError(error) && classify(error).retryable; +} + +function backoffDelay(error: unknown, attempt: number, backoff: BackoffOptions = {}): number { + const retryAfterMs = classify(error).retryAfterMs; + if (retryAfterMs !== undefined) { + return retryAfterMs; + } + const { initialMs = 500, maxMs = 30_000, factor = 2, jitter = true } = backoff; + const delay = Math.min(initialMs * factor ** (attempt - 1), maxMs); + return Math.round(jitter ? delay * (0.5 + Math.random() / 2) : delay); +} + +function combineSignals(...signals: Array): AbortSignal | undefined { + const present = signals.filter((signal): signal is AbortSignal => signal !== undefined); + return present.length > 1 ? AbortSignal.any(present) : present[0]; +} + +async function callWithRetry( + call: GenerateOptions, + attempt: (call: GenerateOptions) => Promise, + options: WithRetryOptions +): Promise { + const { maxRetries = 2, retryOn = isRetryableProviderError, onRetry, timeoutMs } = options; + const signal = combineSignals(options.signal, call.signal); + for (let n = 1; ; n++) { + signal?.throwIfAborted(); + const timeout = timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs); + try { + return await attempt({ ...call, signal: combineSignals(signal, timeout) }); + } catch (error) { + if (n > maxRetries || signal?.aborted || isAbortError(error) || !retryOn(error, n)) { + throw error; + } + const delayMs = backoffDelay(error, n, options.backoff); + onRetry?.({ attempt: n, error, delayMs }); + await abortableDelay(delayMs, signal); + } + } +} + +/** + * Retry a provider's `generate()` and `stream()` on transient failures, with + * exponential backoff that honors a provider's `retryAfterMs` hint. A + * `stream()` call is retried only when it rejects; an error delivered inside + * an already-returned stream is not, since part of it may have been consumed. + * + * @example + * ```ts + * const provider = withRetry(resolveProvider('openai/gpt-4o-mini'), { maxRetries: 3 }); + * ``` + */ +export function withRetry(provider: LLMProvider, options: WithRetryOptions = {}): LLMProvider { + return { + get name() { + return provider.name; + }, + get defaultModel() { + return provider.defaultModel; + }, + generate: (call) => callWithRetry(call, (attempt) => provider.generate(attempt), options), + stream: (call) => callWithRetry(call, (attempt) => provider.stream(attempt), options), + supportsTools: (model) => provider.supportsTools(model), + supportsStreaming: (model) => provider.supportsStreaming(model), + getModels: () => provider.getModels(), + }; +} + +/** + * `withRetry()` configured from a provider config's `maxRetries` and + * `timeout` (ms per attempt), the two `LLMProviderConfig` fields the + * built-in providers do not read themselves. + */ +export function resilientProvider( + provider: LLMProvider, + config: Pick +): LLMProvider { + return withRetry(provider, { maxRetries: config.maxRetries, timeoutMs: config.timeout }); +} + +/** Passed to `WithFallbackOptions.onFallback` when the next provider takes over. */ +export interface FallbackInfo { + from: string; + to: string; + error: unknown; +} + +export interface WithFallbackOptions { + /** Whether a failure moves on to the next provider. Default: any failure except a cancellation. */ + fallbackOn?: (error: unknown) => boolean; + onFallback?: (info: FallbackInfo) => void; +} + +/** + * Try each provider in order until one succeeds; every call starts with the + * first. Rethrows the last error when all fail. `name` and `defaultModel` + * report the provider that served (or is serving) the latest call; + * `supportsTools`, `supportsStreaming` and `getModels` ask the first. + * + * A call's `model` goes only to the first provider, and only when it differs + * from this wrapper's `defaultModel`; otherwise, and always for fallbacks, + * each provider uses its own `defaultModel`. + * + * @example + * ```ts + * const provider = withFallback([ + * withRetry(resolveProvider('openai/gpt-4o-mini')), + * withRetry(resolveProvider('anthropic/claude-3-5-haiku-latest')), + * ]); + * ``` + */ +export function withFallback(providers: LLMProvider[], options: WithFallbackOptions = {}): LLMProvider { + const [first] = providers; + if (!first) { + throw new ConfigurationError('withFallback() needs at least one provider', 'providers'); + } + const { fallbackOn = (error: unknown) => !isAbortError(error), onFallback } = options; + let active = first; + + async function run(call: GenerateOptions, attempt: (provider: LLMProvider, call: GenerateOptions) => Promise) { + const model = call.model && call.model !== active.defaultModel ? call.model : undefined; + let lastError: unknown; + for (const [index, provider] of providers.entries()) { + if (index > 0) { + if (call.signal?.aborted || !fallbackOn(lastError)) { + throw lastError; + } + onFallback?.({ from: active.name, to: provider.name, error: lastError }); + } + active = provider; + try { + return await attempt(provider, { ...call, model: index === 0 ? model : undefined }); + } catch (error) { + lastError = error; + } + } + throw lastError; + } + + return { + get name() { + return active.name; + }, + get defaultModel() { + return active.defaultModel; + }, + generate: (call) => run(call, (provider, attempt) => provider.generate(attempt)), + stream: (call) => run(call, (provider, attempt) => provider.stream(attempt)), + supportsTools: (model) => first.supportsTools(model), + supportsStreaming: (model) => first.supportsStreaming(model), + getModels: () => first.getModels(), + }; +}