From a23b5d788d07b936653c25b6021f86255b72c78c Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 1 Oct 2026 10:31:20 +0300 Subject: [PATCH 1/2] LOU-D6: recordReplay provider (WIP) --- docs/testing.md | 118 ++++++++ llms-full.txt | 118 ++++++++ src/testing/cassette.ts | 178 ++++++++++++ src/testing/fingerprint.ts | 257 +++++++++++++++++ src/testing/index.ts | 10 +- src/testing/recordReplay.test.ts | 476 +++++++++++++++++++++++++++++++ src/testing/recordReplay.ts | 457 +++++++++++++++++++++++++++++ 7 files changed, 1613 insertions(+), 1 deletion(-) create mode 100644 src/testing/cassette.ts create mode 100644 src/testing/fingerprint.ts create mode 100644 src/testing/recordReplay.test.ts create mode 100644 src/testing/recordReplay.ts diff --git a/docs/testing.md b/docs/testing.md index c6ccf9676..074b09d4b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -153,3 +153,121 @@ Other helpers: `model.reset()` rewinds the script and clears recorded calls, and `mockModel` also implements `stream()`: the scripted text is emitted as `text-delta` chunks (split on word boundaries), followed by any `tool-call` chunks and a final `finish` chunk, so streaming consumers can be tested too. + +## Record and replay + +`mockModel` scripts turns by hand. `recordReplay` is the complementary tool: a +VCR for real model behavior. Run your test once against the real provider and +every `generate()` / `stream()` exchange is written to a cassette file. In CI +the same test replays the cassette with no network, no API key and no installed +provider peer package. + +```ts +import { createAgent, resolveProvider } from '@loushy/build-ai-agent'; +import { recordReplay } from '@loushy/build-ai-agent/testing'; + +const provider = recordReplay(() => resolveProvider('openai/gpt-4o-mini'), { + cassette: './__cassettes__/refund-flow.json', + mode: process.env.LOUSHY_RECORD ? 'record' : 'replay', // or 'auto' +}); +const agent = createAgent({ provider, prompt: 'You handle refund requests.' }); +``` + +The first argument is the real provider, or a `() => provider` factory that is +only called when recording (so replay never constructs it). It may be +`undefined` if you only ever replay. + +### Workflow + +1. Record locally with a key: `LOUSHY_RECORD=1 OPENAI_API_KEY=... npx vitest run`. +2. Review and commit the cassette (it is stable, 2-space indented JSON, so + diffs are readable). +3. CI runs `npx vitest run` and replays it. Nothing else is needed. +4. When you change the prompt, tools or flow, replay fails with a + `CassetteMismatchError`; re-record with `LOUSHY_RECORD=1`. + +`mode: 'auto'` replays if the cassette file exists and records otherwise. + +### What replay checks + +By default call N is answered by entry N, but only if the request still matches +what was recorded: model, each message's role, content and tool calls, tool +names and parameter schemas, `temperature` and `maxTokens`. On a mismatch the +error shows the first difference and the re-record hint: + +```text +Call #2 does not match the recorded request in ./__cassettes__/refund-flow.json. +First difference at request.messages[1].content: + recorded: "Refund order 1234" + actual: "Refund order 9999" +If the change is intentional, re-record the cassette (run with LOUSHY_RECORD=1, or set mode: 'record'). +``` + +For parallel or unordered calls pass `match: 'request'`: each call finds the +first unused entry with an identical request, in any order. Running out of +entries is also a `CassetteMismatchError`. + +Volatile values do not break matching. Timestamps, dates (`2026-10-01`, +`October 1, 2026`), UUIDs and prefixed ids (`call_...`, `toolu_...`, +`chatcmpl_...`) are replaced by placeholders on both sides, and tool-call ids +are ignored (tool calls match by name and arguments). For anything else, pass +`normalize`: + +```ts +import { mockModel, recordReplay } from '@loushy/build-ai-agent/testing'; + +const provider = recordReplay(mockModel(['ok']), { + cassette: './__cassettes__/users.json', + normalize: (request) => ({ + ...request, + messages: request.messages.map((m) => ({ ...m, content: m.content.replace(/user-\d+/g, 'user-N') })), + }), +}); +``` + +### Errors, streaming and usage + +- A provider rejection is recorded and replayed as a rejection with the same + `name` and `message`. +- `stream()` records the chunks and replays them as an async iterable without + delays; pass `replayTiming: true` to reproduce the inter-chunk timing. While + recording, the real stream is read to the end before it is returned. +- Token usage is recorded and replayed, so cost and usage features behave the + same in replay. + +### When the cassette is written + +Recording writes the cassette atomically (temp file plus rename) after every +call, so a test that fails midway leaves a valid cassette with every call made +so far, never a half-written one. `await provider.save()` forces a write. There +is no process-exit hook. Each record session starts a fresh cassette and +replaces the old file. + +### Redaction (read this before committing) + +Cassettes are meant to be committed, so the wrapper only writes a fixed set of +request fields (never headers or provider option bags), never writes +`rawResponse`, and redacts strings that look like API keys (`sk-...`, +`sk-ant-...`, `Bearer ...` tokens and a few other common formats) with +`[REDACTED]`. That is a safety net, not a guarantee: prompts and model replies +are stored verbatim, so personal data, internal URLs and customer text end up in +the file. Pass `redact` to scrub them (it runs on every recorded string, in both +requests and responses, and replay applies it when matching): + +```ts +import { mockModel, recordReplay } from '@loushy/build-ai-agent/testing'; + +const provider = recordReplay(mockModel(['ok']), { + cassette: './__cassettes__/support.json', + redact: (text) => text.replace(/[\w.+-]+@[\w-]+\.[\w.]+/g, ''), +}); +``` + +### mockModel or cassettes? + +Use `mockModel` for unit tests of your own logic: you control every turn, can +inject errors and delays, and nothing depends on a model. Use cassettes when +the point is real model behavior (does this prompt and tool set actually lead +to the right tool call?) and hand-scripting turns would only encode your +assumptions. Cassettes need re-recording when prompts change; `mockModel` tests +only change when the behavior you assert on changes. diff --git a/llms-full.txt b/llms-full.txt index 44a60ab1e..e7014fa66 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2777,6 +2777,124 @@ Other helpers: `model.reset()` rewinds the script and clears recorded calls, and `text-delta` chunks (split on word boundaries), followed by any `tool-call` chunks and a final `finish` chunk, so streaming consumers can be tested too. +## Record and replay + +`mockModel` scripts turns by hand. `recordReplay` is the complementary tool: a +VCR for real model behavior. Run your test once against the real provider and +every `generate()` / `stream()` exchange is written to a cassette file. In CI +the same test replays the cassette with no network, no API key and no installed +provider peer package. + +```ts +import { createAgent, resolveProvider } from '@loushy/build-ai-agent'; +import { recordReplay } from '@loushy/build-ai-agent/testing'; + +const provider = recordReplay(() => resolveProvider('openai/gpt-4o-mini'), { + cassette: './__cassettes__/refund-flow.json', + mode: process.env.LOUSHY_RECORD ? 'record' : 'replay', // or 'auto' +}); +const agent = createAgent({ provider, prompt: 'You handle refund requests.' }); +``` + +The first argument is the real provider, or a `() => provider` factory that is +only called when recording (so replay never constructs it). It may be +`undefined` if you only ever replay. + +### Workflow + +1. Record locally with a key: `LOUSHY_RECORD=1 OPENAI_API_KEY=... npx vitest run`. +2. Review and commit the cassette (it is stable, 2-space indented JSON, so + diffs are readable). +3. CI runs `npx vitest run` and replays it. Nothing else is needed. +4. When you change the prompt, tools or flow, replay fails with a + `CassetteMismatchError`; re-record with `LOUSHY_RECORD=1`. + +`mode: 'auto'` replays if the cassette file exists and records otherwise. + +### What replay checks + +By default call N is answered by entry N, but only if the request still matches +what was recorded: model, each message's role, content and tool calls, tool +names and parameter schemas, `temperature` and `maxTokens`. On a mismatch the +error shows the first difference and the re-record hint: + +```text +Call #2 does not match the recorded request in ./__cassettes__/refund-flow.json. +First difference at request.messages[1].content: + recorded: "Refund order 1234" + actual: "Refund order 9999" +If the change is intentional, re-record the cassette (run with LOUSHY_RECORD=1, or set mode: 'record'). +``` + +For parallel or unordered calls pass `match: 'request'`: each call finds the +first unused entry with an identical request, in any order. Running out of +entries is also a `CassetteMismatchError`. + +Volatile values do not break matching. Timestamps, dates (`2026-10-01`, +`October 1, 2026`), UUIDs and prefixed ids (`call_...`, `toolu_...`, +`chatcmpl_...`) are replaced by placeholders on both sides, and tool-call ids +are ignored (tool calls match by name and arguments). For anything else, pass +`normalize`: + +```ts +import { mockModel, recordReplay } from '@loushy/build-ai-agent/testing'; + +const provider = recordReplay(mockModel(['ok']), { + cassette: './__cassettes__/users.json', + normalize: (request) => ({ + ...request, + messages: request.messages.map((m) => ({ ...m, content: m.content.replace(/user-\d+/g, 'user-N') })), + }), +}); +``` + +### Errors, streaming and usage + +- A provider rejection is recorded and replayed as a rejection with the same + `name` and `message`. +- `stream()` records the chunks and replays them as an async iterable without + delays; pass `replayTiming: true` to reproduce the inter-chunk timing. While + recording, the real stream is read to the end before it is returned. +- Token usage is recorded and replayed, so cost and usage features behave the + same in replay. + +### When the cassette is written + +Recording writes the cassette atomically (temp file plus rename) after every +call, so a test that fails midway leaves a valid cassette with every call made +so far, never a half-written one. `await provider.save()` forces a write. There +is no process-exit hook. Each record session starts a fresh cassette and +replaces the old file. + +### Redaction (read this before committing) + +Cassettes are meant to be committed, so the wrapper only writes a fixed set of +request fields (never headers or provider option bags), never writes +`rawResponse`, and redacts strings that look like API keys (`sk-...`, +`sk-ant-...`, `Bearer ...` tokens and a few other common formats) with +`[REDACTED]`. That is a safety net, not a guarantee: prompts and model replies +are stored verbatim, so personal data, internal URLs and customer text end up in +the file. Pass `redact` to scrub them (it runs on every recorded string, in both +requests and responses, and replay applies it when matching): + +```ts +import { mockModel, recordReplay } from '@loushy/build-ai-agent/testing'; + +const provider = recordReplay(mockModel(['ok']), { + cassette: './__cassettes__/support.json', + redact: (text) => text.replace(/[\w.+-]+@[\w-]+\.[\w.]+/g, ''), +}); +``` + +### mockModel or cassettes? + +Use `mockModel` for unit tests of your own logic: you control every turn, can +inject errors and delays, and nothing depends on a model. Use cassettes when +the point is real model behavior (does this prompt and tool set actually lead +to the right tool call?) and hand-scripting turns would only encode your +assumptions. Cassettes need re-recording when prompts change; `mockModel` tests +only change when the behavior you assert on changes. + # Workspace tools Source: docs/workspace-tools.md diff --git a/src/testing/cassette.ts b/src/testing/cassette.ts new file mode 100644 index 000000000..0eb0d2cd3 --- /dev/null +++ b/src/testing/cassette.ts @@ -0,0 +1,178 @@ +/** + * Cassette file format for `recordReplay`: a versioned, reviewable JSON file + * holding every recorded `generate()` / `stream()` exchange. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { randomBytes } from 'node:crypto'; +import { z } from 'zod'; + +/** The cassette format version this SDK reads and writes. */ +const CASSETTE_VERSION = 1; + +const usageSchema = z.object({ + promptTokens: z.number(), + completionTokens: z.number(), + totalTokens: z.number(), +}); + +const toolCallSchema = z.object({ + id: z.string(), + type: z.literal('function'), + function: z.object({ name: z.string(), arguments: z.string() }), +}); + +const errorSchema = z.object({ name: z.string(), message: z.string() }); + +const chunkSchema = z.object({ + type: z.enum(['text-delta', 'tool-call', 'tool-result', 'finish', 'error']), + textDelta: z.string().optional(), + toolCall: toolCallSchema.optional(), + toolResult: z.object({ toolCallId: z.string(), result: z.unknown() }).optional(), + finishReason: z.string().optional(), + usage: usageSchema.optional(), + error: errorSchema.optional(), +}); + +const requestSchema = z.object({ + model: z.string().nullable(), + messages: z.array( + z.object({ + role: z.enum(['system', 'user', 'assistant', 'tool']), + content: z.string(), + name: z.string().optional(), + toolName: z.string().optional(), + isError: z.boolean().optional(), + toolCalls: z.array(z.object({ name: z.string(), arguments: z.string() })).optional(), + }) + ), + tools: z.array(z.object({ name: z.string(), description: z.string(), parameters: z.unknown() })), + temperature: z.number().nullable(), + maxTokens: z.number().nullable(), +}); + +const responseSchema = z.object({ + text: z.string(), + finishReason: z.string(), + usage: usageSchema, + toolCalls: z.array(toolCallSchema).optional(), + /** Present for `stream()` entries: the chunks and the delay before each. */ + chunks: z.array(z.object({ delayMs: z.number(), chunk: chunkSchema })).optional(), +}); + +const entrySchema = z.object({ + kind: z.enum(['generate', 'stream']), + request: requestSchema, + response: responseSchema.optional(), + error: errorSchema.optional(), +}); + +const cassetteSchema = z.object({ + version: z.literal(CASSETTE_VERSION), + sdkVersion: z.string(), + recordedAt: z.string(), + provider: z.object({ name: z.string(), defaultModel: z.string().optional() }), + entries: z.array(entrySchema), +}); + +/** The normalized, redacted request stored (and matched) for each call. */ +export type CassetteRequest = z.infer; +/** A recorded response (for `stream()` entries also the chunk list). */ +export type CassetteResponse = z.infer; +/** A serialized error: name and message only. */ +export type CassetteError = z.infer; +/** One recorded exchange. */ +export type CassetteEntry = z.infer; +/** The parsed cassette file. */ +export type Cassette = z.infer; + +export const RERECORD_HINT = + "If the change is intentional, re-record the cassette (run with LOUSHY_RECORD=1, or set mode: 'record')."; + +/** Start a fresh cassette for a recording session. */ +export function newCassette(provider: Cassette['provider']): Cassette { + return { + version: CASSETTE_VERSION, + sdkVersion: readSdkVersion(), + recordedAt: new Date().toISOString(), + provider, + entries: [], + }; +} + +/** Read and validate a cassette; every failure explains how to fix it. */ +export function readCassette(file: string): Cassette { + let raw: string; + try { + raw = fs.readFileSync(file, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + throw new Error( + `recordReplay: cassette not found at ${file}. Record it first by running the test once ` + + "with a real provider in record mode (LOUSHY_RECORD=1 or mode: 'record'), then commit the file." + ); + } + throw error; + } + return parseCassette(file, raw); +} + +function parseCassette(file: string, raw: string): Cassette { + let json: unknown; + try { + json = JSON.parse(raw); + } catch { + throw new Error(`recordReplay: cassette ${file} is not valid JSON. Delete it and re-record.`); + } + const version = (json as { version?: unknown } | null)?.version; + if (version !== CASSETTE_VERSION) { + throw new Error( + `recordReplay: cassette ${file} has format version ${String(version)}, but this SDK reads version ` + + `${CASSETTE_VERSION}. Re-record it with this SDK version.` + ); + } + const parsed = cassetteSchema.safeParse(json); + if (!parsed.success) { + const issue = parsed.error.issues[0]; + throw new Error( + `recordReplay: cassette ${file} is malformed at ${issue.path.join('.') || '(root)'}: ${issue.message}. ` + + 'Delete it and re-record.' + ); + } + return parsed.data; +} + +/** + * Write the cassette atomically: a temp file in the same directory, then a + * rename. A crash mid-write leaves the previous cassette untouched. + */ +export async function writeCassette(file: string, cassette: Cassette): Promise { + const temp = `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`; + try { + await fs.promises.mkdir(path.dirname(file), { recursive: true }); + await fs.promises.writeFile(temp, `${JSON.stringify(cassette, null, 2)}\n`, 'utf8'); + await fs.promises.rename(temp, file); + } catch (error) { + await fs.promises.rm(temp, { force: true }).catch(() => undefined); + throw error; + } +} + +/** Find this package's version for the cassette header ("unknown" if it cannot be found). */ +function readSdkVersion(): string { + let dir = __dirname; + for (let depth = 0; depth < 5; depth++) { + try { + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8')) as { + name?: string; + version?: string; + }; + if (manifest.name === '@loushy/build-ai-agent' && manifest.version) return manifest.version; + } catch { + // keep walking up + } + dir = path.dirname(dir); + } + return 'unknown'; +} diff --git a/src/testing/fingerprint.ts b/src/testing/fingerprint.ts new file mode 100644 index 000000000..048cb0888 --- /dev/null +++ b/src/testing/fingerprint.ts @@ -0,0 +1,257 @@ +/** + * Request fingerprinting, normalization and redaction for `recordReplay`. + * + * A request is reduced to the fields that decide what the model says (model, + * messages, tools, temperature, maxTokens), volatile values are replaced by + * placeholders and secrets are redacted, so the same logical request always + * produces the same canonical JSON. + */ + +import type { GenerateOptions, Message, ToolDefinition } from '../providers/llm'; +import type { CassetteRequest } from './cassette'; + +const MAX_DEPTH = 24; + +/** Volatile value patterns replaced by a placeholder before matching. */ +const VOLATILE: ReadonlyArray = [ + [/\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?/g, ''], + [/\b1[6-9]\d{11}\b/g, ''], + [/\b\d{4}-\d{2}-\d{2}\b/g, ''], + [ + /\b(?:(?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,? )?(?:January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2},? \d{4}\b/g, + '', + ], + [/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi, ''], + [/\b(?:call|toolu|chatcmpl|msg|run|sess|session|req)_[A-Za-z0-9]{8,}\b/g, ''], +]; + +/** Secret patterns redacted from everything written to a cassette. */ +const SECRETS: readonly RegExp[] = [ + /\bsk-ant-[A-Za-z0-9_-]{8,}/g, + /\bsk-[A-Za-z0-9_-]{16,}/g, + /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, + /\bAIza[0-9A-Za-z_-]{30,}/g, + /\b(?:ghp|gho|ghs|github_pat)_[A-Za-z0-9_]{20,}/g, + /\bAKIA[0-9A-Z]{16}\b/g, +]; + +const REDACTED = '[REDACTED]'; + +/** Zod `_def` keys copied into a fingerprint, with the name they get in it. */ +const ZOD_KEYS: ReadonlyArray = [ + ['description', 'description'], + ['shape', 'properties'], + ['innerType', 'inner'], + ['schema', 'inner'], + ['type', 'items'], + ['valueType', 'values'], + ['keyType', 'keys'], + ['options', 'options'], + ['items', 'items'], + ['values', 'enum'], + ['value', 'const'], + ['checks', 'checks'], +]; + +type ZodDef = Record & { typeName: string }; + +function zodDefOf(value: object): ZodDef | undefined { + const def = (value as { _def?: unknown })._def; + if (typeof def !== 'object' || def === null) return undefined; + return typeof (def as ZodDef).typeName === 'string' ? (def as ZodDef) : undefined; +} + +function zodFingerprint(def: ZodDef, depth: number): Record { + const out: Record = { type: def.typeName.replace(/^Zod/, '').toLowerCase() }; + for (const [from, to] of ZOD_KEYS) { + let child = def[from]; + if (child === undefined) continue; + if (from === 'shape' && typeof child === 'function') child = (child as () => unknown)(); + if (from === 'checks' && Array.isArray(child)) { + child = child.map((check: { kind: string; value?: unknown }) => ({ kind: check.kind, value: check.value })); + } + out[to] = canonicalize(child, depth + 1); + } + return sortKeys(out); +} + +function sortKeys(record: Record): Record { + const sorted: Record = {}; + for (const key of Object.keys(record).sort()) { + if (record[key] !== undefined) sorted[key] = record[key]; + } + return sorted; +} + +/** + * Reduce any value to plain, key-sorted JSON. Zod schemas become a + * JSON-schema-ish fingerprint; functions and `undefined` are dropped. + */ +export function canonicalize(value: unknown, depth = 0): unknown { + if (value === undefined || typeof value === 'function') return undefined; + if (typeof value !== 'object' || value === null) return value; + if (depth > MAX_DEPTH) return '[max depth]'; + if (Array.isArray(value)) return value.map((item) => canonicalize(item, depth + 1) ?? null); + const zodDef = zodDefOf(value); + if (zodDef) return zodFingerprint(zodDef, depth); + const out: Record = {}; + for (const [key, child] of Object.entries(value)) out[key] = canonicalize(child, depth + 1); + return sortKeys(out); +} + +/** Deterministic JSON text of any value (used to compare requests). */ +export function stableStringify(value: unknown): string { + return JSON.stringify(canonicalize(value)); +} + +/** Apply `fn` to every string inside a JSON-like value (keys are left alone). */ +export function mapStrings(value: T, fn: (text: string) => string): T { + if (typeof value === 'string') return fn(value) as T; + if (Array.isArray(value)) return value.map((item) => mapStrings(item, fn)) as T; + if (typeof value === 'object' && value !== null) { + const out: Record = {}; + for (const [key, child] of Object.entries(value)) out[key] = mapStrings(child, fn); + return out as T; + } + return value; +} + +function replaceAll(text: string, patterns: ReadonlyArray): string { + let result = text; + for (const pattern of patterns) { + result = Array.isArray(pattern) + ? result.replace(pattern[0] as RegExp, pattern[1] as string) + : result.replace(pattern as RegExp, REDACTED); + } + return result; +} + +function normalizeArguments(argumentsJson: string): string { + try { + return stableStringify(JSON.parse(argumentsJson)); + } catch { + return argumentsJson; + } +} + +function toCassetteMessage(message: Message): CassetteRequest['messages'][number] { + return { + role: message.role, + content: message.content, + ...(message.name !== undefined ? { name: message.name } : {}), + ...(message.toolName !== undefined ? { toolName: message.toolName } : {}), + ...(message.isError !== undefined ? { isError: message.isError } : {}), + ...(message.toolCalls?.length + ? { + toolCalls: message.toolCalls.map((call) => ({ + name: call.function.name, + arguments: normalizeArguments(call.function.arguments), + })), + } + : {}), + }; +} + +function toCassetteTool(tool: ToolDefinition): CassetteRequest['tools'][number] { + return { + name: tool.function.name, + description: tool.function.description, + parameters: canonicalize(tool.function.parameters) ?? {}, + }; +} + +/** Options a sanitizer is built from. */ +export interface SanitizerOptions { + normalize?: (request: GenerateOptions) => GenerateOptions; + redact?: (text: string) => string; + defaultModel?: string; +} + +/** Turns live requests/responses into what is stored in, and matched against, a cassette. */ +export interface Sanitizer { + /** Canonical request: reduced to match fields, normalized and redacted. */ + request(options: GenerateOptions): CassetteRequest; + /** Redact secrets (and apply the user's `redact`) throughout a response value. */ + redact(value: T): T; +} + +/** Build the request/response sanitizer shared by record and replay. */ +export function createSanitizer(options: SanitizerOptions): Sanitizer { + const redactText = (text: string): string => { + const clean = replaceAll(text, SECRETS); + return options.redact ? options.redact(clean) : clean; + }; + const cleanText = (text: string): string => redactText(replaceAll(text, VOLATILE)); + return { + request(request) { + const source = options.normalize ? options.normalize(request) : request; + const canonical: CassetteRequest = { + model: source.model || options.defaultModel || null, + messages: source.messages.map(toCassetteMessage), + tools: (source.tools ?? []).map(toCassetteTool), + temperature: source.temperature ?? null, + maxTokens: source.maxTokens ?? null, + }; + return JSON.parse(JSON.stringify(mapStrings(canonical, cleanText))) as CassetteRequest; + }, + redact: (value) => mapStrings(value, redactText), + }; +} + +export interface Difference { + /** Path of the first differing value, e.g. `request.messages[1].content`. */ + path: string; + expected: string; + actual: string; +} + +function show(value: unknown): string { + if (value === undefined) return '(missing)'; + const text = JSON.stringify(value); + return text.length > 200 ? `${text.slice(0, 197)}...` : text; +} + +function showStrings(expected: string, actual: string): [string, string] { + let at = 0; + while (at < expected.length && expected[at] === actual[at]) at++; + const window = (text: string): string => { + const start = Math.max(0, at - 30); + const slice = text.slice(start, at + 60); + return `${start > 0 ? '...' : ''}${JSON.stringify(slice)}${text.length > at + 60 ? '...' : ''}`; + }; + return expected.length > 100 || actual.length > 100 + ? [`${window(expected)} (first differs at char ${at})`, window(actual)] + : [show(expected), show(actual)]; +} + +function childPaths(a: unknown, b: unknown): Array { + if (Array.isArray(a) && Array.isArray(b)) { + return Array.from({ length: Math.max(a.length, b.length) }, (_, index) => index); + } + const keys = new Set([...Object.keys(a as object), ...Object.keys(b as object)]); + return [...keys].sort(); +} + +function isContainer(value: unknown): boolean { + return typeof value === 'object' && value !== null; +} + +/** The first place two canonical JSON values differ, or `undefined` if equal. */ +export function firstDifference(expected: unknown, actual: unknown, path = ''): Difference | undefined { + const bothContainers = isContainer(expected) && isContainer(actual); + if (bothContainers && Array.isArray(expected) === Array.isArray(actual)) { + for (const key of childPaths(expected, actual)) { + const child = (value: unknown): unknown => (value as Record)[key]; + const where = typeof key === 'number' ? `${path}[${key}]` : path ? `${path}.${key}` : key; + const found = firstDifference(child(expected), child(actual), where); + if (found) return found; + } + return undefined; + } + if (JSON.stringify(expected) === JSON.stringify(actual)) return undefined; + const [shownExpected, shownActual] = + typeof expected === 'string' && typeof actual === 'string' + ? showStrings(expected, actual) + : [show(expected), show(actual)]; + return { path: path || '(root)', expected: shownExpected, actual: shownActual }; +} diff --git a/src/testing/index.ts b/src/testing/index.ts index ffe4b51d8..79faca0a0 100644 --- a/src/testing/index.ts +++ b/src/testing/index.ts @@ -1,7 +1,8 @@ /** * `@loushy/build-ai-agent/testing` - test doubles for agents. * - * Re-exports the in-memory repository mocks plus the scripted `mockModel`. + * Re-exports the in-memory repository mocks, the scripted `mockModel` and the + * `recordReplay` VCR provider. */ export * from '../data/mocks'; @@ -16,3 +17,10 @@ export type { MockTurn, MockTurnObject, } from './mockModel'; +export { recordReplay, CassetteMismatchError } from './recordReplay'; +export type { + RecordReplayMode, + RecordReplayOptions, + RecordReplayProvider, + RecordReplaySource, +} from './recordReplay'; diff --git a/src/testing/recordReplay.test.ts b/src/testing/recordReplay.test.ts new file mode 100644 index 000000000..6d12fc37b --- /dev/null +++ b/src/testing/recordReplay.test.ts @@ -0,0 +1,476 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { z } from 'zod'; +import { CassetteMismatchError, mockModel, recordReplay } from './index'; +import { createAgent } from '../createAgent'; +import { defineTool } from '../tools/defineTool'; +import type { GenerateOptions, StreamChunk } from '../providers/llm'; + +let dir: string; +let file: string; + +beforeEach(() => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cassette-')); + file = path.join(dir, 'nested', 'flow.json'); +}); + +afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }); +}); + +const ask = (content: string, extra: Partial = {}): GenerateOptions => ({ + messages: [{ role: 'user', content }], + ...extra, +}); + +const forbidden = (): never => { + throw new Error('the wrapped provider must not be constructed in replay mode'); +}; + +const readJson = (): { entries: Array<{ request: { messages: Array<{ content: string }> } }> } => JSON.parse(fs.readFileSync(file, 'utf8')); + +async function record(script: Parameters[0], calls: GenerateOptions[]): Promise { + const provider = recordReplay(mockModel(script), { cassette: file, mode: 'record' }); + for (const call of calls) await provider.generate(call); +} + +describe('recordReplay record then replay', () => { + it('replays recorded responses without constructing the wrapped provider', async () => { + const live = recordReplay( + mockModel( + [ + { toolCalls: [{ name: 'lookup', args: { id: 7 } }], usage: { inputTokens: 10, outputTokens: 4 } }, + { text: 'done', usage: { inputTokens: 20, outputTokens: 2 } }, + ], + { defaultModel: 'gpt-test' } + ), + { cassette: file, mode: 'record' } + ); + const first = await live.generate(ask('one')); + const second = await live.generate(ask('two')); + + const replay = recordReplay(forbidden, { cassette: file, mode: 'replay' }); + + expect(replay.mode).toBe('replay'); + expect(replay.defaultModel).toBe('gpt-test'); + expect(replay.name).toBe('mock'); + expect(await replay.generate(ask('one'))).toEqual(first); + expect(await replay.generate(ask('two'))).toEqual(second); + expect(await replay.getModels()).toEqual(['gpt-test']); + expect(replay.supportsTools('x') && replay.supportsStreaming('x')).toBe(true); + }); + + it('replays an agent run end to end with the same usage numbers', async () => { + const lookup = defineTool({ + name: 'lookup', + description: 'Look up an order', + input: z.object({ id: z.number() }), + execute: async ({ id }) => ({ id, status: 'shipped' }), + }); + const build = (provider: ReturnType) => + createAgent({ prompt: 'You help.', provider, tools: { lookup } }); + const script = [ + { toolCalls: [{ name: 'lookup', args: { id: 7 } }], usage: { inputTokens: 5, outputTokens: 1 } }, + { text: 'Order 7 shipped', usage: { inputTokens: 9, outputTokens: 3 } }, + ]; + + const recorded = await build(recordReplay(mockModel(script), { cassette: file, mode: 'record' })).send('status?'); + const replayed = await build(recordReplay(forbidden, { cassette: file, mode: 'replay' })).send('status?'); + + expect(replayed.text).toBe('Order 7 shipped'); + expect(replayed.text).toBe(recorded.text); + expect(readJson().entries).toHaveLength(2); + }); + + it('writes a versioned, stable, 2-space indented cassette without rawResponse', async () => { + const model = mockModel(['hi']); + const provider = recordReplay( + { ...model, generate: async (o) => ({ ...(await model.generate(o)), rawResponse: { headers: { authorization: 'x' } } }) } as typeof model, + { cassette: file, mode: 'record' } + ); + await provider.generate(ask('hello', { temperature: 0.2, maxTokens: 50 })); + + const text = fs.readFileSync(file, 'utf8'); + const json = JSON.parse(text); + + expect(text).toBe(`${JSON.stringify(json, null, 2)}\n`); + expect(Object.keys(json)).toEqual(['version', 'sdkVersion', 'recordedAt', 'provider', 'entries']); + expect(json.version).toBe(1); + expect(typeof json.sdkVersion).toBe('string'); + expect(json.entries[0].request).toEqual({ + model: 'mock-model', + messages: [{ role: 'user', content: 'hello' }], + tools: [], + temperature: 0.2, + maxTokens: 50, + }); + expect(text).not.toContain('authorization'); + }); +}); + +describe('recordReplay mismatches', () => { + it('throws CassetteMismatchError with a readable diff and the re-record hint', async () => { + await record(['a'], [ask('What is the refund policy for order 1234?')]); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + + const error = await replay.generate(ask('What is the refund policy for order 9999?')).catch((e) => e); + + expect(error).toBeInstanceOf(CassetteMismatchError); + expect(error.name).toBe('CassetteMismatchError'); + expect(error.callNumber).toBe(1); + expect(error.message).toContain('First difference at request.messages[0].content'); + expect(error.message).toContain('order 1234'); + expect(error.message).toContain('order 9999'); + expect(error.message).toContain('LOUSHY_RECORD=1'); + }); + + it('detects changed tools, schemas, temperature and a missing message', async () => { + const tool = (shape: z.ZodRawShape) => ({ + type: 'function' as const, + function: { name: 't', description: 'd', parameters: z.object(shape) }, + }); + await record(['a'], [ask('q', { tools: [tool({ city: z.string() })], temperature: 0 })]); + const replay = (): ReturnType => recordReplay(undefined, { cassette: file, mode: 'replay' }); + + const schema = await replay().generate(ask('q', { tools: [tool({ city: z.number() })], temperature: 0 })).catch((e) => e); + const temp = await replay().generate(ask('q', { tools: [tool({ city: z.string() })], temperature: 1 })).catch((e) => e); + const same = await replay().generate(ask('q', { tools: [tool({ city: z.string() })], temperature: 0 })); + + expect(schema.message).toContain('request.tools[0].parameters.properties.city.type'); + expect(temp.message).toContain('request.temperature'); + expect(same.text).toBe('a'); + }); + + it('fails clearly when the agent makes more calls than were recorded', async () => { + await record(['a'], [ask('q')]); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + await replay.generate(ask('q')); + + await expect(replay.generate(ask('q'))).rejects.toThrow(/Call #2 has no recorded entry.*holds 1 entry/s); + }); + + it('does not let a generate entry answer a stream call', async () => { + await record(['a'], [ask('q')]); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + + await expect(replay.stream(ask('q'))).rejects.toThrow(/First difference at kind/); + }); + + it('rejects a missing, corrupt or wrong-version cassette with a fix', () => { + expect(() => recordReplay(undefined, { cassette: file, mode: 'replay' })).toThrow(/not found.*LOUSHY_RECORD=1/s); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, '{nope'); + expect(() => recordReplay(undefined, { cassette: file, mode: 'replay' })).toThrow(/not valid JSON/); + fs.writeFileSync(file, JSON.stringify({ version: 99, entries: [] })); + expect(() => recordReplay(undefined, { cassette: file, mode: 'replay' })).toThrow(/version 99.*Re-record/s); + fs.writeFileSync(file, JSON.stringify({ version: 1, entries: 3 })); + expect(() => recordReplay(undefined, { cassette: file, mode: 'replay' })).toThrow(/malformed/); + }); +}); + +describe("recordReplay match: 'request'", () => { + it('finds entries by fingerprint regardless of call order', async () => { + await record(['answer a', 'answer b'], [ask('a'), ask('b')]); + const replay = recordReplay(forbidden, { cassette: file, mode: 'replay', match: 'request' }); + + const [b, a] = await Promise.all([replay.generate(ask('b')), replay.generate(ask('a'))]); + + expect([a.text, b.text]).toEqual(['answer a', 'answer b']); + }); + + it('uses identical requests in recorded order and errors when none remain or none match', async () => { + await record(['first', 'second'], [ask('same'), ask('same')]); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay', match: 'request' }); + + expect((await replay.generate(ask('same'))).text).toBe('first'); + expect((await replay.generate(ask('same'))).text).toBe('second'); + await expect(replay.generate(ask('same'))).rejects.toThrow(/no recorded entry/); + + const fresh = recordReplay(undefined, { cassette: file, mode: 'replay', match: 'request' }); + await expect(fresh.generate(ask('other'))).rejects.toThrow(/No unused recorded request matches[\s\S]*request.messages\[0\].content/); + }); +}); + +describe('recordReplay normalization', () => { + it('ignores timestamps, dates, uuids and generated ids when matching', async () => { + await record( + ['ok'], + [ + ask('Today is 2026-10-01T09:15:00.000Z, session 123e4567-e89b-12d3-a456-426614174000, ref call_AbC12345xyz'), + ] + ); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + + const result = await replay.generate( + ask('Today is 2027-01-05T18:00:01.500Z, session 99999999-e89b-12d3-a456-426614174000, ref call_ZzZ98765abc') + ); + + expect(result.text).toBe('ok'); + expect(fs.readFileSync(file, 'utf8')).toContain('Today is , session , ref '); + }); + + it('matches tool calls structurally, ignoring the generated tool-call ids', async () => { + const turn = (id: string): GenerateOptions => ({ + messages: [ + { role: 'user', content: 'go' }, + { role: 'assistant', content: '', toolCalls: [{ id, type: 'function', function: { name: 'a', arguments: '{"y":2,"x":1}' } }] }, + { role: 'tool', content: '{"ok":true}', toolCallId: id, toolName: 'a' }, + ], + }); + await record(['done'], [turn('call_1')]); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + + const result = await replay.generate({ + messages: turn('toolu_different').messages.map((m) => + m.toolCalls ? { ...m, toolCalls: [{ ...m.toolCalls[0], function: { name: 'a', arguments: '{"x":1,"y":2}' } }] } : m + ), + }); + + expect(result.text).toBe('done'); + }); + + it('applies a custom normalize hook on record and replay', async () => { + const normalize = (r: GenerateOptions): GenerateOptions => ({ + ...r, + messages: r.messages.map((m) => ({ ...m, content: m.content.replace(/user-\d+/g, 'user-N') })), + }); + await record(['ok'], [ask('hello user-1')]); + fs.rmSync(file); + const provider = recordReplay(mockModel(['ok']), { cassette: file, mode: 'record', normalize }); + await provider.generate(ask('hello user-1')); + + const replay = recordReplay(undefined, { cassette: file, mode: 'replay', normalize }); + + expect((await replay.generate(ask('hello user-777'))).text).toBe('ok'); + expect(readJson().entries[0].request.messages[0].content).toBe('hello user-N'); + }); +}); + +describe('recordReplay redaction', () => { + it('redacts API-key-like strings and bearer tokens from requests and responses', async () => { + const anthropic = `sk-ant-${'a1B2c3D4'.repeat(4)}`; + const openai = `sk-proj-${'Z9y8X7w6'.repeat(4)}`; + const model = mockModel([ + { text: `your key is ${openai}`, toolCalls: [{ name: 'send', args: { auth: 'Bearer abcdef1234567890.token' } }] }, + ]); + const provider = recordReplay(model, { cassette: file, mode: 'record' }); + + await provider.generate(ask(`use ${anthropic} and Authorization: Bearer eyJhbGciOi.payload-1234`)); + + const text = fs.readFileSync(file, 'utf8'); + expect(text).not.toContain(anthropic); + expect(text).not.toContain(openai); + expect(text).not.toContain('eyJhbGciOi'); + expect(text).not.toContain('abcdef1234567890'); + expect(text.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(4); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + expect((await replay.generate(ask(`use ${anthropic} and Authorization: Bearer eyJhbGciOi.payload-1234`))).text).toBe( + 'your key is [REDACTED]' + ); + }); + + it('applies the custom redact function to every recorded string', async () => { + const provider = recordReplay(mockModel(['ssn is 123-45-6789']), { + cassette: file, + mode: 'record', + redact: (t) => t.replace(/\d{3}-\d{2}-\d{4}/g, ''), + }); + + await provider.generate(ask('my ssn is 123-45-6789')); + + const text = fs.readFileSync(file, 'utf8'); + expect(text).not.toContain('123-45-6789'); + expect(text).toContain('ssn is '); + }); +}); + +describe('recordReplay persistence', () => { + it('writes after every call so a test that fails midway leaves a valid cassette', async () => { + const provider = recordReplay(mockModel(['one', 'two']), { cassette: file, mode: 'record' }); + await provider.generate(ask('a')); + + expect(readJson().entries).toHaveLength(1); + await provider.generate(ask('b')); + expect(readJson().entries).toHaveLength(2); + expect(fs.readdirSync(path.dirname(file))).toEqual(['flow.json']); + }); + + it('keeps the previous cassette intact and leaves no temp file when the write fails', async () => { + const provider = recordReplay(mockModel(['one', 'two']), { cassette: file, mode: 'record' }); + await provider.generate(ask('a')); + const before = fs.readFileSync(file, 'utf8'); + const rename = vi.spyOn(fs.promises, 'rename').mockRejectedValueOnce(new Error('disk full')); + + await expect(provider.generate(ask('b'))).rejects.toThrow('disk full'); + + rename.mockRestore(); + expect(fs.readFileSync(file, 'utf8')).toBe(before); + expect(fs.readdirSync(path.dirname(file))).toEqual(['flow.json']); + }); + + it('save() writes an empty cassette and is a no-op in replay', async () => { + const provider = recordReplay(mockModel([]), { cassette: file, mode: 'record' }); + await provider.save(); + expect(readJson().entries).toEqual([]); + + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + const before = fs.statSync(file).mtimeMs; + await replay.save(); + expect(fs.statSync(file).mtimeMs).toBe(before); + }); + + it('keeps parallel calls in call-start order', async () => { + const provider = recordReplay(mockModel([{ text: 'slow', delayMs: 30 }, 'fast']), { cassette: file, mode: 'record' }); + + await Promise.all([provider.generate(ask('slow')), provider.generate(ask('fast'))]); + + expect(readJson().entries.map((e) => e.request.messages[0].content)).toEqual(['slow', 'fast']); + }); + + it('starts a fresh cassette each record session', async () => { + await record(['a', 'b'], [ask('1'), ask('2')]); + await record(['c'], [ask('3')]); + + expect(readJson().entries).toHaveLength(1); + }); +}); + +describe('recordReplay errors', () => { + it('records provider rejections and replays them with the same name and message', async () => { + const boom = Object.assign(new Error('rate limited, retry in 5s'), { name: 'RateLimitError' }); + const provider = recordReplay(mockModel([{ error: boom }, 'ok']), { cassette: file, mode: 'record' }); + await expect(provider.generate(ask('a'))).rejects.toBe(boom); + await provider.generate(ask('b')); + + const replay = recordReplay(forbidden, { cassette: file, mode: 'replay' }); + const error = await replay.generate(ask('a')).catch((e) => e); + + expect(error).toBeInstanceOf(Error); + expect(error.name).toBe('RateLimitError'); + expect(error.message).toBe('rate limited, retry in 5s'); + expect((await replay.generate(ask('b'))).text).toBe('ok'); + }); + + it('replays a rejected stream() call', async () => { + const provider = recordReplay(mockModel([{ error: new Error('nope') }]), { cassette: file, mode: 'record' }); + await expect(provider.stream(ask('a'))).rejects.toThrow('nope'); + + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + + await expect(replay.stream(ask('a'))).rejects.toThrow('nope'); + }); + + it('rejects immediately when the call is already aborted', async () => { + await record(['a'], [ask('q')]); + const replay = recordReplay(undefined, { cassette: file, mode: 'replay' }); + + await expect(replay.generate(ask('q', { signal: AbortSignal.abort(new Error('stop')) }))).rejects.toThrow('stop'); + }); +}); + +describe('recordReplay stream()', () => { + const collect = async (iterable: AsyncIterable): Promise => { + const out: StreamChunk[] = []; + for await (const chunk of iterable) out.push(chunk); + return out; + }; + + it('round-trips chunks, text, usage, finish reason and tool calls', async () => { + const model = mockModel([ + { text: 'hello brave world', toolCalls: [{ name: 'x', args: { a: 1 } }], usage: { inputTokens: 3, outputTokens: 5 } }, + ]); + const live = await recordReplay(model, { cassette: file, mode: 'record' }).stream(ask('s')); + const liveChunks = await collect(live.fullStream); + + const replayed = await recordReplay(forbidden, { cassette: file, mode: 'replay' }).stream(ask('s')); + + expect(await collect(replayed.fullStream)).toEqual(liveChunks); + expect(await replayed.text).toBe('hello brave world'); + expect(await replayed.usage).toEqual({ promptTokens: 3, completionTokens: 5, totalTokens: 8 }); + expect(await replayed.finishReason).toBe('tool_calls'); + expect((await replayed.toolCalls).map((c) => c.function.name)).toEqual(['x']); + const texts: string[] = []; + for await (const part of replayed.textStream) texts.push(part); + expect(texts.join('')).toBe('hello brave world'); + }); + + it('replays without delays by default and with recorded timing when asked', async () => { + const slowStream = { + ...mockModel(['x']), + stream: async () => ({ + fullStream: (async function* (): AsyncGenerator { + yield { type: 'text-delta', textDelta: 'a' }; + await new Promise((r) => setTimeout(r, 60)); + yield { type: 'text-delta', textDelta: 'b' }; + yield { type: 'finish', finishReason: 'stop' }; + })(), + textStream: (async function* () {})(), + text: Promise.resolve('ab'), + usage: Promise.resolve({ promptTokens: 1, completionTokens: 1, totalTokens: 2 }), + finishReason: Promise.resolve('stop'), + toolCalls: Promise.resolve([]), + }), + }; + const recorder = recordReplay(slowStream, { cassette: file, mode: 'record' }); + await recorder.stream(ask('s')); + + const timed = async (replayTiming: boolean): Promise => { + const start = Date.now(); + const replay = await recordReplay(undefined, { cassette: file, mode: 'replay', replayTiming }).stream(ask('s')); + await collect(replay.fullStream); + return Date.now() - start; + }; + + expect(await timed(false)).toBeLessThan(40); + expect(await timed(true)).toBeGreaterThanOrEqual(50); + }); + + it('records error chunks', async () => { + const withError = { + ...mockModel(['x']), + stream: async () => ({ + fullStream: (async function* (): AsyncGenerator { + yield { type: 'error', error: Object.assign(new Error('mid-stream'), { name: 'StreamError' }) }; + })(), + textStream: (async function* () {})(), + text: Promise.resolve(''), + usage: Promise.resolve({ promptTokens: 0, completionTokens: 0, totalTokens: 0 }), + finishReason: Promise.resolve('error'), + toolCalls: Promise.resolve([]), + }), + }; + await recordReplay(withError, { cassette: file, mode: 'record' }).stream(ask('s')); + + const replay = await recordReplay(undefined, { cassette: file, mode: 'replay' }).stream(ask('s')); + const [chunk] = await collect(replay.fullStream); + + expect(chunk.error?.name).toBe('StreamError'); + expect(chunk.error?.message).toBe('mid-stream'); + }); +}); + +describe("recordReplay mode 'auto' and construction", () => { + it('records when the cassette is missing, then replays once it exists', async () => { + const first = recordReplay(mockModel(['recorded']), { cassette: file }); + expect(first.mode).toBe('record'); + await first.generate(ask('q')); + + const second = recordReplay(forbidden, { cassette: file }); + + expect(second.mode).toBe('replay'); + expect((await second.generate(ask('q'))).text).toBe('recorded'); + }); + + it('only calls a lazy factory when recording', () => { + const factory = vi.fn(() => mockModel(['x'])); + + recordReplay(factory, { cassette: file, mode: 'record' }); + expect(factory).toHaveBeenCalledTimes(1); + }); + + it('explains how to fix record mode without a provider', () => { + expect(() => recordReplay(undefined, { cassette: file, mode: 'record' })).toThrow(/needs a provider to record from/); + }); +}); diff --git a/src/testing/recordReplay.ts b/src/testing/recordReplay.ts new file mode 100644 index 000000000..0c10f70be --- /dev/null +++ b/src/testing/recordReplay.ts @@ -0,0 +1,457 @@ +/** + * `recordReplay`: a VCR for `LLMProvider`s. Run once against a real model and + * record every exchange to a cassette file; replay it in CI with no network + * and no API key. + */ + +import * as fs from 'node:fs'; +import type { + GenerateOptions, + GenerateResult, + LLMProvider, + StreamChunk, + StreamResult, +} from '../providers/llm'; +import { + RERECORD_HINT, + newCassette, + readCassette, + writeCassette, + type Cassette, + type CassetteEntry, + type CassetteError, + type CassetteRequest, + type CassetteResponse, +} from './cassette'; +import { createSanitizer, firstDifference, stableStringify, type Sanitizer } from './fingerprint'; + +/** Whether a {@link recordReplay} provider records or replays. */ +export type RecordReplayMode = 'record' | 'replay' | 'auto'; + +/** The provider to wrap: an instance, a lazy factory, or `undefined` (replay only). */ +export type RecordReplaySource = LLMProvider | (() => LLMProvider) | undefined; + +/** Options for {@link recordReplay}. */ +export interface RecordReplayOptions { + /** Path of the cassette JSON file. Its directory is created when recording. */ + cassette: string; + /** + * - `'record'`: call the wrapped provider and write the cassette (replacing any old one). + * - `'replay'`: never call the wrapped provider; serve the recorded responses. + * - `'auto'` (default): replay if the cassette file exists, else record. + */ + mode?: RecordReplayMode; + /** + * How replay finds the entry for a call. `'sequence'` (default): entry N + * answers call N, and the request must still match. `'request'`: look the + * entry up by request fingerprint regardless of order (parallel calls). + */ + match?: 'sequence' | 'request'; + /** + * Rewrite a request before it is fingerprinted and stored, e.g. to blank a + * value that changes every run. Runs before the built-in normalization. + */ + normalize?: (request: GenerateOptions) => GenerateOptions; + /** + * Redact text before it is stored (applied to every recorded string, after + * the built-in API-key redaction). + */ + redact?: (text: string) => string; + /** Replay `stream()` with the recorded inter-chunk delays. Default `false` (no delays). */ + replayTiming?: boolean; +} + +/** + * A recording/replaying `LLMProvider`. + * + * @example + * ```ts + * const provider = recordReplay(() => resolveProvider('openai/gpt-4o-mini'), { + * cassette: './__cassettes__/refund-flow.json', + * mode: process.env.LOUSHY_RECORD ? 'record' : 'replay', + * }); + * ``` + */ +export interface RecordReplayProvider extends LLMProvider { + /** The mode in effect (`'auto'` is resolved at construction). */ + readonly mode: 'record' | 'replay'; + /** + * Write the cassette now. Recording already writes after every call, so this + * is only needed to force a write (for example when no call was made). + * A no-op in replay mode. + */ + save(): Promise; +} + +/** Thrown in replay mode when a request does not match the cassette (or it ran out of entries). */ +export class CassetteMismatchError extends Error { + constructor( + message: string, + /** The cassette file. */ + readonly cassette: string, + /** 1-based number of the call that failed to match. */ + readonly callNumber: number + ) { + super(message); + this.name = 'CassetteMismatchError'; + } +} + +function toCassetteError(error: unknown, sanitizer: Sanitizer): CassetteError { + const err = error instanceof Error ? error : new Error(String(error)); + return sanitizer.redact({ name: err.name, message: err.message }); +} + +function fromCassetteError(error: CassetteError): Error { + const err = new Error(error.message); + err.name = error.name; + return err; +} + +type StoredChunk = NonNullable[number]['chunk']; + +function serializeChunk(chunk: StreamChunk): StoredChunk { + return { + type: chunk.type, + ...(chunk.textDelta !== undefined ? { textDelta: chunk.textDelta } : {}), + ...(chunk.toolCall ? { toolCall: chunk.toolCall } : {}), + ...(chunk.toolResult ? { toolResult: chunk.toolResult } : {}), + ...(chunk.finishReason !== undefined ? { finishReason: chunk.finishReason } : {}), + ...(chunk.usage ? { usage: chunk.usage } : {}), + ...(chunk.error ? { error: { name: chunk.error.name, message: chunk.error.message } } : {}), + }; +} + +function deserializeChunk(stored: StoredChunk): StreamChunk { + const { error, toolResult, ...rest } = stored; + return { + ...rest, + ...(toolResult ? { toolResult: { toolCallId: toolResult.toolCallId, result: toolResult.result } } : {}), + ...(error ? { error: fromCassetteError(error) } : {}), + }; +} + +function clone(value: T): T { + return JSON.parse(JSON.stringify(value)) as T; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Rebuild a `StreamResult` from a recorded stream entry. */ +function replayStream(response: CassetteResponse, replayTiming: boolean): StreamResult { + const chunks = response.chunks ?? []; + const fullStream = async function* (): AsyncGenerator { + for (const { delayMs, chunk } of chunks) { + if (replayTiming && delayMs > 0) await sleep(delayMs); + yield deserializeChunk(clone(chunk)); + } + }; + const textStream = async function* (): AsyncGenerator { + for await (const chunk of fullStream()) { + if (chunk.type === 'text-delta' && chunk.textDelta !== undefined) yield chunk.textDelta; + } + }; + return { + fullStream: fullStream(), + textStream: textStream(), + text: Promise.resolve(response.text), + usage: Promise.resolve(clone(response.usage)), + finishReason: Promise.resolve(response.finishReason), + toolCalls: Promise.resolve(clone(response.toolCalls ?? [])), + }; +} + +function toGenerateResult(response: CassetteResponse): GenerateResult { + return { + text: response.text, + finishReason: response.finishReason as GenerateResult['finishReason'], + usage: clone(response.usage), + ...(response.toolCalls?.length ? { toolCalls: clone(response.toolCalls) } : {}), + }; +} + +class Recorder implements RecordReplayProvider { + readonly mode = 'record'; + readonly name: string; + readonly defaultModel: string | undefined; + private readonly cassette: Cassette; + private readonly slots: Array = []; + private readonly sanitizer: Sanitizer; + private writing: Promise = Promise.resolve(); + + constructor( + private readonly provider: LLMProvider, + private readonly options: RecordReplayOptions + ) { + this.name = provider.name; + this.defaultModel = provider.defaultModel; + this.cassette = newCassette({ + name: provider.name, + ...(provider.defaultModel ? { defaultModel: provider.defaultModel } : {}), + }); + this.sanitizer = createSanitizer({ + normalize: options.normalize, + redact: options.redact, + defaultModel: provider.defaultModel, + }); + } + + generate(options: GenerateOptions): Promise { + return this.record('generate', options, async () => { + const result = await this.provider.generate(options); + return { result, response: this.generateResponse(result) }; + }); + } + + stream(options: GenerateOptions): Promise { + return this.record('stream', options, async () => { + const recorded = await this.consumeStream(await this.provider.stream(options)); + return { result: replayStream(recorded, false), response: recorded }; + }); + } + + supportsTools(model: string): boolean { + return this.provider.supportsTools(model); + } + + supportsStreaming(model: string): boolean { + return this.provider.supportsStreaming(model); + } + + getModels(): Promise { + return this.provider.getModels(); + } + + async save(): Promise { + this.cassette.entries = this.slots.filter((entry): entry is CassetteEntry => entry !== undefined); + const snapshot = clone(this.cassette); + this.writing = this.writing.catch(() => undefined).then(() => writeCassette(this.options.cassette, snapshot)); + await this.writing; + } + + private async record( + kind: CassetteEntry['kind'], + options: GenerateOptions, + call: () => Promise<{ result: T; response: CassetteResponse }> + ): Promise { + const request = this.sanitizer.request(options); + const slot = this.slots.length; + this.slots.push(undefined); + let outcome: { result: T; response: CassetteResponse }; + try { + outcome = await call(); + } catch (error) { + this.slots[slot] = { kind, request, error: toCassetteError(error, this.sanitizer) }; + await this.save(); + throw error; + } + this.slots[slot] = { kind, request, response: outcome.response }; + await this.save(); + return outcome.result; + } + + private generateResponse(result: GenerateResult): CassetteResponse { + // rawResponse is deliberately not recorded: it can carry headers and ids. + return this.sanitizer.redact({ + text: result.text, + finishReason: result.finishReason, + usage: { + promptTokens: result.usage.promptTokens, + completionTokens: result.usage.completionTokens, + totalTokens: result.usage.totalTokens, + }, + ...(result.toolCalls?.length ? { toolCalls: result.toolCalls } : {}), + }); + } + + private async consumeStream(stream: StreamResult): Promise { + const chunks: NonNullable = []; + let last = performance.now(); + for await (const chunk of stream.fullStream) { + const now = performance.now(); + chunks.push({ delayMs: Math.round(now - last), chunk: serializeChunk(chunk) }); + last = now; + } + const [text, usage, finishReason, toolCalls] = await Promise.all([ + stream.text, + stream.usage, + stream.finishReason, + stream.toolCalls, + ]); + return this.sanitizer.redact({ + text, + finishReason, + usage: { + promptTokens: usage.promptTokens, + completionTokens: usage.completionTokens, + totalTokens: usage.totalTokens, + }, + ...(toolCalls.length ? { toolCalls } : {}), + chunks, + }); + } +} + +class Player implements RecordReplayProvider { + readonly mode = 'replay'; + readonly name: string; + readonly defaultModel: string | undefined; + private readonly cassette: Cassette; + private readonly sanitizer: Sanitizer; + private readonly used = new Set(); + private calls = 0; + + constructor( + provider: LLMProvider | undefined, + private readonly options: RecordReplayOptions + ) { + this.cassette = readCassette(options.cassette); + this.name = provider?.name ?? this.cassette.provider.name; + this.defaultModel = provider?.defaultModel ?? this.cassette.provider.defaultModel; + this.sanitizer = createSanitizer({ + normalize: options.normalize, + redact: options.redact, + defaultModel: this.defaultModel, + }); + } + + async generate(options: GenerateOptions): Promise { + return toGenerateResult(this.take('generate', options)); + } + + async stream(options: GenerateOptions): Promise { + return replayStream(this.take('stream', options), this.options.replayTiming ?? false); + } + + supportsTools(): boolean { + return true; + } + + supportsStreaming(): boolean { + return true; + } + + async getModels(): Promise { + return this.defaultModel ? [this.defaultModel] : []; + } + + async save(): Promise { + // Replay never writes. + } + + private take(kind: CassetteEntry['kind'], options: GenerateOptions): CassetteResponse { + if (options.signal?.aborted) throw options.signal.reason; + const request = this.sanitizer.request(options); + const callNumber = ++this.calls; + const index = + this.options.match === 'request' + ? this.findByRequest(kind, request, callNumber) + : this.findNext(kind, request, callNumber); + this.used.add(index); + const entry = this.cassette.entries[index]; + if (entry.error) throw fromCassetteError(entry.error); + return entry.response as CassetteResponse; + } + + private findNext(kind: CassetteEntry['kind'], request: CassetteRequest, callNumber: number): number { + const index = callNumber - 1; + const entry = this.cassette.entries[index]; + if (!entry) throw this.exhausted(callNumber); + this.assertSame(entry, kind, request, callNumber); + return index; + } + + private findByRequest(kind: CassetteEntry['kind'], request: CassetteRequest, callNumber: number): number { + const key = stableStringify({ kind, request }); + const index = this.cassette.entries.findIndex( + (entry, i) => !this.used.has(i) && stableStringify({ kind: entry.kind, request: entry.request }) === key + ); + if (index >= 0) return index; + const firstUnused = this.cassette.entries.findIndex((_, i) => !this.used.has(i)); + if (firstUnused < 0) throw this.exhausted(callNumber); + this.assertSame(this.cassette.entries[firstUnused], kind, request, callNumber, 'No unused recorded request matches. '); + throw this.exhausted(callNumber); + } + + private assertSame( + entry: CassetteEntry, + kind: CassetteEntry['kind'], + request: CassetteRequest, + callNumber: number, + prefix = '' + ): void { + const diff = firstDifference({ kind: entry.kind, request: entry.request }, { kind, request }); + if (!diff) return; + throw new CassetteMismatchError( + `${prefix}Call #${callNumber} does not match the recorded request in ${this.options.cassette}.\n` + + `First difference at ${diff.path}:\n recorded: ${diff.expected}\n actual: ${diff.actual}\n` + + RERECORD_HINT, + this.options.cassette, + callNumber + ); + } + + private exhausted(callNumber: number): CassetteMismatchError { + const total = this.cassette.entries.length; + return new CassetteMismatchError( + `Call #${callNumber} has no recorded entry: ${this.options.cassette} holds ${total} ` + + `entr${total === 1 ? 'y' : 'ies'} and all were used. The agent now makes more model calls than when ` + + `it was recorded. ${RERECORD_HINT}`, + this.options.cassette, + callNumber + ); + } +} + +function resolveMode(options: RecordReplayOptions): 'record' | 'replay' { + const mode = options.mode ?? 'auto'; + if (mode === 'auto') return fs.existsSync(options.cassette) ? 'replay' : 'record'; + return mode; +} + +/** + * Wrap an `LLMProvider` so its exchanges are recorded to a cassette file and + * later replayed deterministically, with no network and no API key. + * + * Record mode forwards to the real provider and writes the cassette atomically + * (temp file plus rename) after every call, so a test that fails midway leaves + * a valid cassette holding every call made so far; there is no exit hook. + * Provider errors are recorded and replayed as rejections with the same + * name and message. Replay mode never touches the wrapped provider, so pass a + * factory (or `undefined`) and CI needs neither the key nor the peer package. + * + * Cassettes are meant to be committed: API-key-like strings (`sk-...`, + * `sk-ant-...`, bearer tokens) are redacted, `rawResponse` is never stored and + * only whitelisted request fields are written. Use `redact` for anything else + * sensitive in your prompts. + * + * @param provider - The real provider, a `() => LLMProvider` factory (called + * only when recording), or `undefined` when you only ever replay. + * @param options - See {@link RecordReplayOptions}. + * @throws In replay mode, if the cassette is missing, malformed or has the + * wrong version. Replayed calls throw {@link CassetteMismatchError}. + * + * @example + * ```ts + * import { recordReplay } from '@loushy/build-ai-agent/testing'; + * + * const provider = recordReplay(() => resolveProvider('openai/gpt-4o-mini'), { + * cassette: './__cassettes__/refund-flow.json', + * mode: process.env.LOUSHY_RECORD ? 'record' : 'replay', + * }); + * const agent = createAgent({ provider, prompt: 'You handle refunds.' }); + * ``` + */ +export function recordReplay(provider: RecordReplaySource, options: RecordReplayOptions): RecordReplayProvider { + // Replay must never construct the provider: a factory is only ever called when recording. + if (resolveMode(options) === 'replay') return new Player(typeof provider === 'function' ? undefined : provider, options); + const real = typeof provider === 'function' ? provider() : provider; + if (!real) { + throw new Error( + "recordReplay: record mode needs a provider to record from, but none was given. Pass the real provider (or a () => provider factory) as the first argument, or use mode: 'replay'." + ); + } + return new Recorder(real, options); +} From 722a6c5cf55e089777ceb1528ddf38b11a1240a5 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 1 Oct 2026 10:38:22 +0300 Subject: [PATCH 2/2] LOU-D6: record/replay provider for deterministic tests against real model behavior Adds recordReplay() to @loushy/build-ai-agent/testing: a VCR that wraps any LLMProvider, records every generate()/stream() exchange to a versioned JSON cassette and replays it with no network and no API key (lazy factory so the provider is never constructed in replay). Sequential-and-verified or by-request matching, CassetteMismatchError with a first-difference diff, volatile-value normalization, API-key redaction plus a custom redact hook, atomic writes after every recorded call, recorded errors and streams, usage preserved, auto mode. Documented in docs/testing.md. Co-Authored-By: Claude Sonnet 5.5 --- src/testing/fingerprint.ts | 34 +++++++++++++++++++------------- src/testing/recordReplay.test.ts | 12 ++++++----- src/testing/recordReplay.ts | 5 +++-- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/testing/fingerprint.ts b/src/testing/fingerprint.ts index 048cb0888..3bdb6cdc0 100644 --- a/src/testing/fingerprint.ts +++ b/src/testing/fingerprint.ts @@ -87,7 +87,7 @@ function sortKeys(record: Record): Record { * Reduce any value to plain, key-sorted JSON. Zod schemas become a * JSON-schema-ish fingerprint; functions and `undefined` are dropped. */ -export function canonicalize(value: unknown, depth = 0): unknown { +function canonicalize(value: unknown, depth = 0): unknown { if (value === undefined || typeof value === 'function') return undefined; if (typeof value !== 'object' || value === null) return value; if (depth > MAX_DEPTH) return '[max depth]'; @@ -105,7 +105,7 @@ export function stableStringify(value: unknown): string { } /** Apply `fn` to every string inside a JSON-like value (keys are left alone). */ -export function mapStrings(value: T, fn: (text: string) => string): T { +function mapStrings(value: T, fn: (text: string) => string): T { if (typeof value === 'string') return fn(value) as T; if (Array.isArray(value)) return value.map((item) => mapStrings(item, fn)) as T; if (typeof value === 'object' && value !== null) { @@ -232,22 +232,21 @@ function childPaths(a: unknown, b: unknown): Array { return [...keys].sort(); } -function isContainer(value: unknown): boolean { +function isContainer(value: unknown): value is object { return typeof value === 'object' && value !== null; } -/** The first place two canonical JSON values differ, or `undefined` if equal. */ -export function firstDifference(expected: unknown, actual: unknown, path = ''): Difference | undefined { - const bothContainers = isContainer(expected) && isContainer(actual); - if (bothContainers && Array.isArray(expected) === Array.isArray(actual)) { - for (const key of childPaths(expected, actual)) { - const child = (value: unknown): unknown => (value as Record)[key]; - const where = typeof key === 'number' ? `${path}[${key}]` : path ? `${path}.${key}` : key; - const found = firstDifference(child(expected), child(actual), where); - if (found) return found; - } - return undefined; +function childDifference(expected: unknown, actual: unknown, path: string): Difference | undefined { + for (const key of childPaths(expected, actual)) { + const child = (value: unknown): unknown => (value as Record)[key]; + const where = typeof key === 'number' ? `${path}[${key}]` : path ? `${path}.${key}` : key; + const found = firstDifference(child(expected), child(actual), where); + if (found) return found; } + return undefined; +} + +function leafDifference(expected: unknown, actual: unknown, path: string): Difference | undefined { if (JSON.stringify(expected) === JSON.stringify(actual)) return undefined; const [shownExpected, shownActual] = typeof expected === 'string' && typeof actual === 'string' @@ -255,3 +254,10 @@ export function firstDifference(expected: unknown, actual: unknown, path = ''): : [show(expected), show(actual)]; return { path: path || '(root)', expected: shownExpected, actual: shownActual }; } + +/** The first place two canonical JSON values differ, or `undefined` if equal. */ +export function firstDifference(expected: unknown, actual: unknown, path = ''): Difference | undefined { + const comparable = + isContainer(expected) && isContainer(actual) && Array.isArray(expected) === Array.isArray(actual); + return comparable ? childDifference(expected, actual, path) : leafDifference(expected, actual, path); +} diff --git a/src/testing/recordReplay.test.ts b/src/testing/recordReplay.test.ts index 6d12fc37b..cb456497c 100644 --- a/src/testing/recordReplay.test.ts +++ b/src/testing/recordReplay.test.ts @@ -416,15 +416,17 @@ describe('recordReplay stream()', () => { const recorder = recordReplay(slowStream, { cassette: file, mode: 'record' }); await recorder.stream(ask('s')); - const timed = async (replayTiming: boolean): Promise => { - const start = Date.now(); + const delaysRequested = async (replayTiming: boolean): Promise => { + const timers = vi.spyOn(globalThis, 'setTimeout'); const replay = await recordReplay(undefined, { cassette: file, mode: 'replay', replayTiming }).stream(ask('s')); await collect(replay.fullStream); - return Date.now() - start; + const delays = timers.mock.calls.map((call) => Number(call[1])); + timers.mockRestore(); + return delays; }; - expect(await timed(false)).toBeLessThan(40); - expect(await timed(true)).toBeGreaterThanOrEqual(50); + expect(await delaysRequested(false)).toEqual([]); + expect((await delaysRequested(true)).some((ms) => ms >= 50)).toBe(true); }); it('records error chunks', async () => { diff --git a/src/testing/recordReplay.ts b/src/testing/recordReplay.ts index 0c10f70be..3f7aff979 100644 --- a/src/testing/recordReplay.ts +++ b/src/testing/recordReplay.ts @@ -308,8 +308,9 @@ class Player implements RecordReplayProvider { private readonly options: RecordReplayOptions ) { this.cassette = readCassette(options.cassette); - this.name = provider?.name ?? this.cassette.provider.name; - this.defaultModel = provider?.defaultModel ?? this.cassette.provider.defaultModel; + const identity: { name: string; defaultModel?: string } = provider ?? this.cassette.provider; + this.name = identity.name; + this.defaultModel = identity.defaultModel; this.sanitizer = createSanitizer({ normalize: options.normalize, redact: options.redact,