diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a8cb1381..e168ebc7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - 2026-09-28 ### Added +- `loushy eval --record / --replay / --drift` (LOU-D46): `--record` runs every eval case against its agent's real provider through `recordReplay()` and writes one cassette per case next to the eval file (`__cassettes__//.json`, the normal cassette format); `--replay` runs every case from its cassette with no network, and a missing cassette fails the case with the `--record` command to run. Plain `loushy eval` with `CI` set replays the cases that have a cassette; without `CI` it is unchanged. `--drift` re-records into a temp directory and compares each case with its committed cassette (ordered tool names, JSON-normalized tool arguments, step count, finish reason; tokens with `--drift-usage`): the summary gets a `Drift:` table and each drifted case a soft `drift` assertion (a JUnit `` note), or a gate failure with `--strict`. Existing eval files need no change: while a case runs, `AgentExecutor.execute()` wraps its provider for that case. `EvalResult` gains `cassettes`. Fixed on the way: trajectory and classic eval results now carry their `file` (it was read before the test ran and was always empty). See docs/evals.md#record-replay-and-drift. - OpenTelemetry GenAI metrics and cost (LOU-D48): `createOtelTraceExporter()` (`@loushy/build-ai-agent/otel`) now also records the GenAI client metrics `gen_ai.client.token.usage` (histogram, `{token}`, one record per `gen_ai.token.type` `input`/`output`) and `gen_ai.client.operation.duration` (histogram, `s`, for every model call and tool call) through `@opentelemetry/api`'s metrics API: the global `MeterProvider`, or the new `meter` option; `metrics: false` turns them off. Without a registered `MeterProvider` they are discarded by OpenTelemetry's no-op meter. Spans gain `loushy.cost_usd` (estimated USD: per step on `chat` spans, cumulative on `invoke_agent` spans, absent when the price table does not know a model) and `loushy.usage.estimated` on `invoke_agent` spans (it was already on `chat` spans). Any `TraceExporter` sees the cumulative attributes: `withSpan()` sums the cost of finished child spans onto their parent. See docs/observability.md. - One `store` option (LOU-D30): `createAgent({ store })` takes an `AgentStore` (`{ sessions?, checkpoints?, approvals? }`, new export) and wires all three from it. `SqliteStore` is one; the new `memoryStore()` returns in-memory sessions, checkpoints and approvals, and docs/sessions.md shows the file-based combination (`FileSessionStore`, `LocalStorageCheckpointStore`, `StorageServiceApprovalStore`). `agent.session({ id })` then keeps its transcript in `store.sessions` and checkpoints every turn in `store.checkpoints`; `store.approvals` is the default `approvalStore`. `SendOptions.sessionId` makes `send()` / `stream()` a durable run checkpointed in `store.checkpoints` (an error without one), so a one-shot durable job needs no `AgentExecutor`, and `agent.approvals.resolve()` keeps checkpointing it. The new `agent.resume(sessionId)` finishes an interrupted `send(..., { sessionId })` run, or else the session's pending turn (as `session.resume()`), and returns `null` when nothing is pending. A session's own `store` / `checkpointStore` and an explicit `approvalStore` still win, part by part. Without `store`, nothing changes. - One tool-error shape (LOU-U14): every failed tool call reaches the model as `{ error, toolName, message, kind }` (message capped at 2,000 characters, transcript message flagged `isError`), whichever path failed: `kind` is `'execution'`, `'validation'` (plus `issues`), `'not-found'`, `'rejected'` (plus `note`), `'not-run'`, `'mcp'` or `'sandbox'`. New exports `toolErrorResult()`, `ToolErrorResult`, `ToolErrorKind` and `ToolErrorInput`; `McpToolError` and the sandbox guard's error (named `SandboxRequiredError`, not exported) carry a `toolErrorKind`. `kind` is additive on thrown-error and validation results. The paths that built their own shapes changed, see Changed. See docs/tools.md. diff --git a/README.md b/README.md index 6a17500d8..fdea56a2b 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ agent can be an `agent.yaml` spec served with `npx loushy dev agent.yaml` - **Hooks, guardrails, sandboxing**: veto tool calls, gate a patch on fail-closed checks, run tools in Docker. [Guardrails](docs/guardrails.md) - **Flows and triggers**: fixed multi-step workflows; webhook, Slack and cron adapters. [Flows](docs/flows.md), [Triggers](docs/api-overview.md#triggers) - **Tracing**: OpenTelemetry GenAI spans (`invoke_agent`, `chat`, `execute_tool`); content capture is opt-in. [Observability](docs/observability.md) -- **Testing and evals**: `mockModel`, `recordReplay` cassettes, `defineEval()` trajectory assertions, `loushy eval`. [Testing](docs/testing.md), [Evals](docs/evals.md) +- **Testing and evals**: `mockModel`, `recordReplay` cassettes, `defineEval()` trajectory assertions, `loushy eval` with `--record` / `--replay` cassettes and `--drift` trajectory diffs. [Testing](docs/testing.md), [Evals](docs/evals.md) - **CLI**: `init`, `doctor`, `dev`, `mcp`, `eval`, `build` and `studio`. [CLI](docs/cli.md) - **Agent Forge**: `loushy studio` opens a visual canvas, run debugger and chat with approval cards. [Agent Forge](docs/agent-forge.md) diff --git a/docs/cli.md b/docs/cli.md index 18e401b85..3f0f0fb1f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -27,7 +27,7 @@ loushy build --target= --agent= [--out=] loushy studio [--port N] [--host H] [--prod|--dev] loushy mcp [--http --port N --host H] loushy doctor [agent.yaml|json] [--json] -loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] +loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] ``` `npm create loushy-agent my-agent` runs `loushy init` with the same arguments. diff --git a/docs/evals.md b/docs/evals.md index 462c95e9a..30d2e136d 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -167,7 +167,7 @@ repository) runs them. ## `loushy eval` ```text -loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] +loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] ``` | Option | Meaning | @@ -179,6 +179,9 @@ loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judg | `--strict` | soft failures fail the run | | `--judge` | run `*.judge.eval.ts` files instead of the normal ones | | `--config path` | use your own vitest config instead of the generated one | +| `--record` | run against the real provider and write one cassette per case ([below](#record-replay-and-drift)) | +| `--replay` | run every case from its cassette, with no network; a missing cassette fails the case | +| `--drift` | re-record into a temp directory and report how each case's trajectory changed (`--drift-usage` also compares tokens) | It spawns the vitest installed in your project (vitest is *your* dependency; if it is missing the command prints `npm install --save-dev vitest` and exits 2), @@ -229,3 +232,56 @@ jobs: Run `--tag smoke` on every pull request and the full set nightly; run `--judge` only where you accept real LLM calls and their cost. + +## Record, replay and drift + +An eval against a real model is slow, costs money and needs a key; the same +eval on `mockModel` only tests the script you wrote. `loushy eval` sits in +between: it records each case once against the real provider through +[`recordReplay`](testing.md#record-and-replay) and replays the recording in CI. +Your eval files do not change. + +```bash +npx loushy eval --record # real provider: writes the cassettes, commit them +npx loushy eval --replay # every case from its cassette, no network, no key +npx loushy eval --drift # re-record and diff each case's trajectory +``` + +- **`--record`** runs every case with its agent's real provider and writes one + cassette per case next to the eval file: + `__cassettes__//.json` (names are slugged, for example + `refund-flow/polite.json`; an eval without `cases` writes `default.json`). + Give each case a unique `label` so the names stay stable. A case whose agent + uses more than one provider (a sub-agent on another model) gets + `.2.json` and so on. The files use the normal cassette format, with + API keys redacted; review and commit them. +- **`--replay`** never calls the model. A case with no cassette fails with + `no cassette for " []" at ...` and the `--record` command to run; + when the agent's requests changed, the replayed model call fails with a + `CassetteMismatchError` naming the first difference. Plain + `loushy eval` with `CI` set replays every case that has a cassette and runs the + rest live; without `CI` it runs live as before. +- **`--drift`** re-records every case into a temp directory (the committed + cassettes are not touched) and compares each with its committed cassette: + the ordered tool names, each call's arguments (JSON-normalized, so key order + does not count), the step count and the finish reason. Token usage changes + on every real run, so it is compared only with `--drift-usage`. The summary + gets a `Drift:` table (eval, case, field, committed, current); each drifted + case gets a soft `drift` assertion, so it shows as `PASS (soft fail)` and as a + `` note in the JUnit report. With `--strict`, drift is a gate + failure: the case fails, JUnit has a ``, and the run exits 1. + +```text +Drift: +EVAL CASE FIELD COMMITTED CURRENT +refund flow polite args lookup_order {"orderId":"42"} lookup_order {"orderId":"43"} +``` + +Run `--drift` nightly or before a model upgrade, and `--replay` (or plain +`loushy eval` in CI) on every pull request. Judges are not recorded: grade with +a real judge only in `*.judge.eval.ts` files. + +How it works: while a case runs under one of these modes, every +`AgentExecutor.execute()` call (which `createAgent()` agents, streaming, +sub-agents and approval resumes go through) gets its provider wrapped with +`recordReplay()` for that case's cassette. diff --git a/docs/testing.md b/docs/testing.md index 074b09d4b..4ad34aa1d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -188,6 +188,11 @@ only called when recording (so replay never constructs it). It may be `mode: 'auto'` replays if the cassette file exists and records otherwise. +For `defineEval()` evals you do not need to wrap the provider yourself: +`loushy eval --record` writes one cassette per eval case, `--replay` runs from +them, and `--drift` reports how each case's trajectory changed. See +[Record, replay and drift](evals.md#record-replay-and-drift). + ### What replay checks By default call N is answered by entry N, but only if the request still matches diff --git a/llms-full.txt b/llms-full.txt index 92e0e44aa..f1166bd3a 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -86,7 +86,7 @@ agent can be an `agent.yaml` spec served with `npx loushy dev agent.yaml` - **Hooks, guardrails, sandboxing**: veto tool calls, gate a patch on fail-closed checks, run tools in Docker. [Guardrails](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/guardrails.md) - **Flows and triggers**: fixed multi-step workflows; webhook, Slack and cron adapters. [Flows](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/flows.md), [Triggers](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/api-overview.md#triggers) - **Tracing**: OpenTelemetry GenAI spans (`invoke_agent`, `chat`, `execute_tool`); content capture is opt-in. [Observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md) -- **Testing and evals**: `mockModel`, `recordReplay` cassettes, `defineEval()` trajectory assertions, `loushy eval`. [Testing](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md), [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md) +- **Testing and evals**: `mockModel`, `recordReplay` cassettes, `defineEval()` trajectory assertions, `loushy eval` with `--record` / `--replay` cassettes and `--drift` trajectory diffs. [Testing](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md), [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md) - **CLI**: `init`, `doctor`, `dev`, `mcp`, `eval`, `build` and `studio`. [CLI](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/cli.md) - **Agent Forge**: `loushy studio` opens a visual canvas, run debugger and chat with approval cards. [Agent Forge](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/agent-forge.md) @@ -1988,7 +1988,7 @@ loushy build --target= --agent= [--out=] loushy studio [--port N] [--host H] [--prod|--dev] loushy mcp [--http --port N --host H] loushy doctor [agent.yaml|json] [--json] -loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] +loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] ``` `npm create loushy-agent my-agent` runs `loushy init` with the same arguments. @@ -3140,7 +3140,7 @@ repository) runs them. ## `loushy eval` ```text -loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] +loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] ``` | Option | Meaning | @@ -3152,6 +3152,9 @@ loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judg | `--strict` | soft failures fail the run | | `--judge` | run `*.judge.eval.ts` files instead of the normal ones | | `--config path` | use your own vitest config instead of the generated one | +| `--record` | run against the real provider and write one cassette per case ([below](#record-replay-and-drift)) | +| `--replay` | run every case from its cassette, with no network; a missing cassette fails the case | +| `--drift` | re-record into a temp directory and report how each case's trajectory changed (`--drift-usage` also compares tokens) | It spawns the vitest installed in your project (vitest is *your* dependency; if it is missing the command prints `npm install --save-dev vitest` and exits 2), @@ -3203,6 +3206,59 @@ jobs: Run `--tag smoke` on every pull request and the full set nightly; run `--judge` only where you accept real LLM calls and their cost. +## Record, replay and drift + +An eval against a real model is slow, costs money and needs a key; the same +eval on `mockModel` only tests the script you wrote. `loushy eval` sits in +between: it records each case once against the real provider through +[`recordReplay`](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md#record-and-replay) and replays the recording in CI. +Your eval files do not change. + +```bash +npx loushy eval --record # real provider: writes the cassettes, commit them +npx loushy eval --replay # every case from its cassette, no network, no key +npx loushy eval --drift # re-record and diff each case's trajectory +``` + +- **`--record`** runs every case with its agent's real provider and writes one + cassette per case next to the eval file: + `__cassettes__//.json` (names are slugged, for example + `refund-flow/polite.json`; an eval without `cases` writes `default.json`). + Give each case a unique `label` so the names stay stable. A case whose agent + uses more than one provider (a sub-agent on another model) gets + `.2.json` and so on. The files use the normal cassette format, with + API keys redacted; review and commit them. +- **`--replay`** never calls the model. A case with no cassette fails with + `no cassette for " []" at ...` and the `--record` command to run; + when the agent's requests changed, the replayed model call fails with a + `CassetteMismatchError` naming the first difference. Plain + `loushy eval` with `CI` set replays every case that has a cassette and runs the + rest live; without `CI` it runs live as before. +- **`--drift`** re-records every case into a temp directory (the committed + cassettes are not touched) and compares each with its committed cassette: + the ordered tool names, each call's arguments (JSON-normalized, so key order + does not count), the step count and the finish reason. Token usage changes + on every real run, so it is compared only with `--drift-usage`. The summary + gets a `Drift:` table (eval, case, field, committed, current); each drifted + case gets a soft `drift` assertion, so it shows as `PASS (soft fail)` and as a + `` note in the JUnit report. With `--strict`, drift is a gate + failure: the case fails, JUnit has a ``, and the run exits 1. + +```text +Drift: +EVAL CASE FIELD COMMITTED CURRENT +refund flow polite args lookup_order {"orderId":"42"} lookup_order {"orderId":"43"} +``` + +Run `--drift` nightly or before a model upgrade, and `--replay` (or plain +`loushy eval` in CI) on every pull request. Judges are not recorded: grade with +a real judge only in `*.judge.eval.ts` files. + +How it works: while a case runs under one of these modes, every +`AgentExecutor.execute()` call (which `createAgent()` agents, streaming, +sub-agents and approval resumes go through) gets its provider wrapped with +`recordReplay()` for that case's cassette. + # Flows Source: docs/flows.md @@ -5226,6 +5282,11 @@ only called when recording (so replay never constructs it). It may be `mode: 'auto'` replays if the cassette file exists and records otherwise. +For `defineEval()` evals you do not need to wrap the provider yourself: +`loushy eval --record` writes one cassette per eval case, `--replay` runs from +them, and `--drift` reports how each case's trajectory changed. See +[Record, replay and drift](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md#record-replay-and-drift). + ### What replay checks By default call N is answered by entry N, but only if the request still matches diff --git a/src/cli/__fixtures__/eval/replay.eval.ts b/src/cli/__fixtures__/eval/replay.eval.ts new file mode 100644 index 000000000..67c72a194 --- /dev/null +++ b/src/cli/__fixtures__/eval/replay.eval.ts @@ -0,0 +1,39 @@ +/** + * Fixture for the `--record` / `--replay` / `--drift` tests in + * src/cli/eval.test.ts (LOU-D46). The "real" provider is a mockModel set up + * from env vars: FIXTURE_PROVIDER=offline makes every model call fail (so + * replay provably never calls it) and FIXTURE_ORDER_ID changes the tool + * arguments (drift). Its cassettes are written to __cassettes__/ next to this + * file and deleted by the test. + */ +import { z } from 'zod'; +import { createAgent } from '../../../createAgent'; +import { defineEval } from '../../../evals'; +import { defineTool } from '../../../tools/defineTool'; +import { mockModel } from '../../../testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'shipped' }), +}); + +const realProvider = () => + process.env.FIXTURE_PROVIDER === 'offline' + ? mockModel([]) + : mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: process.env.FIXTURE_ORDER_ID ?? '42' } }] }, + { text: 'Refunds are possible within 30 days.' }, + ]); + +defineEval({ + name: 'recorded refund', + agent: () => createAgent({ prompt: 'You handle refunds.', tools: [lookupOrder], provider: realProvider() }), + cases: [{ input: 'Refund order 42', label: 'plain' }], + async test(t, c) { + await t.send(c.input); + t.completed(); + t.calledTool('lookup_order'); + }, +}); diff --git a/src/cli/eval.test.ts b/src/cli/eval.test.ts index ea70f0b32..288f55d93 100644 --- a/src/cli/eval.test.ts +++ b/src/cli/eval.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, afterEach } from 'vitest'; +import { describe, it, expect, vi, afterEach, beforeAll, afterAll } from 'vitest'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; @@ -43,6 +43,13 @@ describe('parseEvalArgs', () => { }); }); + it('reads the cassette flags; --drift-usage implies --drift and only one mode is allowed', () => { + expect(parseEvalArgs(['--record'])).toMatchObject({ record: true }); + expect(parseEvalArgs(['--drift-usage'])).toMatchObject({ drift: true, driftUsage: true }); + expect(() => parseEvalArgs(['--record', '--replay'])).toThrow(/only one of --record, --replay and --drift/); + expect(() => parseEvalArgs(['--replay', '--drift'])).toThrow(/only one of/); + }); + it('explains how to fix an unknown option or a missing value', () => { expect(() => parseEvalArgs(['--bogus'])).toThrow(/unknown option '--bogus'.*Usage: loushy eval/); expect(() => parseEvalArgs(['--junit'])).toThrow(/--junit needs a value/); @@ -219,3 +226,63 @@ describe('loushy eval end to end (real vitest, mockModel fixtures)', () => { ); }, 60_000); }); + +describe('loushy eval --record / --replay / --drift (real vitest, mockModel as the "real" provider)', () => { + const fixture = path.join(FIXTURES, 'replay.eval.ts'); + const cassettes = path.join(FIXTURES, '__cassettes__'); + const cassette = path.join(cassettes, 'recorded-refund', 'plain.json'); + const quietVitest = createVitestSpawner('ignore'); + const cli = (args: string[]) => { + const log = vi.fn(); + return runEval(args, { log, spawnVitest: quietVitest }).then((code) => ({ code, output: log.mock.calls.map((c) => String(c[0])).join('\n') })); + }; + const clean = () => fs.rmSync(cassettes, { recursive: true, force: true }); + + beforeAll(clean); + afterAll(clean); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('--replay without a cassette fails the case and names the --record command', async () => { + const { code, output } = await cli([fixture, '--replay']); + expect(code).toBe(1); + expect(output).toContain('no cassette for "recorded refund [plain]"'); + expect(output).toContain(`npx loushy eval --record ${path.relative(process.cwd(), fixture)}`); + }, 60_000); + + it('--record writes one cassette per case; --replay (or a CI run) replays it with no provider call', async () => { + expect((await cli([fixture, '--record'])).code).toBe(0); + expect((JSON.parse(fs.readFileSync(cassette, 'utf8')) as { entries: unknown[] }).entries).toHaveLength(2); + + vi.stubEnv('FIXTURE_PROVIDER', 'offline'); + const json = tempFile('results.json'); + expect((await cli([fixture, '--replay', '--json', json])).code).toBe(0); + const parsed = JSON.parse(fs.readFileSync(json, 'utf8')) as { results: EvalResult[] }; + expect(parsed.results[0]).toMatchObject({ passed: true, cassettes: [cassette], toolCalls: [{ name: 'lookup_order', args: { orderId: '42' } }] }); + + vi.stubEnv('CI', 'true'); + expect((await cli([fixture])).code).toBe(0); + // A plain local run keeps today's behaviour: live, so the offline provider fails it. + vi.stubEnv('CI', ''); + expect((await cli([fixture])).code).toBe(1); + }, 120_000); + + it('--drift reports a per-case diff table and JUnit entries, failing only with --strict', async () => { + const same = await cli([fixture, '--drift']); + expect(same.code).toBe(0); + expect(same.output).toContain('Drift: none'); + + vi.stubEnv('FIXTURE_ORDER_ID', '43'); + const junit = tempFile('junit.xml'); + const drifted = await cli([fixture, '--drift', '--junit', junit]); + expect(drifted.code).toBe(0); + expect(drifted.output).toMatch(/recorded refund\s+plain\s+args\s+lookup_order \{"orderId":"42"\}\s+lookup_order \{"orderId":"43"\}/); + expect(fs.readFileSync(junit, 'utf8')).toContain('soft failure: drift from plain.json: args lookup_order'); + expect(fs.readFileSync(cassette, 'utf8')).toContain('\\"orderId\\":\\"42\\"'); + + const strict = await cli([fixture, '--drift', '--strict', '--junit', junit]); + expect(strict.code).toBe(1); + expect(fs.readFileSync(junit, 'utf8')).toContain('): Promise<{ result: EvalResult; reply: string }> { + let reply = ''; + const result = await withEvalCassettes({ file, name: 'Refund flow', label: 'Order #42' }, async () => { + reply = (await createAgent({ provider, prompt: 'You handle refunds.' }).send('Refund 42')).text; + return { name: 'Refund flow', case: 'Order #42', tags: [], passed: true, assertions: [], durationMs: 0, steps: 1, toolCalls: [] }; + }); + return { result, reply }; +} + +describe('cassettePath', () => { + it('names cassettes deterministically next to the eval file', () => { + expect(cassettePath('/p/evals/refund.eval.ts', 'Refund flow', 'Order #42')).toBe( + path.join('/p/evals', '__cassettes__', 'refund-flow', 'order-42.json') + ); + expect(cassettePath('/p/a.eval.ts', 'e', undefined, 1)).toBe(path.join('/p', '__cassettes__', 'e', 'default.2.json')); + }); + + it('maps a committed cassette to a stable drift location', () => { + expect(driftCassettePath('/tmp/d', '/p/x.json')).toBe(driftCassettePath('/tmp/d', '/p/x.json')); + expect(driftCassettePath('/tmp/d', '/p/x.json')).not.toBe(driftCassettePath('/tmp/d', '/p/y.json')); + }); +}); + +describe('withEvalCassettes', () => { + it('runs the case untouched when loushy eval set no mode', async () => { + const file = evalFile(); + const { result, reply } = await runCase(file, mockModel(['live'])); + expect(reply).toBe('live'); + expect(result.cassettes).toBeUndefined(); + expect(fs.existsSync(path.join(path.dirname(file), '__cassettes__'))).toBe(false); + }); + + it('records the case, then replays it without calling the provider', async () => { + const file = evalFile(); + vi.stubEnv(CASSETTES_ENV, 'record'); + const recorded = await runCase(file, mockModel(['recorded reply'])); + const cassette = cassettePath(file, 'Refund flow', 'Order #42'); + expect(recorded.result.cassettes).toEqual([cassette]); + expect(fs.existsSync(cassette)).toBe(true); + + vi.stubEnv(CASSETTES_ENV, 'replay'); + const replayed = await runCase(file, mockModel([])); + expect(replayed.reply).toBe('recorded reply'); + }); + + it('fails a replay without a cassette with the --record command, and runs live in auto mode', async () => { + const file = evalFile(); + vi.stubEnv(CASSETTES_ENV, 'replay'); + await expect(runCase(file, mockModel(['x']))).rejects.toThrow(/no cassette for "Refund flow \[Order #42\]".*npx loushy eval --record/); + vi.stubEnv(CASSETTES_ENV, 'auto'); + expect((await runCase(file, mockModel(['live']))).reply).toBe('live'); + }); + + it('records into the drift dir instead of over the committed cassette', async () => { + const file = evalFile(); + const driftDir = fs.mkdtempSync(path.join(os.tmpdir(), 'loushy-drift-')); + vi.stubEnv(CASSETTES_ENV, 'record'); + vi.stubEnv(DRIFT_DIR_ENV, driftDir); + const { result } = await runCase(file, mockModel(['again'])); + const committed = cassettePath(file, 'Refund flow', 'Order #42'); + expect(result.cassettes).toEqual([committed]); + expect(fs.existsSync(committed)).toBe(false); + expect(fs.existsSync(driftCassettePath(driftDir, committed))).toBe(true); + }); +}); diff --git a/src/evals/cassettes.ts b/src/evals/cassettes.ts new file mode 100644 index 000000000..911c15111 --- /dev/null +++ b/src/evals/cassettes.ts @@ -0,0 +1,115 @@ +/** + * Record/replay for `loushy eval` (LOU-D46). When the CLI sets + * `LOUSHY_EVAL_CASSETTES`, every model call an eval case makes goes through + * `recordReplay()`, one cassette per case and provider, committed next to the + * eval file at `__cassettes__//.json`. + * + * The hook: while a case runs, `AgentExecutor.execute()` - which + * `createAgent()` agents, `AgentExecutor.stream()`, sub-agents and approval + * resumes all go through - swaps `options.provider` for the case's wrapper. + * The running case is found with AsyncLocalStorage, so eval files need no + * change and concurrent cases never share a cassette. + */ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { AgentExecutor, type ExecuteOptions } from '../execution/AgentExecutor'; +import type { LLMProvider } from '../providers/llm'; +import { recordReplay } from '../testing/recordReplay'; +import type { EvalResult } from './evalResult'; + +/** `record`, `replay`, or `auto` (replay a case whose cassette exists, run the rest live). */ +export const CASSETTES_ENV = 'LOUSHY_EVAL_CASSETTES'; +/** With `record`: write cassettes here (`--drift`) instead of next to the eval file. */ +export const DRIFT_DIR_ENV = 'LOUSHY_EVAL_DRIFT_DIR'; + +type Mode = 'record' | 'replay' | 'auto'; + +interface CaseRun { + file: string; + name: string; + label?: string; + mode: Mode; + driftDir?: string; + /** Real provider -> the wrapper this case uses for it. */ + wrappers: Map; + /** Committed cassette paths the case recorded or replayed. */ + used: string[]; +} + +const activeCase = new AsyncLocalStorage(); +const wrappers = new WeakSet(); +let installed = false; + +function slug(text: string): string { + return text.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 60) || 'case'; +} + +/** The committed cassette for a case's `index`-th (0-based) provider: `.json`, then `.2.json`, ... */ +export function cassettePath(evalFile: string, evalName: string, caseLabel: string | undefined, index = 0): string { + const base = slug(caseLabel ?? 'default') + (index > 0 ? `.${index + 1}` : ''); + return path.join(path.dirname(evalFile), '__cassettes__', slug(evalName), `${base}.json`); +} + +/** Where `--drift` re-records the case whose committed cassette is `committed`. */ +export function driftCassettePath(driftDir: string, committed: string): string { + return path.join(driftDir, `${createHash('sha256').update(path.resolve(committed)).digest('hex').slice(0, 16)}.json`); +} + +function wrapperFor(run: CaseRun, provider: LLMProvider): LLMProvider { + if (wrappers.has(provider)) return provider; + const existing = run.wrappers.get(provider); + if (existing) return existing; + const committed = cassettePath(run.file, run.name, run.label, run.wrappers.size); + const cassette = run.driftDir ? driftCassettePath(run.driftDir, committed) : committed; + const exists = fs.existsSync(cassette); + if (run.mode === 'replay' && !exists) { + const label = run.label ? `${run.name} [${run.label}]` : run.name; + throw new Error( + `loushy eval --replay: no cassette for "${label}" at ${path.relative(process.cwd(), cassette)}. ` + + `Record it with: npx loushy eval --record ${path.relative(process.cwd(), run.file)}` + ); + } + let wrapper = provider; + if (run.mode === 'record' || exists) { + wrapper = recordReplay(provider, { cassette, mode: run.mode === 'record' ? 'record' : 'replay' }); + wrappers.add(wrapper); + run.used.push(committed); + } + run.wrappers.set(provider, wrapper); + return wrapper; +} + +function installHook(): void { + if (installed) return; + installed = true; + const execute = AgentExecutor.execute.bind(AgentExecutor); + AgentExecutor.execute = async (options: ExecuteOptions) => { + const run = activeCase.getStore(); + return execute(run && options.provider ? { ...options, provider: wrapperFor(run, options.provider) } : options); + }; +} + +function cassetteMode(): Mode | undefined { + const mode = process.env[CASSETTES_ENV]; + return mode === 'record' || mode === 'replay' || mode === 'auto' ? mode : undefined; +} + +/** + * Runs one eval case under the cassette mode `loushy eval` asked for and + * notes the cassettes it used on the result. Without a mode (a plain + * `vitest run`) it just runs `runCase`. + */ +export async function withEvalCassettes( + info: { file: string | undefined; name: string; label?: string }, + runCase: () => Promise +): Promise { + const mode = cassetteMode(); + if (!mode || !info.file) return runCase(); + installHook(); + const driftDir = process.env[DRIFT_DIR_ENV] || undefined; + const run: CaseRun = { ...info, file: info.file, mode, driftDir, wrappers: new Map(), used: [] }; + const result = await activeCase.run(run, runCase); + return run.used.length > 0 ? { ...result, cassettes: run.used } : result; +} diff --git a/src/evals/defineEval.ts b/src/evals/defineEval.ts index 3ee9ac2ef..521e62a95 100644 --- a/src/evals/defineEval.ts +++ b/src/evals/defineEval.ts @@ -11,6 +11,7 @@ import type * as Vitest from 'vitest'; import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../execution/AgentExecutor'; +import { withEvalCassettes } from './cassettes'; import { scoreAssertion } from './evalResult'; import { matchesTagFilter, recordEvalResult } from './recorder'; import { parseToolCalls } from './toolMatch'; @@ -123,22 +124,23 @@ async function runAndReport( label: string | undefined, file: string | undefined ): Promise { - const result = await runTrajectoryCase(config, c, label, file); + const result = await withEvalCassettes({ file, name: config.name, label }, () => runTrajectoryCase(config, c, label, file)); recordEvalResult(result); if (!result.passed) throw new Error(describeFailure(result)); } function defineTrajectoryEval(config: TrajectoryEvalConfig): void { const { test, expect } = currentVitest(); - const file = currentTestPath(expect); + // expect.getState().testPath is only set while a test runs, not at collection. + const file = () => currentTestPath(expect); const register = matchesTagFilter(config.tags ?? []) ? test : test.skip; if (config.cases === undefined) { - register(config.name, () => runAndReport(config, NO_CASE, undefined, file)); + register(config.name, () => runAndReport(config, NO_CASE, undefined, file())); return; } config.cases.forEach((c, index) => { const label = caseLabel(c, index); - register(`${config.name} [${label}]`, () => runAndReport(config, c, label, file)); + register(`${config.name} [${label}]`, () => runAndReport(config, c, label, file())); }); } @@ -169,36 +171,37 @@ function currentVitest(): Pick { function defineClassicEval(config: EvalConfig): void { const { name, score, threshold, tags, ...executeFields } = config; const { test, expect } = currentVitest(); - const file = currentTestPath(expect); const register = matchesTagFilter(tags ?? []) ? test : test.skip; register(name, async () => { - const started = Date.now(); - const result = await AgentExecutor.execute({ - agent: executeFields.agent, - input: executeFields.input, - provider: executeFields.provider, - toolRegistry: executeFields.toolRegistry, - maxSteps: executeFields.maxSteps, - temperature: executeFields.temperature, - maxTokens: executeFields.maxTokens, + const file = currentTestPath(expect); + const evalResult = await withEvalCassettes({ file, name }, async () => { + const started = Date.now(); + const result = await AgentExecutor.execute({ + agent: executeFields.agent, + input: executeFields.input, + provider: executeFields.provider, + toolRegistry: executeFields.toolRegistry, + maxSteps: executeFields.maxSteps, + temperature: executeFields.temperature, + maxTokens: executeFields.maxTokens, + }); + const assertion = scoreAssertion(await score(result), threshold); + return { + name, + tags: tags ?? [], + passed: assertion.passed, + assertions: [assertion], + durationMs: Date.now() - started, + steps: result.steps, + toolCalls: parseToolCalls(result.toolCalls ?? []), + usage: result.usage, + file, + }; }); + recordEvalResult(evalResult); - const resultScore = await score(result); - const assertion = scoreAssertion(resultScore, threshold); - recordEvalResult({ - name, - tags: tags ?? [], - passed: assertion.passed, - assertions: [assertion], - durationMs: Date.now() - started, - steps: result.steps, - toolCalls: parseToolCalls(result.toolCalls ?? []), - usage: result.usage, - file, - }); - - expect(resultScore).toBeGreaterThanOrEqual(threshold); + expect(evalResult.assertions[0].score).toBeGreaterThanOrEqual(threshold); }); } diff --git a/src/evals/drift.test.ts b/src/evals/drift.test.ts new file mode 100644 index 000000000..ef1945637 --- /dev/null +++ b/src/evals/drift.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect } from 'vitest'; +import type { Cassette } from '../testing/cassette'; +import { diffTrajectories, trajectoryOf, type Trajectory } from './drift'; + +type Entry = Cassette['entries'][number]; + +const request: Entry['request'] = { model: 'm', messages: [], tools: [], temperature: null, maxTokens: null }; + +function toolStep(calls: Array<[string, string]>, totalTokens = 10): Entry { + return { + kind: 'generate', + request, + response: { + text: '', + finishReason: 'tool_calls', + usage: { promptTokens: totalTokens, completionTokens: 0, totalTokens }, + toolCalls: calls.map(([name, args], i) => ({ id: `c${i}`, type: 'function', function: { name, arguments: args } })), + }, + }; +} + +const stop = (totalTokens = 5): Entry => ({ + kind: 'generate', + request, + response: { text: 'done', finishReason: 'stop', usage: { promptTokens: totalTokens, completionTokens: 0, totalTokens } }, +}); + +describe('trajectoryOf', () => { + it('reads ordered tool calls with key-sorted args, steps, finish reason and tokens', () => { + const trajectory = trajectoryOf({ entries: [toolStep([['lookup', '{"b":2,"a":1}'], ['refund', 'not json']]), stop()] }); + expect(trajectory).toEqual({ + tools: [ + { name: 'lookup', args: '{"a":1,"b":2}' }, + { name: 'refund', args: 'not json' }, + ], + steps: 2, + finishReason: 'stop', + totalTokens: 15, + }); + }); + + it('uses the error name when the last call failed and omits tokens without usage', () => { + const trajectory = trajectoryOf({ entries: [{ kind: 'generate', request, error: { name: 'RateLimitError', message: 'slow down' } }] }); + expect(trajectory).toEqual({ tools: [], steps: 1, finishReason: 'RateLimitError' }); + }); +}); + +describe('diffTrajectories', () => { + const base: Trajectory = trajectoryOf({ entries: [toolStep([['lookup', '{"id":"42"}'], ['refund', '{"id":"42"}']]), stop()] }); + + it('reports no drift for the same trajectory, whatever the args key order', () => { + const same = trajectoryOf({ entries: [toolStep([['lookup', '{ "id": "42" }'], ['refund', '{"id":"42"}']]), stop()] }); + expect(diffTrajectories(base, same)).toEqual([]); + }); + + it('reports tool order drift (and not per-call args drift on top of it)', () => { + const swapped = trajectoryOf({ entries: [toolStep([['refund', '{"id":"42"}'], ['lookup', '{"id":"42"}']]), stop()] }); + expect(diffTrajectories(base, swapped)).toEqual([{ field: 'tools', committed: 'lookup > refund', current: 'refund > lookup' }]); + }); + + it('reports args drift per call', () => { + const changed = trajectoryOf({ entries: [toolStep([['lookup', '{"id":"43"}'], ['refund', '{"id":"42"}']]), stop()] }); + expect(diffTrajectories(base, changed)).toEqual([{ field: 'args', committed: 'lookup {"id":"42"}', current: 'lookup {"id":"43"}' }]); + }); + + it('reports step count and finish reason drift', () => { + const cut = trajectoryOf({ entries: [toolStep([['lookup', '{"id":"42"}'], ['refund', '{"id":"42"}']])] }); + expect(diffTrajectories(base, cut)).toEqual([ + { field: 'steps', committed: '2', current: '1' }, + { field: 'finishReason', committed: 'stop', current: 'tool_calls' }, + ]); + }); + + it('ignores token (cost) drift unless asked for it', () => { + const pricier = trajectoryOf({ entries: [toolStep([['lookup', '{"id":"42"}'], ['refund', '{"id":"42"}']], 900), stop()] }); + expect(diffTrajectories(base, pricier)).toEqual([]); + expect(diffTrajectories(base, pricier, { usage: true })).toEqual([{ field: 'tokens', committed: '15', current: '905' }]); + }); +}); diff --git a/src/evals/drift.ts b/src/evals/drift.ts new file mode 100644 index 000000000..6124f0339 --- /dev/null +++ b/src/evals/drift.ts @@ -0,0 +1,72 @@ +/** + * Trajectory drift (LOU-D46): what an eval case did, read from its cassette, + * and how two such trajectories differ. Pure, so `loushy eval --drift` can + * compare a committed cassette with a fresh recording of the same case. + */ +import type { Cassette } from '../testing/cassette'; +import { stableStringify } from '../testing/fingerprint'; + +/** What a recorded case did. */ +export interface Trajectory { + /** Tool calls in order; `args` is the JSON-normalized (key-sorted) argument string. */ + tools: { name: string; args: string }[]; + /** Model calls (steps). */ + steps: number; + /** Finish reason of the last model call (or the name of the error it threw). */ + finishReason?: string; + /** Total tokens, when the provider reported usage. */ + totalTokens?: number; +} + +/** One difference between a committed and a current trajectory. */ +export interface DriftEntry { + field: 'tools' | 'args' | 'steps' | 'finishReason' | 'tokens'; + committed: string; + current: string; +} + +function normalizeArgs(raw: string): string { + try { + return stableStringify(JSON.parse(raw)); + } catch { + return raw; + } +} + +/** The trajectory a cassette recorded. */ +export function trajectoryOf(cassette: Pick): Trajectory { + const { entries } = cassette; + const tools = entries.flatMap((entry) => + (entry.response?.toolCalls ?? []).map((call) => ({ name: call.function.name, args: normalizeArgs(call.function.arguments) })) + ); + const usage = entries.flatMap((entry) => (entry.response?.usage ? [entry.response.usage.totalTokens] : [])); + const last = entries.at(-1); + return { + tools, + steps: entries.length, + finishReason: last?.response?.finishReason ?? last?.error?.name, + ...(usage.length > 0 ? { totalTokens: usage.reduce((sum, tokens) => sum + tokens, 0) } : {}), + }; +} + +/** + * Differences from `committed` to `current`: tool order, then (when the + * order matches) each call's arguments, step count and finish reason. Token + * usage varies between runs of a real model, so it is compared only with + * `{ usage: true }`. + */ +export function diffTrajectories(committed: Trajectory, current: Trajectory, options: { usage?: boolean } = {}): DriftEntry[] { + const drift: DriftEntry[] = []; + const compare = (field: DriftEntry['field'], before: string, after: string) => { + if (before !== after) drift.push({ field, committed: before, current: after }); + }; + const names = (t: Trajectory) => t.tools.map((call) => call.name).join(' > ') || 'none'; + compare('tools', names(committed), names(current)); + if (names(committed) === names(current)) { + committed.tools.forEach((call, i) => compare('args', `${call.name} ${call.args}`, `${call.name} ${current.tools[i].args}`)); + } + compare('steps', String(committed.steps), String(current.steps)); + compare('finishReason', committed.finishReason ?? '-', current.finishReason ?? '-'); + if (options.usage) compare('tokens', String(committed.totalTokens ?? '-'), String(current.totalTokens ?? '-')); + return drift; +} diff --git a/src/evals/evalResult.ts b/src/evals/evalResult.ts index 9092fcfac..4adf2b106 100644 --- a/src/evals/evalResult.ts +++ b/src/evals/evalResult.ts @@ -51,6 +51,8 @@ export interface EvalResult { error?: string; /** Test file the eval was defined in, when known. */ file?: string; + /** Cassettes the case recorded or replayed under `loushy eval --record` / `--replay` / `--drift`. */ + cassettes?: string[]; } /** Gate assertions that failed. */