From 306eee5950fec92f2d76136d12df2c657994443a Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 1 Oct 2026 10:15:11 +0300 Subject: [PATCH 1/2] LOU-D7, LOU-D8: trajectory evals and the loushy eval command (WIP) --- .fallowrc.json | 2 +- README.md | 44 ++- bin/loushy.js | 9 + docs/api-overview.md | 5 +- docs/evals.md | 231 ++++++++++++ src/cli/__fixtures__/eval/broken.eval.ts | 15 + src/cli/__fixtures__/eval/refund.eval.ts | 51 +++ src/cli/eval.test.ts | 218 +++++++++++ src/cli/eval.ts | 198 ++++++++++ src/cli/evalReport.test.ts | 204 ++++++++++ src/cli/evalReport.ts | 193 ++++++++++ src/evals/checks.ts | 126 +++++++ src/evals/defineEval.ts | 153 +++++++- src/evals/evalResult.ts | 77 ++++ src/evals/index.ts | 3 + src/evals/llmJudge.ts | 7 +- src/evals/recorder.ts | 33 ++ src/evals/toolMatch.ts | 54 +++ src/evals/trajectory.test.ts | 455 +++++++++++++++++++++++ src/evals/trajectory.ts | 298 +++++++++++++++ tsup.config.ts | 1 + vitest.config.ts | 2 + 22 files changed, 2360 insertions(+), 19 deletions(-) create mode 100644 docs/evals.md create mode 100644 src/cli/__fixtures__/eval/broken.eval.ts create mode 100644 src/cli/__fixtures__/eval/refund.eval.ts create mode 100644 src/cli/eval.test.ts create mode 100644 src/cli/eval.ts create mode 100644 src/cli/evalReport.test.ts create mode 100644 src/cli/evalReport.ts create mode 100644 src/evals/checks.ts create mode 100644 src/evals/evalResult.ts create mode 100644 src/evals/recorder.ts create mode 100644 src/evals/toolMatch.ts create mode 100644 src/evals/trajectory.test.ts create mode 100644 src/evals/trajectory.ts diff --git a/.fallowrc.json b/.fallowrc.json index 0ed0688a4..0fb5bf862 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -1,7 +1,7 @@ { "$schema": "./node_modules/fallow/schema.json", "entry": [ - "src/cli/{dev,build,studio,mcp,doctor}.ts", + "src/cli/{dev,build,studio,mcp,doctor,eval}.ts", "src/data/index.ts", "**/*.test-d.ts", "**/*.{test,spec}.{ts,tsx}", diff --git a/README.md b/README.md index a1b25f4b6..455c861d5 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ console.log(result.usage.totalTokens, result.finishReason, result.steps); - [API Overview](docs/api-overview.md) - the main exports; `npm run docs:build` generates the full TypeDoc reference - [Tracing and observability](docs/observability.md) - OpenTelemetry GenAI spans, attribute table, content opt-in - [Testing](docs/testing.md) - unit-test agents deterministically with the scripted `mockModel` +- [Evals](docs/evals.md) - trajectory evals with `defineEval()`, datasets, `mockModel`, judge evals, `loushy eval` with JUnit/JSON reports - [Skills](docs/skills.md) - on-demand instructions: `defineSkill()`, `loadSkills()`, how they save context - [Agent Forge](docs/agent-forge.md) - the visual dashboard (`loushy studio`): quickstart, first-agent walkthrough, hook authoring - Full guides site: [linuxdevil.github.io/agent-sdk-docs](https://linuxdevil.github.io/agent-sdk-docs/) @@ -459,22 +460,51 @@ and real OpenTelemetry bridges). ### Evals -Agent-behavior regression tests, run via `vitest run` alongside your normal -test suite — no new test runner: +Agent-behavior regression tests: assert on the tools an agent called, their +order and arguments, its steps and its reply. Deterministic with `mockModel`, +datasets via `cases`, soft vs gate assertions, and `loushy eval` for a summary +table plus JUnit/JSON reports in CI: ```typescript // support-agent.eval.ts -import { defineEval, exactMatch, toolCallOrder } from '@loushy/build-ai-agent'; +import { z } from 'zod'; +import { createAgent, defineEval, defineTool, includes } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'shipped' }), +}); defineEval({ name: 'looks up the order before replying', - agent, provider, toolRegistry, - input: 'Where is order #123?', - score: toolCallOrder([{ tool: 'lookupOrder' }]), - threshold: 1, + agent: () => + createAgent({ + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '123' } }] }, + { text: 'Order 123 has shipped.' }, + ]), + }), + async test(t) { + await t.send('Where is order #123?'); + t.completed(); + t.calledTool('lookup_order', { args: { orderId: '123' } }); + t.check('says shipped', t.reply, includes('shipped')); + }, }); ``` +```bash +npx loushy eval --junit reports/evals.xml +``` + +See [Evals](docs/evals.md) for the assertions table, datasets, judge evals and +a CI example. The original `{ agent, input, provider, score, threshold }` form +of `defineEval()` keeps working. + ### MCP tools & sandboxing ```typescript diff --git a/bin/loushy.js b/bin/loushy.js index ed7e3c3b8..4a181183c 100644 --- a/bin/loushy.js +++ b/bin/loushy.js @@ -10,6 +10,7 @@ const USAGE = [ ' loushy studio [--port N] [--host H] [--prod|--dev]', ' loushy mcp [--http --port N --host H]', ' loushy doctor [agent.yaml|json] [--json]', + ' loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge]', ].join('\n'); /** @@ -91,12 +92,20 @@ async function runMcp(rest) { process.exitCode = await start(rest); } +// Runs eval files under the project's vitest and reports the results; the +// exit code is the verdict (see src/cli/eval.ts). +async function runEvalCommand(rest) { + const { runEval } = require(path.join(__dirname, '..', 'dist', 'cli', 'eval.js')); + process.exitCode = await runEval(rest); +} + const COMMANDS = new Map([ ['dev', runDev], ['build', runBuildCommand], ['studio', runStudio], ['mcp', runMcp], ['doctor', runDoctorCommand], + ['eval', runEvalCommand], ]); async function main() { diff --git a/docs/api-overview.md b/docs/api-overview.md index cc73d3783..c26d583ce 100644 --- a/docs/api-overview.md +++ b/docs/api-overview.md @@ -334,8 +334,9 @@ The built-in context windows and prices are a dated snapshot (see the retrieval ## Flows, evals, observability and security - `FlowBuilder` / `FlowExecutor` - multi-step workflow graphs. -- `defineEval()`, scorers such as `exactMatch` and `toolCallOrder`, and - `llmJudge()` - agent evals run under vitest. +- `defineEval()`, scorers such as `exactMatch` and `toolCallOrder`, checks such + as `includes` and `atLeast`, and `llmJudge()` - agent evals run under vitest + or `loushy eval`; see [Evals](evals.md). - `withSpan()` and `TraceExporter` - tracing for `AgentExecutor.execute()`. `TraceExporter` is a bring-your-own-exporter interface (no exporter ships by default); for real OpenTelemetry spans, import diff --git a/docs/evals.md b/docs/evals.md new file mode 100644 index 000000000..462c95e9a --- /dev/null +++ b/docs/evals.md @@ -0,0 +1,231 @@ +# Evals + +Evals are regression tests for agent *behaviour*: which tools the agent called, +in what order, with which arguments, how many steps it took, and what it said. +They are written with `defineEval()`, live in `*.eval.ts` files, run under +[vitest](https://vitest.dev), and are best run in CI with `loushy eval`, which +prints a summary and writes JUnit and JSON reports. + +```bash +npm install --save-dev vitest +npx loushy eval +``` + +## Writing a trajectory eval + +Give `defineEval()` an agent (from `createAgent()`) and a `test` function. Send +messages with `t.send()`, then assert on the run. Using `mockModel` as the +agent's provider makes the eval deterministic: no network, no API key, no +flakiness. **This is the default way to write an eval**; use a real model only +for the [judge evals](#judge-evals) below. + +```ts +// refund.eval.ts +import { z } from 'zod'; +import { createAgent, defineEval, defineTool, includes } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order by id', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'delivered' }), +}); + +defineEval({ + name: 'refund flow', + tags: ['smoke'], + // A factory builds a fresh agent (and a fresh script) for every case. + agent: () => + createAgent({ + prompt: 'You handle refund requests.', + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '42' } }] }, + { text: 'Order 42 can be refunded within 30 days.' }, + ]), + }), + async test(t) { + await t.send('Refund order 42'); + t.completed(); + t.calledTool('lookup_order', { args: { orderId: '42' } }); + t.notCalledTool('issue_refund'); + t.check('mentions policy', t.reply, includes('30 days')); + t.maxSteps(4); + }, +}); +``` + +`defineEval()` also still accepts the original form +(`{ name, agent, input, provider, score, threshold }`): it keeps working +unchanged, and its result shows up in `loushy eval` as one `score` assertion. + +### Assertions + +Every assertion is recorded; the case fails at the end and lists **every** +failed gate, so one run shows all that is wrong. + +| Call | Kind | Passes when | +| --- | --- | --- | +| `t.completed()` | gate | the latest run ended with a normal `stop` (not an error, abort, pending approval, or `maxSteps` cut-off) | +| `t.calledTool(name, { args?, times? })` | gate | the tool was called; `args` is a partial deep match (the call's arguments must contain these keys with equal values); `times` is an exact count of matching calls | +| `t.notCalledTool(name)` | gate | the tool was never called | +| `t.toolOrder([a, b])` | gate | the tools were called in this order (other calls may sit in between) | +| `t.maxSteps(n)` | gate | the agent took at most `n` model steps | +| `t.maxTokens(n)` | gate | the agent used at most `n` tokens | +| `t.maxCostUsd(n)` | gate | the run's reported cost is at most `n` USD. Skipped (counts as passed, marked `skipped`) when the SDK reports no cost | +| `t.check(name, value, check)` | gate | `check` passes for `value` | +| `t.soft(name, value, check)` | soft | never fails the run (unless `--strict`); the score is recorded and reported | +| `await t.judge(rubric)` | n/a | returns a 0 to 1 score from an LLM judge (see [Judge evals](#judge-evals)) | + +Checks, for `t.check()` and `t.soft()`: `includes(text)`, `matches(regex)`, +`equals(value)` (yes/no, score 1 or 0), and `atLeast(n)`, `atMost(n)` (the +number is the score). + +Failure messages say what was actually seen: + +```text +calledTool('lookup_order') failed: tools called were [search_docs, issue_refund] +calledTool('lookup_order', {"args":{"orderId":"41"}}) failed: 'lookup_order' was called 1 time(s) but no call matched the expected args; closest call differs in orderId: expected "41", got "42" +completed() failed: finishReason was 'tool_calls' (the run hit maxSteps while still calling tools) +``` + +Other context members: `t.reply` (latest reply text), `t.result` (latest +`ExecutionResult`), `t.toolCalls` (every call so far, with parsed arguments). +You can `send()` more than once; steps, tokens and tool calls add up. + +### Datasets + +Pass `cases` and `test` runs once per case, each reported separately. A case is +any object; its `label` (or `name`, or its `input`) names it. + +```ts +import { z } from 'zod'; +import { createAgent, defineEval, defineTool } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order by id', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId }), +}); + +defineEval({ + name: 'order lookups', + agent: () => + createAgent({ + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '7' } }] }, + { text: 'Found it.' }, + ]), + }), + cases: [ + { input: 'Where is order 7?', orderId: '7' }, + { input: 'Status of #7', orderId: '7', label: 'hash syntax' }, + ], + async test(t, c) { + await t.send(c.input); + t.calledTool('lookup_order', { args: { orderId: c.orderId } }); + }, +}); +``` + +Pass a **factory** (`agent: () => createAgent(...)`) so each case gets its own +agent and its own `mockModel` script. With a single shared agent, a scripted +model runs out of turns on the second case. + +## Judge evals + +`t.judge(rubric)` grades the latest reply with an LLM and returns a score from +0 to 1. It needs a judge provider, and loushy never calls a real LLM unless you +configured one: without `judge`, `t.judge()` throws an error saying how to fix +it. + +```ts +// tone.judge.eval.ts: run with `loushy eval --judge` +import { createAgent, defineEval, atLeast } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +defineEval({ + name: 'tone', + agent: createAgent({ provider: mockModel(['Happy to help with that refund!']) }), + // A real provider in practice; mockModel keeps this example offline. + judge: { provider: mockModel(['0.9']), model: 'judge-model' }, + async test(t) { + await t.send('I want my money back'); + t.soft('polite', await t.judge('Is the reply polite?'), atLeast(0.7)); + }, +}); +``` + +Files named `*.judge.eval.ts` are never picked up by a normal run (and +`npm test`); only `loushy eval --judge` (or `npm run test:evals:judge` in this +repository) runs them. + +## `loushy eval` + +```text +loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] +``` + +| Option | Meaning | +| --- | --- | +| `globs...` | eval files to run (default: every `**/*.eval.ts`, or `**/*.judge.eval.ts` with `--judge`) | +| `--tag t` | only run evals whose `tags` include `t` (repeat or comma-separate for several) | +| `--junit path` | write a JUnit XML report | +| `--json path` | write the summary and every structured result as JSON | +| `--strict` | soft failures fail the run | +| `--judge` | run `*.judge.eval.ts` files instead of the normal ones | +| `--config path` | use your own vitest config instead of the generated one | + +It spawns the vitest installed in your project (vitest is *your* dependency; if +it is missing the command prints `npm install --save-dev vitest` and exits 2), +then prints one row per case with its scores and duration, totals, and the gate +failures and soft failures listed separately. + +**Exit code:** `1` on any gate failure (or soft failure with `--strict`, or when +vitest itself fails, for example a file that does not load), `2` when it cannot +run at all (bad arguments, vitest missing), otherwise `0`. + +How results are collected: each case appends one JSON line to a file named by +the `LOUSHY_EVAL_RESULTS` environment variable, which `loushy eval` sets and +reads back. This is more robust than a custom vitest reporter: it works across +vitest versions and worker pools, and needs no module loaded from your project. + +### JUnit + +The report is standard JUnit: one `testsuite` per eval, one `testcase` per case. +A failed gate is a `` carrying the diagnostic message; a +case that threw is an ``; soft failures are `` notes (or +failures under `--strict`). + +### In CI + +```yaml no-verify +# .github/workflows/evals.yml +name: evals +on: [pull_request] +jobs: + evals: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: 22 } + - run: npm ci + - run: npx loushy eval --junit reports/evals.xml --json reports/evals.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: eval-reports + path: reports/ + - uses: mikepenz/action-junit-report@v4 + if: always() + with: + report_paths: reports/evals.xml +``` + +Run `--tag smoke` on every pull request and the full set nightly; run +`--judge` only where you accept real LLM calls and their cost. diff --git a/src/cli/__fixtures__/eval/broken.eval.ts b/src/cli/__fixtures__/eval/broken.eval.ts new file mode 100644 index 000000000..e14f0202e --- /dev/null +++ b/src/cli/__fixtures__/eval/broken.eval.ts @@ -0,0 +1,15 @@ +/** + * Fixture for src/cli/eval.test.ts: an eval that fails its gate assertion. + */ +import { createAgent } from '../../../createAgent'; +import { defineEval } from '../../../evals'; +import { mockModel } from '../../../testing'; + +defineEval({ + name: 'broken flow', + agent: createAgent({ prompt: 'p', provider: mockModel(['I did nothing useful.']) }), + async test(t) { + await t.send('Refund order 42'); + t.calledTool('lookup_order'); + }, +}); diff --git a/src/cli/__fixtures__/eval/refund.eval.ts b/src/cli/__fixtures__/eval/refund.eval.ts new file mode 100644 index 000000000..6164a73c0 --- /dev/null +++ b/src/cli/__fixtures__/eval/refund.eval.ts @@ -0,0 +1,51 @@ +/** + * Fixture for src/cli/eval.test.ts: a tiny trajectory eval on a mockModel + * agent. Run through `loushy eval` by the test, never by the default vitest + * run (excluded in vitest.config.ts). + */ +import { z } from 'zod'; +import { createAgent } from '../../../createAgent'; +import { atLeast, defineEval, includes } from '../../../evals'; +import { defineTool } from '../../../tools/defineTool'; +import { mockModel } from '../../../testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'shipped' }), +}); + +const makeAgent = () => + createAgent({ + prompt: 'You handle refunds.', + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '42' } }] }, + { text: 'Refunds are possible within 30 days.' }, + ]), + }); + +defineEval({ + name: 'refund flow', + tags: ['smoke'], + agent: makeAgent, + cases: [{ input: 'Refund order 42' }, { input: 'Please refund order 42', label: 'polite' }], + async test(t, c) { + await t.send(c.input); + t.completed(); + t.calledTool('lookup_order', { args: { orderId: '42' } }); + t.check('mentions policy', t.reply, includes('30 days')); + t.soft('brevity', 0.5, atLeast(0.9)); + }, +}); + +defineEval({ + name: 'nightly only', + tags: ['nightly'], + agent: makeAgent, + async test(t) { + await t.send('hi'); + t.completed(); + }, +}); diff --git a/src/cli/eval.test.ts b/src/cli/eval.test.ts new file mode 100644 index 000000000..37254fd55 --- /dev/null +++ b/src/cli/eval.test.ts @@ -0,0 +1,218 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + buildVitestConfig, + parseEvalArgs, + resolveVitestBin, + runEval, + toRootRelativeGlob, + type VitestSpawner, +} from './eval'; +import type { EvalResult } from '../evals/evalResult'; +import { RESULTS_ENV, TAGS_ENV } from '../evals/recorder'; + +const FIXTURES = path.resolve(__dirname, '__fixtures__', 'eval'); + +function tempFile(name: string): string { + return path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'loushy-eval-test-')), name); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('parseEvalArgs', () => { + it('defaults to no globs and normal (non-strict, non-judge) mode', () => { + expect(parseEvalArgs([])).toEqual({ globs: [], tags: [], strict: false, judge: false }); + }); + + it('reads globs, repeatable/comma --tag, and both flag styles', () => { + expect( + parseEvalArgs(['a/*.eval.ts', '--tag', 'smoke,fast', '--tag=nightly', '--junit', 'out/j.xml', '--json=out/r.json', '--strict', '--judge', 'b.eval.ts', '--config', 'v.config.ts']) + ).toEqual({ + globs: ['a/*.eval.ts', 'b.eval.ts'], + tags: ['smoke', 'fast', 'nightly'], + junit: 'out/j.xml', + json: 'out/r.json', + strict: true, + judge: true, + config: 'v.config.ts', + }); + }); + + it('explains how to fix an unknown option or a missing value', () => { + expect(() => parseEvalArgs(['--bogus'])).toThrow(/unknown option '--bogus'.*Usage: loushy eval/); + expect(() => parseEvalArgs(['--junit'])).toThrow(/--junit needs a value/); + expect(() => parseEvalArgs(['--tag', '--strict'])).toThrow(/--tag needs a value/); + }); +}); + +describe('buildVitestConfig', () => { + it('collects *.eval files but never judge evals by default', () => { + const config = buildVitestConfig({ globs: [], judge: false }); + expect(config).toContain('"**/*.eval.{ts,mts,js,mjs}"'); + expect(config).toContain('"**/*.judge.eval.*"'); + expect(config).not.toContain('LOUSHY_ALLOW_LLM_JUDGE'); + }); + + it('collects only judge evals and allows the judge with --judge', () => { + const config = buildVitestConfig({ globs: [], judge: true }); + expect(config).toContain('"**/*.judge.eval.{ts,mts,js,mjs}"'); + expect(config).toContain('"LOUSHY_ALLOW_LLM_JUDGE": "1"'); + }); + + it('uses the given globs as the include list', () => { + expect(buildVitestConfig({ globs: ['evals/**/*.ts'], judge: false })).toContain('"evals/**/*.ts"'); + }); +}); + +describe('toRootRelativeGlob', () => { + it('keeps relative globs and makes absolute paths root-relative with forward slashes', () => { + const cwd = path.resolve('proj'); + expect(toRootRelativeGlob('evals/**/*.eval.ts', cwd)).toBe('evals/**/*.eval.ts'); + expect(toRootRelativeGlob(path.join(cwd, 'evals', 'a.eval.ts'), cwd)).toBe('evals/a.eval.ts'); + }); +}); + +describe('resolveVitestBin', () => { + it('finds vitest from this project and returns undefined where it is not installed', () => { + expect(resolveVitestBin(process.cwd())).toMatch(/vitest\.mjs$/); + expect(resolveVitestBin(os.tmpdir())).toBeUndefined(); + }); +}); + +function result(overrides: Partial): EvalResult { + return { name: 'e', tags: [], passed: true, assertions: [], durationMs: 5, steps: 1, toolCalls: [], ...overrides }; +} + +/** A fake vitest that records its invocation and appends canned results to the results file. */ +function fakeVitest(results: EvalResult[], code = 0) { + const calls: { args: string[]; env: NodeJS.ProcessEnv; config: string }[] = []; + const spawnVitest: VitestSpawner = async (_bin, args, env) => { + calls.push({ args, env, config: fs.readFileSync(args[args.indexOf('--config') + 1], 'utf8') }); + fs.appendFileSync(env[RESULTS_ENV] as string, results.map((r) => `${JSON.stringify(r)}\n`).join('')); + return code; + }; + return { spawnVitest, calls }; +} + +const softResult = result({ + name: 'soft', + assertions: [{ name: 'tone', kind: 'soft', passed: false, score: 0.2, threshold: 0.7, message: 'too curt' }], +}); +const gateResult = result({ + name: 'gate', + passed: false, + assertions: [{ name: 'completed()', kind: 'gate', passed: false, score: 0, threshold: 1, message: 'nope' }], +}); + +describe('runEval with a fake vitest', () => { + it('exits 2 and prints the install command when vitest is not installed', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + expect(await runEval([], { resolveVitest: () => undefined })).toBe(2); + expect(error.mock.calls[0][0]).toContain('npm install --save-dev vitest'); + }); + + it('exits 2 for bad arguments', async () => { + const error = vi.spyOn(console, 'error').mockImplementation(() => {}); + expect(await runEval(['--nope'], { resolveVitest: () => 'vitest.mjs' })).toBe(2); + expect(error.mock.calls[0][0]).toContain("unknown option '--nope'"); + }); + + it('passes the tag filter, results file and generated config to vitest and exits 0 on success', async () => { + const fake = fakeVitest([result({ name: 'ok' })]); + const log = vi.fn(); + const code = await runEval(['--tag', 'smoke', 'x.eval.ts'], { resolveVitest: () => 'vitest.mjs', spawnVitest: fake.spawnVitest, log, cwd: '/proj' }); + expect(code).toBe(0); + expect(fake.calls[0].args.slice(0, 1)).toEqual(['run']); + expect(fake.calls[0].args).toContain('/proj'); + expect(fake.calls[0].env[TAGS_ENV]).toBe('smoke'); + expect(fake.calls[0].config).toContain('"x.eval.ts"'); + expect(log.mock.calls[0][0]).toContain('1 eval(s): 1 passed, 0 failed'); + }); + + it('exits 1 on a gate failure and 0 on a soft failure, unless --strict', async () => { + const run = (extra: string[], results: EvalResult[]) => + runEval(extra, { resolveVitest: () => 'v', spawnVitest: fakeVitest(results).spawnVitest, log: () => {} }); + expect(await run([], [gateResult])).toBe(1); + expect(await run([], [softResult])).toBe(0); + expect(await run(['--strict'], [softResult])).toBe(1); + }); + + it('exits 1 when vitest itself fails with no recorded failure, so a load error never looks green', async () => { + const code = await runEval([], { resolveVitest: () => 'v', spawnVitest: fakeVitest([], 1).spawnVitest, log: () => {} }); + expect(code).toBe(1); + }); + + it('says so when no results were recorded', async () => { + const log = vi.fn(); + await runEval([], { resolveVitest: () => 'v', spawnVitest: fakeVitest([]).spawnVitest, log }); + expect(log.mock.calls[0][0]).toContain('no eval results were recorded'); + }); + + it('passes --judge through as the judge env var and uses the user config with --config', async () => { + const config = tempFile('v.config.mjs'); + fs.writeFileSync(config, 'export default {};'); + const fake = fakeVitest([result({})]); + await runEval(['--judge', '--config', config, 'only.judge.eval.ts'], { resolveVitest: () => 'v', spawnVitest: fake.spawnVitest, log: () => {} }); + expect(fake.calls[0].env.LOUSHY_ALLOW_LLM_JUDGE).toBe('1'); + expect(fake.calls[0].args).toEqual(['run', '--config', config, '--root', process.cwd(), 'only.judge.eval.ts']); + }); + + it('writes the JUnit and JSON reports, creating missing directories', async () => { + const dir = path.dirname(tempFile('x')); + const junit = path.join(dir, 'nested', 'junit.xml'); + const json = path.join(dir, 'nested', 'results.json'); + const fake = fakeVitest([gateResult]); + await runEval(['--junit', junit, '--json', json], { resolveVitest: () => 'v', spawnVitest: fake.spawnVitest, log: () => {} }); + expect(fs.readFileSync(junit, 'utf8')).toContain(' { + const cli = (args: string[]) => { + const log = vi.fn(); + vi.spyOn(console, 'log').mockImplementation(() => {}); + return runEval(args, { log }).then((code) => ({ code, output: log.mock.calls.map((c) => String(c[0])).join('\n') })); + }; + + it('runs a passing eval: exit 0, JUnit and JSON written, soft failure reported but not failing', async () => { + const junit = tempFile('junit.xml'); + const json = tempFile('results.json'); + const { code, output } = await cli([path.join(FIXTURES, 'refund.eval.ts'), '--junit', junit, '--json', json]); + + expect(code).toBe(0); + expect(output).toContain('3 eval(s): 3 passed, 0 failed, 2 with soft failures'); + expect(output).toContain('PASS (soft fail)'); + const xml = fs.readFileSync(junit, 'utf8'); + expect(xml).toContain(' r.case === 'polite')).toMatchObject({ + passed: true, + steps: 2, + toolCalls: [{ name: 'lookup_order', args: { orderId: '42' } }], + }); + }, 60_000); + + it('--tag runs only matching evals, and --strict turns the soft failure into exit 1', async () => { + const { code, output } = await cli([path.join(FIXTURES, 'refund.eval.ts'), '--tag', 'smoke', '--strict']); + expect(code).toBe(1); + expect(output).toContain('2 eval(s): 0 passed, 2 failed'); + expect(output).not.toContain('nightly only'); + }, 60_000); + + it('a failing gate exits 1 with the diagnostic message in the JUnit failure', async () => { + const junit = tempFile('junit.xml'); + const { code, output } = await cli([path.join(FIXTURES, 'broken.eval.ts'), '--junit', junit]); + expect(code).toBe(1); + expect(output).toContain("calledTool('lookup_order') failed: tools called were none"); + expect(fs.readFileSync(junit, 'utf8')).toContain( + " Promise; + +/** Test seams of {@link runEval}. */ +export interface EvalDeps { + /** Path to the project's vitest executable, or undefined when not installed. */ + resolveVitest?: (cwd: string) => string | undefined; + spawnVitest?: VitestSpawner; + cwd?: string; + log?: (message: string) => void; +} + +/** Finds `node_modules/vitest`'s CLI script from `cwd`, or undefined when vitest is not installed. */ +export function resolveVitestBin(cwd: string): string | undefined { + try { + const pkgPath = createRequire(path.join(cwd, 'noop.js')).resolve('vitest/package.json'); + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')) as { bin?: string | Record }; + const bin = typeof pkg.bin === 'string' ? pkg.bin : pkg.bin?.vitest; + return bin ? path.join(path.dirname(pkgPath), bin) : undefined; + } catch { + return undefined; + } +} + +const defaultSpawn: VitestSpawner = (bin, args, env) => + new Promise((resolve, reject) => { + const child = spawn(process.execPath, [bin, ...args], { stdio: 'inherit', env }); + child.on('error', reject); + child.on('close', (code) => resolve(code ?? 1)); + }); + +/** + * The generated vitest config: a plain object (no imports, so it loads from + * a temp directory). Normal runs never include `*.judge.eval.*`; judge runs + * only include them and set the env var llmJudge() requires. + */ +export function buildVitestConfig(args: Pick): string { + const defaults = args.judge ? ['**/*.judge.eval.{ts,mts,js,mjs}'] : ['**/*.eval.{ts,mts,js,mjs}']; + const exclude = ['**/node_modules/**', '**/dist/**', ...(args.judge ? [] : ['**/*.judge.eval.*'])]; + const test = { + globals: true, + environment: 'node', + include: args.globs.length > 0 ? args.globs : defaults, + exclude, + env: args.judge ? { LOUSHY_ALLOW_LLM_JUDGE: '1' } : {}, + }; + return `export default ${JSON.stringify({ test }, null, 2)};\n`; +} + +function writeReport(file: string, content: string): void { + fs.mkdirSync(path.dirname(path.resolve(file)), { recursive: true }); + fs.writeFileSync(file, content); +} + +interface VitestInvocation { + args: string[]; + env: NodeJS.ProcessEnv; + resultsFile: string; +} + +/** Drops VITEST_* variables, so running inside another vitest run (a test, an npm script) cannot confuse the child. */ +function withoutVitestVars(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return Object.fromEntries(Object.entries(env).filter(([key]) => !key.startsWith('VITEST'))); +} + +/** vitest matches globs relative to the root with `/` separators; absolute and Windows-style paths are converted. */ +export function toRootRelativeGlob(glob: string, cwd: string): string { + const relative = path.isAbsolute(glob) ? path.relative(cwd, glob) : glob; + return relative.split(path.sep).join('/'); +} + +function prepareInvocation(args: EvalCliArgs, cwd: string, workDir: string): VitestInvocation { + const resultsFile = path.join(workDir, 'results.jsonl'); + fs.writeFileSync(resultsFile, ''); + const globs = args.globs.map((glob) => toRootRelativeGlob(glob, cwd)); + let config = args.config; + if (!config) { + config = path.join(workDir, 'vitest.eval.config.mjs'); + fs.writeFileSync(config, buildVitestConfig({ ...args, globs })); + } + // With the user's own config, globs narrow its files as vitest filters. + const filters = args.config ? globs : []; + return { + args: ['run', '--config', config, '--root', cwd, ...filters], + env: { + ...withoutVitestVars(process.env), + [RESULTS_ENV]: resultsFile, + [TAGS_ENV]: args.tags.join(','), + ...(args.judge ? { LOUSHY_ALLOW_LLM_JUDGE: '1' } : {}), + }, + resultsFile, + }; +} + +async function runAndReport(args: EvalCliArgs, deps: Required, vitestBin: string): Promise { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'loushy-eval-')); + try { + const invocation = prepareInvocation(args, deps.cwd, workDir); + const vitestCode = await deps.spawnVitest(vitestBin, invocation.args, invocation.env); + const results = parseResults(fs.readFileSync(invocation.resultsFile, 'utf8')); + deps.log(results.length > 0 ? `\n${renderTable(results, args.strict)}` : '\nloushy eval: no eval results were recorded.'); + if (args.junit) writeReport(args.junit, renderJunit(results, args.strict)); + if (args.json) writeReport(args.json, renderJson(results, args.strict)); + // A non-zero vitest code with no recorded failure (a file that failed to + // load, an ordinary test failing) must not read as success. + return results.some((r) => failsRun(r, args.strict)) || vitestCode !== 0 ? 1 : 0; + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } +} + +/** + * CLI entry point. Resolves with the exit code: 0 all good, 1 a gate failure + * (or soft failure with `--strict`, or vitest itself failing), 2 the command + * could not run (bad arguments, vitest missing). + */ +export async function runEval(rest: string[], deps: EvalDeps = {}): Promise { + const resolved: Required = { + resolveVitest: deps.resolveVitest ?? resolveVitestBin, + spawnVitest: deps.spawnVitest ?? defaultSpawn, + cwd: deps.cwd ?? process.cwd(), + log: deps.log ?? ((message) => console.log(message)), + }; + try { + const args = parseEvalArgs(rest); + const vitestBin = resolved.resolveVitest(resolved.cwd); + if (!vitestBin) { + console.error('loushy eval: vitest is not installed in this project. Install it with:\n npm install --save-dev vitest'); + return 2; + } + return await runAndReport(args, resolved, vitestBin); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + return 2; + } +} diff --git a/src/cli/evalReport.test.ts b/src/cli/evalReport.test.ts new file mode 100644 index 000000000..8f3e852ed --- /dev/null +++ b/src/cli/evalReport.test.ts @@ -0,0 +1,204 @@ +import { describe, it, expect } from 'vitest'; +import type { EvalResult } from '../evals/evalResult'; +import { failsRun, parseResults, renderJson, renderJunit, renderTable, summarize } from './evalReport'; + +const passing: EvalResult = { + name: 'refund flow', + case: 'Refund order 42', + tags: ['smoke'], + passed: true, + assertions: [ + { name: 'completed()', kind: 'gate', passed: true, score: 1, threshold: 1 }, + { name: 'tone', kind: 'soft', passed: true, score: 0.85, threshold: 0.7 }, + ], + durationMs: 12, + steps: 2, + toolCalls: [{ name: 'lookup_order', args: { orderId: '42' } }], +}; + +const softFail: EvalResult = { + ...passing, + case: 'polite', + assertions: [ + { name: 'brevity', kind: 'soft', passed: false, score: 0.5, threshold: 0.9, message: "soft check 'brevity' failed: score 0.5 is below the threshold 0.9" }, + ], + durationMs: 1500, +}; + +const gateFail: EvalResult = { + name: 'broken & "co"', + tags: [], + passed: false, + assertions: [ + { + name: "calledTool('lookup_order')", + kind: 'gate', + passed: false, + score: 0, + threshold: 1, + message: "calledTool('lookup_order') failed: tools called were [search_docs, issue_refund]", + }, + ], + durationMs: 7, + steps: 1, + toolCalls: [], +}; + +const errored: EvalResult = { + name: 'errored', + tags: [], + passed: false, + assertions: [], + durationMs: 3, + steps: 0, + toolCalls: [], + error: 't.judge() needs a judge provider.\u0007', +}; + +const all = [passing, softFail, gateFail, errored]; + +describe('aggregation', () => { + it('summarizes totals, treating soft failures as passing unless strict', () => { + expect(summarize(all, false)).toEqual({ total: 4, passed: 2, failed: 2, softFailed: 1, strict: false, durationMs: 1522 }); + expect(summarize(all, true)).toMatchObject({ passed: 1, failed: 3, softFailed: 1, strict: true }); + }); + + it('failsRun covers gate failures, errors and strict soft failures', () => { + expect(failsRun(passing, true)).toBe(false); + expect(failsRun(softFail, false)).toBe(false); + expect(failsRun(softFail, true)).toBe(true); + expect(failsRun(gateFail, false)).toBe(true); + expect(failsRun(errored, false)).toBe(true); + }); + + it('parses JSON lines and skips blank or truncated ones', () => { + const text = `${JSON.stringify(passing)}\n\n{"name": "cut off\n${JSON.stringify(gateFail)}\r\n`; + expect(parseResults(text).map((r) => r.name)).toEqual(['refund flow', gateFail.name]); + expect(parseResults('')).toEqual([]); + }); +}); + +describe('renderTable', () => { + it('lists every case, totals, and separates gate from soft failures', () => { + expect(renderTable(all, false)).toMatchInlineSnapshot(` + "EVAL CASE RESULT SCORES DURATION + refund flow Refund order 42 PASS tone=0.85 12ms + refund flow polite PASS (soft fail) brevity=0.50 1.50s + broken & "co" - FAIL - 7ms + errored - FAIL - 3ms + + 4 eval(s): 2 passed, 2 failed, 1 with soft failures (1.52s) + + Gate failures: + broken & "co": calledTool('lookup_order') failed: tools called were [search_docs, issue_refund] + errored: t.judge() needs a judge provider. + + Soft failures (not failing the run): + refund flow [polite]: soft check 'brevity' failed: score 0.5 is below the threshold 0.9" + `); + }); + + it('marks soft failures as failing under --strict', () => { + expect(renderTable([passing, softFail], true)).toMatchInlineSnapshot(` + "EVAL CASE RESULT SCORES DURATION + refund flow Refund order 42 PASS tone=0.85 12ms + refund flow polite FAIL brevity=0.50 1.50s + + 2 eval(s): 1 passed, 1 failed, 1 with soft failures (1.51s) + + Soft failures (failing the run: --strict): + refund flow [polite]: soft check 'brevity' failed: score 0.5 is below the threshold 0.9" + `); + }); +}); + +describe('renderJunit', () => { + it('writes testsuites/testsuite/testcase with diagnostic failures and escapes XML', () => { + expect(renderJunit(all, false)).toMatchInlineSnapshot(` + " + + + + + soft failure: soft check 'brevity' failed: score 0.5 is below the threshold 0.9 + + + + + calledTool('lookup_order') failed: tools called were [search_docs, issue_refund] + + + + + t.judge() needs a judge provider. + + + + " + `); + }); + + it('turns soft failures into failures under --strict', () => { + expect(renderJunit([softFail], true)).toMatchInlineSnapshot(` + " + + + + soft check 'brevity' failed: score 0.5 is below the threshold 0.9 + + + + " + `); + }); + + it('is an empty but valid document without results', () => { + expect(renderJunit([], false)).toMatchInlineSnapshot(` + " + + + " + `); + }); +}); + +describe('renderJson', () => { + it('contains the summary and every structured result', () => { + expect(renderJson([gateFail], false)).toMatchInlineSnapshot(` + "{ + "summary": { + "total": 1, + "passed": 0, + "failed": 1, + "softFailed": 0, + "strict": false, + "durationMs": 7 + }, + "results": [ + { + "name": "broken & \\"co\\"", + "tags": [], + "passed": false, + "assertions": [ + { + "name": "calledTool('lookup_order')", + "kind": "gate", + "passed": false, + "score": 0, + "threshold": 1, + "message": "calledTool('lookup_order') failed: tools called were [search_docs, issue_refund]" + } + ], + "durationMs": 7, + "steps": 1, + "toolCalls": [] + } + ] + } + " + `); + const parsed = JSON.parse(renderJson(all, true)) as { summary: { failed: number }; results: EvalResult[] }; + expect(parsed.summary.failed).toBe(3); + expect(parsed.results).toHaveLength(4); + }); +}); diff --git a/src/cli/evalReport.ts b/src/cli/evalReport.ts new file mode 100644 index 000000000..b0bad1ab6 --- /dev/null +++ b/src/cli/evalReport.ts @@ -0,0 +1,193 @@ +/** + * Pure aggregation and rendering for `loushy eval` (LOU-D8): the summary + * table, JUnit XML and JSON. No I/O, so every output is snapshot-testable. + */ +import { + gateFailures, + softFailures, + type AssertionResult, + type EvalResult, +} from '../evals/evalResult'; + +/** Totals over a run of evals. */ +export interface EvalSummary { + total: number; + /** Cases whose gates all passed (and, with `strict`, whose soft assertions did too). */ + passed: number; + /** Cases that failed a gate, threw, or (with `strict`) failed a soft assertion. */ + failed: number; + /** Cases that passed their gates but have failed soft assertions. */ + softFailed: number; + strict: boolean; + durationMs: number; +} + +/** Parses the JSON-lines results file; malformed lines are skipped. */ +export function parseResults(text: string): EvalResult[] { + const results: EvalResult[] = []; + for (const line of text.split(/\r?\n/)) { + if (!line.trim()) continue; + try { + results.push(JSON.parse(line) as EvalResult); + } catch { + // A line cut off by a crashed worker; the vitest exit code still reports the crash. + } + } + return results; +} + +/** True when `result` fails the run: a gate failure or error, or a soft failure under `strict`. */ +export function failsRun(result: EvalResult, strict: boolean): boolean { + return !result.passed || (strict && softFailures(result).length > 0); +} + +/** Totals for `results`. */ +export function summarize(results: readonly EvalResult[], strict: boolean): EvalSummary { + const failed = results.filter((r) => failsRun(r, strict)).length; + return { + total: results.length, + passed: results.length - failed, + failed, + softFailed: results.filter((r) => r.passed && softFailures(r).length > 0).length, + strict, + durationMs: results.reduce((sum, r) => sum + r.durationMs, 0), + }; +} + +function formatScore(assertion: AssertionResult): string { + const score = Number.isInteger(assertion.score) ? String(assertion.score) : assertion.score.toFixed(2); + return `${assertion.name}=${score}`; +} + +/** Scored assertions (anything that is not a plain yes/no gate) for the table's SCORES column. */ +function scoresColumn(result: EvalResult): string { + const scored = result.assertions.filter((a) => a.kind === 'soft' || !Number.isInteger(a.score) || a.name === 'score'); + return scored.map(formatScore).join(', ') || '-'; +} + +function resultColumn(result: EvalResult, strict: boolean): string { + if (failsRun(result, strict)) return 'FAIL'; + return softFailures(result).length > 0 ? 'PASS (soft fail)' : 'PASS'; +} + +function formatDuration(ms: number): string { + return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(2)}s`; +} + +function renderRows(rows: string[][]): string[] { + const widths = rows[0].map((_, col) => Math.max(...rows.map((row) => row[col].length))); + return rows.map((row) => row.map((cell, col) => cell.padEnd(widths[col])).join(' ').trimEnd()); +} + +function failureLines(title: string, results: readonly EvalResult[], pick: (r: EvalResult) => string[]): string[] { + const entries = results.flatMap((r) => pick(r).map((message) => ` ${label(r)}: ${message}`)); + return entries.length === 0 ? [] : ['', `${title}:`, ...entries]; +} + +function label(result: EvalResult): string { + return result.case ? `${result.name} [${result.case}]` : result.name; +} + +function gateMessages(result: EvalResult): string[] { + const messages = gateFailures(result).map((a) => a.message ?? a.name); + return result.error ? [...messages, result.error] : messages; +} + +function softMessages(result: EvalResult): string[] { + return softFailures(result).map((a) => a.message ?? a.name); +} + +/** + * The console summary: one row per eval case, totals, then gate failures + * and soft failures listed separately. + */ +export function renderTable(results: readonly EvalResult[], strict: boolean): string { + const summary = summarize(results, strict); + const header = ['EVAL', 'CASE', 'RESULT', 'SCORES', 'DURATION']; + const rows = results.map((r) => [r.name, r.case ?? '-', resultColumn(r, strict), scoresColumn(r), formatDuration(r.durationMs)]); + const totals = `${summary.total} eval(s): ${summary.passed} passed, ${summary.failed} failed, ${summary.softFailed} with soft failures (${formatDuration(summary.durationMs)})`; + const gateFailed = results.filter((r) => !r.passed); + return [ + ...renderRows([header, ...rows]), + '', + totals, + ...failureLines('Gate failures', gateFailed, gateMessages), + ...failureLines(strict ? 'Soft failures (failing the run: --strict)' : 'Soft failures (not failing the run)', results, softMessages), + ].join('\n'); +} + +/** Machine-readable report: the summary plus every structured result. */ +export function renderJson(results: readonly EvalResult[], strict: boolean): string { + return `${JSON.stringify({ summary: summarize(results, strict), results }, null, 2)}\n`; +} + +/** Drops control characters XML 1.0 cannot represent (everything below 0x20 except tab, LF, CR). */ +function stripIllegalXml(text: string): string { + return [...text].filter((ch) => ch.charCodeAt(0) >= 0x20 || '\t\n\r'.includes(ch)).join(''); +} + +function isErrorOnly(result: EvalResult): boolean { + return Boolean(result.error) && gateFailures(result).length === 0; +} + +function xml(text: string): string { + return stripIllegalXml(text) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function seconds(ms: number): string { + return (ms / 1000).toFixed(3); +} + +function testcaseXml(result: EvalResult, strict: boolean): string { + const open = ` \n ${xml(result.error)}\n `; + } + const failures = gates.length > 0 ? gates : strict ? softs : []; + if (failures.length > 0) { + return `${open}>\n ${xml(failures.join('\n'))}\n `; + } + if (softs.length > 0) { + return `${open}>\n ${xml(softs.map((m) => `soft failure: ${m}`).join('\n'))}\n `; + } + return `${open} />`; +} + +function counts(cases: readonly EvalResult[], strict: boolean): { failures: number; errors: number; time: string } { + const errors = cases.filter(isErrorOnly).length; + return { + errors, + failures: cases.filter((r) => failsRun(r, strict)).length - errors, + time: seconds(cases.reduce((sum, r) => sum + r.durationMs, 0)), + }; +} + +function suiteXml(name: string, cases: readonly EvalResult[], strict: boolean): string { + const { failures, errors, time } = counts(cases, strict); + const head = ` `; + return [head, ...cases.map((r) => testcaseXml(r, strict)), ' '].join('\n'); +} + +/** + * JUnit XML (`testsuites` > `testsuite` per eval > `testcase` per case). + * Gate failures carry the diagnostic message in ``; soft failures + * are `` notes, or failures under `strict`. + */ +export function renderJunit(results: readonly EvalResult[], strict: boolean): string { + const byEval = new Map(); + for (const result of results) byEval.set(result.name, [...(byEval.get(result.name) ?? []), result]); + const { failures, errors, time } = counts(results, strict); + return [ + '', + ``, + ...[...byEval].map(([name, cases]) => suiteXml(name, cases, strict)), + '', + '', + ].join('\n'); +} diff --git a/src/evals/checks.ts b/src/evals/checks.ts new file mode 100644 index 000000000..ec05ba132 --- /dev/null +++ b/src/evals/checks.ts @@ -0,0 +1,126 @@ +/** + * Value checks for `t.check()` / `t.soft()` (LOU-D7). + * + * A check judges one value (a reply string, a judge score, a number) and + * says why it failed. `includes`, `matches` and `equals` are yes/no; + * `atLeast` and `atMost` compare a number against a threshold. + */ + +/** Outcome of running a {@link Check}. */ +export interface CheckOutcome { + /** 0 or 1 for yes/no checks; the compared value for `atLeast`/`atMost`. */ + score: number; + passed: boolean; + /** Why the check failed; absent when it passed. */ + message?: string; + threshold?: number; +} + +/** + * A reusable assertion over a value of type `T`. + * + * @example + * ```ts + * const mentionsPolicy: Check = includes('30 days'); + * ``` + */ +export interface Check { + evaluate(value: T): CheckOutcome; +} + +function preview(value: unknown): string { + const text = typeof value === 'string' ? value : JSON.stringify(value); + const shown = text.length > 160 ? `${text.slice(0, 157)}...` : text; + return typeof value === 'string' ? JSON.stringify(shown) : shown; +} + +function yesNo(passed: boolean, failure: string): CheckOutcome { + return { score: passed ? 1 : 0, passed, threshold: 1, message: passed ? undefined : failure }; +} + +/** + * Passes when the string contains `needle` (case-sensitive). + * + * @example + * ```ts + * t.check('mentions policy', t.reply, includes('30 days')); + * ``` + */ +export function includes(needle: string): Check { + return { + evaluate: (value) => + yesNo(value.includes(needle), `expected ${preview(value)} to include ${JSON.stringify(needle)}`), + }; +} + +/** + * Passes when the string matches `pattern`. + * + * @example + * ```ts + * t.check('has an order id', t.reply, matches(/#\d+/)); + * ``` + */ +export function matches(pattern: RegExp): Check { + return { + evaluate: (value) => { + // A fresh RegExp without the `g` flag keeps `test()` stateless across calls. + const stateless = new RegExp(pattern.source, pattern.flags.replace('g', '')); + return yesNo(stateless.test(value), `expected ${preview(value)} to match ${pattern}`); + }, + }; +} + +/** + * Passes when the value equals `expected` (compared by JSON form). + * + * @example + * ```ts + * t.check('exact reply', t.reply, equals('Done.')); + * ``` + */ +export function equals(expected: T): Check { + return { + evaluate: (value) => + yesNo( + JSON.stringify(value) === JSON.stringify(expected), + `expected ${preview(value)} to equal ${preview(expected)}` + ), + }; +} + +/** + * Passes when the number is at least `threshold`; the number is the score. + * + * @example + * ```ts + * t.soft('tone', await t.judge('Is the reply polite?'), atLeast(0.7)); + * ``` + */ +export function atLeast(threshold: number): Check { + return { + evaluate: (value) => { + const passed = value >= threshold; + const message = passed ? undefined : `score ${value} is below the threshold ${threshold}`; + return { score: value, passed, threshold, message }; + }, + }; +} + +/** + * Passes when the number is at most `limit`; the number is the score. + * + * @example + * ```ts + * t.check('latency', elapsedMs, atMost(2000)); + * ``` + */ +export function atMost(limit: number): Check { + return { + evaluate: (value) => { + const passed = value <= limit; + const message = passed ? undefined : `value ${value} is above the limit ${limit}`; + return { score: value, passed, threshold: limit, message }; + }, + }; +} diff --git a/src/evals/defineEval.ts b/src/evals/defineEval.ts index dde1d9858..3ee9ac2ef 100644 --- a/src/evals/defineEval.ts +++ b/src/evals/defineEval.ts @@ -11,6 +11,16 @@ import type * as Vitest from 'vitest'; import { AgentExecutor, ExecuteOptions, ExecutionResult } from '../execution/AgentExecutor'; +import { scoreAssertion } from './evalResult'; +import { matchesTagFilter, recordEvalResult } from './recorder'; +import { parseToolCalls } from './toolMatch'; +import { + describeFailure, + runTrajectoryCase, + type AgentSource, + type EvalJudgeConfig, + type EvalTestContext, +} from './trajectory'; /** * Configuration for a single eval case. @@ -45,6 +55,91 @@ export interface EvalConfig { score: (result: ExecutionResult) => number | Promise; /** Minimum score (inclusive) for the eval to pass. */ threshold: number; + /** Tags for `loushy eval --tag`. */ + tags?: string[]; +} + +/** + * A trajectory eval (LOU-D7): `test(t, c)` sends messages to a real agent + * and asserts on how it behaved. Runs once per entry of `cases`. + * + * @example + * ```ts + * defineEval({ + * name: 'refund flow', + * agent, + * cases: [{ input: 'Refund order 42', tool: 'lookup_order' }], + * async test(t, c) { + * await t.send(c.input); + * t.completed(); + * t.calledTool(c.tool); + * }, + * }); + * ``` + */ +export interface TrajectoryEvalConfig> { + /** Eval name, shown in vitest and `loushy eval` output. */ + name: string; + /** + * The agent under test (from `createAgent()`), or a factory that builds a + * fresh one per case so cases cannot share state. Use `mockModel` as its + * provider for a deterministic CI eval. + */ + agent: AgentSource; + /** Tags for `loushy eval --tag`. */ + tags?: string[]; + /** Dataset: `test` runs once per case. A case's `label` (or `name`, or its `input`) names it in reports. */ + cases?: readonly C[]; + /** Judge provider for `t.judge()`. Without it `t.judge()` throws; no LLM is ever called implicitly. */ + judge?: EvalJudgeConfig; + /** The test body. Gate assertions fail the case; `t.soft()` ones are only reported. */ + test(t: EvalTestContext, c: C): void | Promise; +} + +const NO_CASE = {} as never; + +function caseLabel(c: unknown, index: number): string { + const record = (typeof c === 'object' && c !== null ? c : {}) as Record; + for (const key of ['label', 'name']) { + if (typeof record[key] === 'string') return record[key] as string; + } + if (typeof record.input === 'string') { + return record.input.length > 48 ? `${record.input.slice(0, 45)}...` : record.input; + } + return `case ${index + 1}`; +} + +function currentTestPath(expect: Pick['expect']): string | undefined { + try { + return expect.getState().testPath; + } catch { + return undefined; + } +} + +async function runAndReport( + config: TrajectoryEvalConfig, + c: unknown, + label: string | undefined, + file: string | undefined +): Promise { + const result = await runTrajectoryCase(config, c, label, file); + recordEvalResult(result); + if (!result.passed) throw new Error(describeFailure(result)); +} + +function defineTrajectoryEval(config: TrajectoryEvalConfig): void { + const { test, expect } = currentVitest(); + const file = currentTestPath(expect); + const register = matchesTagFilter(config.tags ?? []) ? test : test.skip; + if (config.cases === undefined) { + register(config.name, () => runAndReport(config, NO_CASE, undefined, file)); + return; + } + config.cases.forEach((c, index) => { + const label = caseLabel(c, index); + register(`${config.name} [${label}]`, () => runAndReport(config, c, label, file)); + }); } /** @@ -70,16 +165,15 @@ function currentVitest(): Pick { throw new Error('defineEval() must be called from a test file running under vitest'); } -/** - * Define a single eval as a vitest test. Calls AgentExecutor.execute() - * exactly once with the config's execution fields, scores the result, and - * asserts `score >= threshold`. - */ -export function defineEval(config: EvalConfig): void { - const { name, score, threshold, ...executeFields } = config; + +function defineClassicEval(config: EvalConfig): void { + const { name, score, threshold, tags, ...executeFields } = config; const { test, expect } = currentVitest(); + const file = currentTestPath(expect); + const register = matchesTagFilter(tags ?? []) ? test : test.skip; - test(name, async () => { + register(name, async () => { + const started = Date.now(); const result = await AgentExecutor.execute({ agent: executeFields.agent, input: executeFields.input, @@ -91,7 +185,50 @@ export function defineEval(config: EvalConfig): void { }); const resultScore = await score(result); + const assertion = scoreAssertion(resultScore, threshold); + recordEvalResult({ + name, + tags: tags ?? [], + passed: assertion.passed, + assertions: [assertion], + durationMs: Date.now() - started, + steps: result.steps, + toolCalls: parseToolCalls(result.toolCalls ?? []), + usage: result.usage, + file, + }); expect(resultScore).toBeGreaterThanOrEqual(threshold); }); } + +/** + * Define an eval as vitest test(s). Two forms: + * + * - Classic: `{ name, agent, input, provider, score, threshold }` calls + * AgentExecutor.execute() once, scores the result and asserts + * `score >= threshold`. + * - Trajectory: `{ name, agent, cases?, test(t, c) }` drives a + * `createAgent()` agent and asserts on its tool calls, steps and reply. + * + * Run evals with `loushy eval` for a summary table and JUnit/JSON reports. + * + * @example + * ```ts + * defineEval({ + * name: 'greets', + * agent: createAgent({ provider: mockModel(['Hello!']) }), + * async test(t) { + * await t.send('hi'); + * t.completed(); + * t.check('greets', t.reply, includes('Hello')); + * }, + * }); + * ``` + */ +export function defineEval(config: EvalConfig): void; +export function defineEval>(config: TrajectoryEvalConfig): void; +export function defineEval(config: EvalConfig | TrajectoryEvalConfig): void { + if ('test' in config) defineTrajectoryEval(config); + else defineClassicEval(config); +} diff --git a/src/evals/evalResult.ts b/src/evals/evalResult.ts new file mode 100644 index 000000000..9092fcfac --- /dev/null +++ b/src/evals/evalResult.ts @@ -0,0 +1,77 @@ +/** + * Structured eval results (LOU-D7). + * + * Every eval case - the classic `score`/`threshold` form and the trajectory + * `test(t)` form - produces one `EvalResult`. `loushy eval` collects them to + * print a summary and write JUnit/JSON reports. + */ + +/** How an assertion counts: a `gate` fails the eval, a `soft` one is only reported. */ +export type AssertionKind = 'gate' | 'soft'; + +/** One assertion made while an eval case ran. */ +export interface AssertionResult { + /** Name shown in reports, e.g. `calledTool('lookup_order')` or `tone`. */ + name: string; + kind: AssertionKind; + passed: boolean; + /** 1 or 0 for yes/no assertions, the measured value for scored ones (judge, atLeast). */ + score: number; + /** Minimum score for the assertion to pass, when it has one. */ + threshold?: number; + /** Diagnostic detail; always present when `passed` is false. */ + message?: string; + /** True when the assertion could not be evaluated (for example no cost data) and counted as passed. */ + skipped?: boolean; +} + +/** A tool call the agent made, with parsed arguments. */ +export interface EvalToolCall { + name: string; + args: unknown; +} + +/** The outcome of one eval case. */ +export interface EvalResult { + /** The `name` given to `defineEval()`. */ + name: string; + /** Label of the dataset case; absent for an eval without `cases`. */ + case?: string; + tags: string[]; + /** False when any gate assertion failed or the case threw. */ + passed: boolean; + assertions: AssertionResult[]; + durationMs: number; + /** Model steps across every `t.send()` of the case. */ + steps: number; + toolCalls: EvalToolCall[]; + /** Token usage summed over the case's runs, when any run reported it. */ + usage?: { promptTokens: number; completionTokens: number; totalTokens: number }; + /** Set when the case threw (a failed `send()`, an unconfigured `judge()`, ...). */ + error?: string; + /** Test file the eval was defined in, when known. */ + file?: string; +} + +/** Gate assertions that failed. */ +export function gateFailures(result: EvalResult): AssertionResult[] { + return result.assertions.filter((a) => a.kind === 'gate' && !a.passed); +} + +/** Soft assertions that failed. */ +export function softFailures(result: EvalResult): AssertionResult[] { + return result.assertions.filter((a) => a.kind === 'soft' && !a.passed); +} + +/** Builds the one-assertion result of a classic `score`/`threshold` eval. */ +export function scoreAssertion(score: number, threshold: number): AssertionResult { + const passed = score >= threshold; + return { + name: 'score', + kind: 'gate', + passed, + score, + threshold, + message: passed ? undefined : `score ${score} is below the threshold ${threshold}`, + }; +} diff --git a/src/evals/index.ts b/src/evals/index.ts index 205597707..e58ebb84a 100644 --- a/src/evals/index.ts +++ b/src/evals/index.ts @@ -7,3 +7,6 @@ export * from './defineEval'; export * from './scorers'; export * from './llmJudge'; +export * from './checks'; +export * from './evalResult'; +export type { EvalJudgeConfig, EvalTestContext, CalledToolOptions, AgentSource } from './trajectory'; diff --git a/src/evals/llmJudge.ts b/src/evals/llmJudge.ts index 8b54741a9..9bff160f7 100644 --- a/src/evals/llmJudge.ts +++ b/src/evals/llmJudge.ts @@ -20,6 +20,11 @@ export interface LLMJudgeConfig { /** Grading rubric / instructions shown to the judge model. */ rubric: string; temperature?: number; + /** + * Skip the "only inside the judge-eval runner" guard. `t.judge()` sets this + * because the judge provider was configured explicitly on the eval. + */ + allowOutsideJudgeRunner?: boolean; } /** @@ -69,7 +74,7 @@ export function llmJudge(config: LLMJudgeConfig): (result: ExecutionResult) => P // llmJudge() but is accidentally picked up by the main vitest run (e.g. // a misnamed or mis-globbed file) fails loudly here instead of silently // making a real, budgeted LLM call as part of default/CI test runs. - if (process.env.LOUSHY_ALLOW_LLM_JUDGE !== '1') { + if (!config.allowOutsideJudgeRunner && process.env.LOUSHY_ALLOW_LLM_JUDGE !== '1') { throw new Error( 'llmJudge() was invoked outside the judge-eval runner. ' + 'llmJudge()-based evals must live in a "*.judge.eval.ts" file and run via ' + diff --git a/src/evals/recorder.ts b/src/evals/recorder.ts new file mode 100644 index 000000000..b8fdb5395 --- /dev/null +++ b/src/evals/recorder.ts @@ -0,0 +1,33 @@ +/** + * How eval results leave the vitest worker (LOU-D8). + * + * `loushy eval` sets `LOUSHY_EVAL_RESULTS` to a file path; every eval case + * appends its `EvalResult` there as one JSON line. A file (rather than a + * custom vitest reporter) is deliberate: it works with every vitest version + * and worker pool, needs no module resolved from the user's project, and + * `appendFileSync` of one short line is safe across parallel workers. + * `LOUSHY_EVAL_TAGS` carries the `--tag` filter the other way. + */ +import * as fs from 'node:fs'; +import type { EvalResult } from './evalResult'; + +/** Environment variable naming the JSON-lines file results are appended to. */ +export const RESULTS_ENV = 'LOUSHY_EVAL_RESULTS'; +/** Environment variable holding a comma-separated `--tag` filter. */ +export const TAGS_ENV = 'LOUSHY_EVAL_TAGS'; + +/** Appends `result` to the results file when `loushy eval` asked for one. */ +export function recordEvalResult(result: EvalResult): void { + const file = process.env[RESULTS_ENV]; + if (!file) return; + fs.appendFileSync(file, `${JSON.stringify(result)}\n`); +} + +/** True when no `--tag` filter is active or `tags` shares a tag with it. */ +export function matchesTagFilter(tags: readonly string[]): boolean { + const wanted = (process.env[TAGS_ENV] ?? '') + .split(',') + .map((tag) => tag.trim()) + .filter(Boolean); + return wanted.length === 0 || tags.some((tag) => wanted.includes(tag)); +} diff --git a/src/evals/toolMatch.ts b/src/evals/toolMatch.ts new file mode 100644 index 000000000..5f0954a2d --- /dev/null +++ b/src/evals/toolMatch.ts @@ -0,0 +1,54 @@ +/** + * Tool-call matching and diffing for trajectory assertions (LOU-D7). + */ +import type { EvalToolCall } from './evalResult'; + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function show(value: unknown): string { + return value === undefined ? 'undefined' : JSON.stringify(value); +} + +/** + * Differences between `expected` and `actual` where `expected` is a partial + * deep match: objects need only contain the expected keys, arrays and + * primitives must be equal. Returns one line per difference (empty = match). + */ +export function diffArgs(expected: unknown, actual: unknown, path = ''): string[] { + if (isPlainObject(expected) && isPlainObject(actual)) { + return Object.entries(expected).flatMap(([key, value]) => + diffArgs(value, actual[key], path ? `${path}.${key}` : key) + ); + } + if (JSON.stringify(expected) === JSON.stringify(actual)) return []; + return [`${path || 'args'}: expected ${show(expected)}, got ${show(actual)}`]; +} + +/** Parses the raw `{ function: { name, arguments } }` tool calls of an execution result. */ +export function parseToolCalls( + raw: ReadonlyArray<{ function: { name: string; arguments: string } }> +): EvalToolCall[] { + return raw.map((call) => { + try { + return { name: call.function.name, args: JSON.parse(call.function.arguments) as unknown }; + } catch { + return { name: call.function.name, args: call.function.arguments }; + } + }); +} + +/** True when `names` appear in `calls` in this order (other calls may interleave). */ +export function isSubsequence(calls: readonly EvalToolCall[], names: readonly string[]): boolean { + let next = 0; + for (const call of calls) { + if (call.name === names[next]) next++; + } + return next === names.length; +} + +/** `[a, b]` or `none`, for failure messages. */ +export function describeCalled(calls: readonly EvalToolCall[]): string { + return calls.length === 0 ? 'none' : `[${calls.map((c) => c.name).join(', ')}]`; +} diff --git a/src/evals/trajectory.test.ts b/src/evals/trajectory.test.ts new file mode 100644 index 000000000..f9be9bf95 --- /dev/null +++ b/src/evals/trajectory.test.ts @@ -0,0 +1,455 @@ +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; +import { createAgent } from '../createAgent'; +import { defineTool } from '../tools/defineTool'; +import { mockModel, type MockTurn } from '../testing'; +import { atLeast, atMost, equals, includes, matches } from './checks'; +import { defineEval } from './defineEval'; +import { describeFailure, runTrajectoryCase, type EvalTestContext } from './trajectory'; +import type { EvalResult } from './evalResult'; +import { matchesTagFilter, recordEvalResult, RESULTS_ENV, TAGS_ENV } from './recorder'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'shipped' }), +}); +const checkPolicy = defineTool({ + name: 'check_policy', + description: 'Check the refund policy', + input: z.object({}), + execute: () => ({ days: 30 }), +}); +const issueRefund = defineTool({ + name: 'issue_refund', + description: 'Refund an order', + input: z.object({ orderId: z.string() }), + execute: () => ({ ok: true }), +}); + +const REFUND_SCRIPT: MockTurn[] = [ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '42', verbose: true } }], usage: { inputTokens: 10, outputTokens: 5 } }, + { toolCalls: [{ name: 'check_policy' }], usage: { inputTokens: 10, outputTokens: 5 } }, + { text: 'You can return it within 30 days.', usage: { inputTokens: 10, outputTokens: 5 } }, +]; + +function refundAgent(script: MockTurn[] = REFUND_SCRIPT, maxSteps?: number) { + return createAgent({ + prompt: 'You handle refunds.', + provider: mockModel(script), + tools: [lookupOrder, checkPolicy, issueRefund], + maxSteps, + }); +} + +async function run( + test: (t: EvalTestContext) => void | Promise, + options: { agent?: ReturnType; judge?: Parameters[0]['judge'] } = {} +): Promise { + const spec = { name: 'refund', agent: options.agent ?? refundAgent(), judge: options.judge, test }; + return runTrajectoryCase(spec, {}, undefined, undefined); +} + +function failedMessages(result: EvalResult): string[] { + return result.assertions.filter((a) => !a.passed).map((a) => a.message ?? ''); +} + +describe('trajectory assertions: passing', () => { + it('passes every assertion for a matching run and reports steps, tool calls and usage', async () => { + const result = await run(async (t) => { + await t.send('Refund order 42'); + t.completed(); + t.calledTool('lookup_order'); + t.calledTool('lookup_order', { args: { orderId: '42' }, times: 1 }); + t.notCalledTool('issue_refund'); + t.toolOrder(['lookup_order', 'check_policy']); + t.check('mentions policy', t.reply, includes('30 days')); + t.maxSteps(3); + t.maxTokens(100); + }); + expect(result.passed).toBe(true); + expect(failedMessages(result)).toEqual([]); + expect(result.steps).toBe(3); + expect(result.toolCalls.map((c) => c.name)).toEqual(['lookup_order', 'check_policy']); + expect(result.usage?.totalTokens).toBe(45); + expect(result.assertions.map((a) => a.name)).toContain("calledTool('lookup_order', {\"args\":{\"orderId\":\"42\"},\"times\":1})"); + }); + + it('accepts a subsequence for toolOrder with other calls in between', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.toolOrder(['lookup_order', 'check_policy']); + t.toolOrder(['check_policy']); + t.toolOrder([]); + }); + expect(result.passed).toBe(true); + }); +}); + +describe('trajectory assertions: diagnostic failures', () => { + it('completed() explains a run that was cut off by maxSteps', async () => { + const result = await run( + async (t) => { + await t.send('go'); + t.completed(); + }, + { agent: refundAgent(REFUND_SCRIPT, 1) } + ); + expect(result.passed).toBe(false); + expect(failedMessages(result)).toEqual([ + "completed() failed: finishReason was 'tool_calls' (the run hit maxSteps while still calling tools)", + ]); + }); + + it('completed() before any send() says to call t.send() first', async () => { + const result = await run((t) => t.completed()); + expect(failedMessages(result)).toEqual(['completed() failed: no run yet - call t.send() first']); + }); + + it('calledTool() lists the tools that were called', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.calledTool('issue_refund'); + }); + expect(failedMessages(result)).toEqual([ + "calledTool('issue_refund') failed: tools called were [lookup_order, check_policy]", + ]); + }); + + it('calledTool() says "none" when nothing was called', async () => { + const agent = refundAgent(['Hello']); + const result = await run(async (t) => { + await t.send('go'); + t.calledTool('lookup_order'); + }, { agent }); + expect(failedMessages(result)).toEqual(["calledTool('lookup_order') failed: tools called were none"]); + }); + + it('calledTool() shows the argument diff of the closest call', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.calledTool('lookup_order', { args: { orderId: '41', verbose: true } }); + }); + expect(failedMessages(result)).toEqual([ + "calledTool('lookup_order', {\"args\":{\"orderId\":\"41\",\"verbose\":true}}) failed: 'lookup_order' was called 1 time(s) but no call matched the expected args; closest call differs in orderId: expected \"41\", got \"42\"", + ]); + }); + + it('calledTool() reports missing argument keys', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.calledTool('lookup_order', { args: { reason: 'damaged' } }); + }); + expect(failedMessages(result)[0]).toContain('reason: expected "damaged", got undefined'); + }); + + it('calledTool() reports a wrong call count', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.calledTool('lookup_order', { times: 2 }); + }); + expect(failedMessages(result)).toEqual([ + "calledTool('lookup_order', {\"times\":2}) failed: 'lookup_order' was called 1 time(s), expected 2", + ]); + }); + + it('notCalledTool() says how often the tool was called', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.notCalledTool('check_policy'); + }); + expect(failedMessages(result)).toEqual([ + "notCalledTool('check_policy') failed: 'check_policy' was called 1 time(s); tools called were [lookup_order, check_policy]", + ]); + }); + + it('toolOrder() shows the actual order', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.toolOrder(['check_policy', 'lookup_order']); + }); + expect(failedMessages(result)).toEqual([ + 'toolOrder([check_policy, lookup_order]) failed: tools called were [lookup_order, check_policy]', + ]); + }); + + it('check() names the failed score and quotes the value', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.check('mentions refund', t.reply, includes('refund')); + }); + expect(failedMessages(result)).toEqual([ + "check 'mentions refund' failed: expected \"You can return it within 30 days.\" to include \"refund\"", + ]); + }); + + it('maxSteps(), maxTokens() report the measured values', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.maxSteps(2); + t.maxTokens(10); + }); + expect(failedMessages(result)).toEqual([ + 'maxSteps(2) failed: the agent took 3 steps', + 'maxTokens(10) failed: the agent used 45 tokens', + ]); + }); + + it('maxCostUsd() is skipped (and counts as passed) when the run reports no cost', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.maxCostUsd(0.01); + }); + expect(result.passed).toBe(true); + expect(result.assertions[0]).toMatchObject({ name: 'maxCostUsd(0.01)', passed: true, skipped: true }); + expect(result.assertions[0].message).toContain('no cost'); + }); + + it('maxCostUsd() enforces the limit when usage.costUsd is reported', async () => { + const costly = { + send: async () => ({ + text: 'ok', + messages: [], + toolCalls: [], + usage: { promptTokens: 1, completionTokens: 1, totalTokens: 2, costUsd: 0.5 }, + finishReason: 'stop', + steps: 1, + }), + }; + const result = await run( + async (t) => { + await t.send('go'); + t.maxCostUsd(0.1); + t.maxCostUsd(1); + }, + { agent: costly as unknown as ReturnType } + ); + expect(failedMessages(result)).toEqual(['maxCostUsd(0.1) failed: the agent cost $0.5']); + }); +}); + +describe('checks', () => { + it('score and explain pass and fail', () => { + expect(includes('a').evaluate('cat')).toMatchObject({ passed: true, score: 1 }); + expect(matches(/\d+/g).evaluate('order 42')).toMatchObject({ passed: true }); + expect(matches(/\d+/g).evaluate('order 42')).toMatchObject({ passed: true }); // global flag must not be stateful + expect(matches(/\d+/).evaluate('none').message).toBe('expected "none" to match /\\d+/'); + expect(equals({ a: 1 }).evaluate({ a: 1 }).passed).toBe(true); + expect(equals({ a: 1 }).evaluate({ a: 2 }).message).toBe('expected {"a":2} to equal {"a":1}'); + expect(atLeast(0.7).evaluate(0.8)).toEqual({ score: 0.8, passed: true, threshold: 0.7, message: undefined }); + expect(atLeast(0.7).evaluate(0.5).message).toBe('score 0.5 is below the threshold 0.7'); + expect(atMost(10).evaluate(11).message).toBe('value 11 is above the limit 10'); + expect(atMost(10).evaluate(10).passed).toBe(true); + }); + + it('truncates long values in messages', () => { + const message = includes('x').evaluate('a'.repeat(300)).message ?? ''; + expect(message).toContain('...'); + expect(message.length).toBeLessThan(250); + }); +}); + +describe('soft vs gate', () => { + it('a failed soft assertion is recorded with its score but does not fail the case', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.soft('tone', 0.4, atLeast(0.7)); + t.soft('polite', t.reply, includes('please')); + }); + expect(result.passed).toBe(true); + expect(result.assertions).toEqual([ + { name: 'tone', kind: 'soft', passed: false, score: 0.4, threshold: 0.7, message: "soft check 'tone' failed: score 0.4 is below the threshold 0.7" }, + expect.objectContaining({ name: 'polite', kind: 'soft', passed: false, score: 0 }), + ]); + }); + + it('a failed gate assertion fails the case, and every gate failure is listed', async () => { + const result = await run(async (t) => { + await t.send('go'); + t.check('first', 1, atLeast(2)); + t.check('second', 'x', equals('y')); + t.soft('ignored', 0, atLeast(1)); + }); + expect(result.passed).toBe(false); + expect(describeFailure(result)).toBe( + [ + 'refund failed:', + " - check 'first' failed: score 1 is below the threshold 2", + " - check 'second' failed: expected \"x\" to equal \"y\"", + ].join('\n') + ); + }); +}); + +describe('errors', () => { + it('records an error thrown by the test body instead of throwing', async () => { + const result = await run(async (t) => { + await t.send('go'); + throw new Error('boom'); + }); + expect(result.passed).toBe(false); + expect(result.error).toBe('boom'); + expect(describeFailure(result)).toContain(' - boom'); + }); + + it('records a failing factory as an error', async () => { + const result = await runTrajectoryCase( + { + name: 'x', + agent: () => { + throw new Error('no agent today'); + }, + test: (t) => t.send('go').then(() => undefined), + }, + {}, + 'a case', + 'file.eval.ts' + ); + expect(result).toMatchObject({ passed: false, error: 'no agent today', case: 'a case', file: 'file.eval.ts' }); + expect(describeFailure(result)).toContain('x [a case] failed:'); + }); + + it('t.judge() without a judge provider throws a clear error and never calls an LLM', async () => { + const result = await run(async (t) => { + await t.send('go'); + await t.judge('Is it polite?'); + }); + expect(result.passed).toBe(false); + expect(result.error).toContain('t.judge() needs a judge provider'); + expect(result.error).toContain('judge: { provider, model }'); + expect(result.error).toContain('*.judge.eval.ts'); + }); + + it('t.judge() needs a run first', async () => { + const judge = { provider: mockModel(['1']), model: 'm' }; + const result = await run((t) => t.judge('x').then(() => undefined), { judge }); + expect(result.error).toBe('t.judge() grades the latest reply - call t.send() first.'); + }); +}); + +describe('t.judge() with an explicit judge provider', () => { + it('grades the latest reply even outside the judge runner, using the mock judge', async () => { + const judgeModel = mockModel(['0.9']); + const result = await run( + async (t) => { + await t.send('go'); + t.soft('tone', await t.judge('Is the reply polite?'), atLeast(0.7)); + }, + { judge: { provider: judgeModel, model: 'judge-model' } } + ); + expect(result.passed).toBe(true); + expect(result.assertions[0]).toMatchObject({ name: 'tone', kind: 'soft', passed: true, score: 0.9 }); + const prompt = judgeModel.calls[0].messages[0].content; + expect(prompt).toContain('Is the reply polite?'); + expect(prompt).toContain('You can return it within 30 days.'); + }); +}); + +describe('agent factory isolation', () => { + it('builds a fresh agent per case when given a factory', async () => { + const built: number[] = []; + const spec = { + name: 'factory', + agent: () => { + built.push(built.length); + return createAgent({ provider: mockModel(['hello']) }); + }, + test: async (t: EvalTestContext) => { + await t.send('hi'); + t.check('reply', t.reply, equals('hello')); + }, + }; + const results = [ + await runTrajectoryCase(spec, {}, 'one', undefined), + await runTrajectoryCase(spec, {}, 'two', undefined), + ]; + expect(built).toEqual([0, 1]); + expect(results.map((r) => r.passed)).toEqual([true, true]); + }); + + it('a shared agent instance would run out of scripted turns on the second case (why factories exist)', async () => { + const shared = createAgent({ provider: mockModel(['hello']) }); + const spec = { name: 'shared', agent: shared, test: (t: EvalTestContext) => t.send('hi').then(() => undefined) }; + expect((await runTrajectoryCase(spec, {}, 'one', undefined)).passed).toBe(true); + const second = await runTrajectoryCase(spec, {}, 'two', undefined); + expect(second.passed).toBe(false); + expect(second.error).toContain('the script only has 1 turn'); + }); +}); + +describe('recorder', () => { + const sample: EvalResult = { + name: 'e', + tags: ['smoke'], + passed: true, + assertions: [], + durationMs: 1, + steps: 1, + toolCalls: [], + }; + + it('appends one JSON line per result when LOUSHY_EVAL_RESULTS is set, and nothing otherwise', () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'loushy-rec-')), 'r.jsonl'); + recordEvalResult(sample); + expect(fs.existsSync(file)).toBe(false); + process.env[RESULTS_ENV] = file; + try { + recordEvalResult(sample); + recordEvalResult({ ...sample, name: 'f' }); + } finally { + delete process.env[RESULTS_ENV]; + } + const lines = fs.readFileSync(file, 'utf8').trim().split('\n'); + expect(lines.map((l) => (JSON.parse(l) as EvalResult).name)).toEqual(['e', 'f']); + }); + + it('filters by tag', () => { + expect(matchesTagFilter([])).toBe(true); + process.env[TAGS_ENV] = 'smoke, nightly'; + try { + expect(matchesTagFilter(['smoke'])).toBe(true); + expect(matchesTagFilter(['other'])).toBe(false); + expect(matchesTagFilter([])).toBe(false); + } finally { + delete process.env[TAGS_ENV]; + } + }); +}); + +// --- real defineEval() registrations: datasets, labels, classic form unchanged ----------------- + +const seen: string[] = []; + +defineEval({ + name: 'dataset eval', + agent: () => createAgent({ provider: mockModel([(req) => `echo ${String(req.messages.at(-1)?.content)}`]) }), + cases: [ + { input: 'alpha', expect: 'echo alpha' }, + { input: 'beta', label: 'the beta case', expect: 'echo beta' }, + ], + async test(t, c) { + seen.push(c.input); + await t.send(c.input); + t.completed(); + t.check('echoes', t.reply, equals(c.expect)); + }, +}); + +defineEval({ + name: 'no cases', + agent: createAgent({ provider: mockModel(['fine']) }), + async test(t) { + await t.send('x'); + t.check('reply', t.reply, includes('fine')); + }, +}); + +describe('defineEval dataset registration', () => { + it('ran the test once per case, in order', () => { + expect(seen).toEqual(['alpha', 'beta']); + }); +}); diff --git a/src/evals/trajectory.ts b/src/evals/trajectory.ts new file mode 100644 index 000000000..35053a50c --- /dev/null +++ b/src/evals/trajectory.ts @@ -0,0 +1,298 @@ +/** + * Trajectory evals (LOU-D7): the `t` test context handed to + * `defineEval({ async test(t, c) { ... } })`. + * + * Assertions never throw; each one is recorded on the context (gate or + * soft) and the case fails once at the end, with every failed gate listed. + */ +import type { SimpleAgent } from '../createAgent'; +import type { ExecutionResult } from '../execution/AgentExecutor'; +import type { LLMProvider } from '../providers/llm'; +import { llmJudge } from './llmJudge'; +import type { Check } from './checks'; +import type { AssertionKind, AssertionResult, EvalResult, EvalToolCall } from './evalResult'; +import { gateFailures } from './evalResult'; +import { describeCalled, diffArgs, isSubsequence, parseToolCalls } from './toolMatch'; + +/** The judge provider a trajectory eval grades with, set via `defineEval({ judge })`. */ +export interface EvalJudgeConfig { + /** Provider that grades replies. Use `mockModel` in CI, a real provider in `*.judge.eval.ts`. */ + provider: LLMProvider; + /** Model id the judge runs on. */ + model: string; + temperature?: number; +} + +/** An agent, or a factory that builds a fresh one per case. */ +export type AgentSource = SimpleAgent | (() => SimpleAgent | Promise); + +/** Options for {@link EvalTestContext.calledTool}. */ +export interface CalledToolOptions { + /** Partial deep match: the call's arguments must contain these keys with equal values. */ + args?: Record; + /** Exact number of (matching) calls. Without it, one or more calls pass. */ + times?: number; +} + +/** + * The test context of a trajectory eval. Send messages, then assert on the + * run. Gate assertions fail the eval; `soft()` ones are reported only. + * + * @example + * ```ts + * async test(t) { + * await t.send('Refund order 42'); + * t.completed(); + * t.calledTool('lookup_order', { args: { orderId: '42' } }); + * t.check('mentions policy', t.reply, includes('30 days')); + * } + * ``` + */ +export interface EvalTestContext { + /** Runs the agent on `message`. May be called more than once; assertions look at all runs. */ + send(message: string): Promise; + /** Text of the latest reply (empty before the first `send()`). */ + readonly reply: string; + /** The latest run, or `undefined` before the first `send()`. */ + readonly result: ExecutionResult | undefined; + /** Every tool call made so far, in order, with parsed arguments. */ + readonly toolCalls: readonly EvalToolCall[]; + /** Gate: the latest run ended with a normal stop (not error, aborted, awaiting approval or maxSteps). */ + completed(): void; + /** Gate: the tool was called (optionally with matching `args` / an exact `times`). */ + calledTool(name: string, options?: CalledToolOptions): void; + /** Gate: the tool was never called. */ + notCalledTool(name: string): void; + /** Gate: these tools were called in this order (other calls may sit in between). */ + toolOrder(names: readonly string[]): void; + /** Gate: total model steps are at most `limit`. */ + maxSteps(limit: number): void; + /** Gate: total tokens are at most `limit`. */ + maxTokens(limit: number): void; + /** Gate: reported cost is at most `limit` USD. Skipped when the run reports no cost. */ + maxCostUsd(limit: number): void; + /** Named gate assertion: `check` must pass for `value`. */ + check(name: string, value: T, check: Check): void; + /** Named soft assertion: recorded with its score, never fails the eval (unless `--strict`). */ + soft(name: string, value: T, check: Check): void; + /** + * Grades the latest reply against `rubric` with the eval's judge provider + * and returns a 0..1 score. Throws unless `defineEval({ judge })` is set. + */ + judge(rubric: string): Promise; +} + +const NO_JUDGE_MESSAGE = + 't.judge() needs a judge provider. Pass `judge: { provider, model }` to defineEval() - ' + + 'use mockModel for a deterministic CI eval, or put the eval in a "*.judge.eval.ts" file ' + + '(run by `loushy eval --judge`) with a real provider. loushy never calls a real LLM unless you configure one.'; + +function sumUsage(results: readonly ExecutionResult[]): EvalResult['usage'] { + if (results.length === 0) return undefined; + const total = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }; + for (const { usage } of results) { + total.promptTokens += usage?.promptTokens ?? 0; + total.completionTokens += usage?.completionTokens ?? 0; + total.totalTokens += usage?.totalTokens ?? 0; + } + return total; +} + +/** Implementation of {@link EvalTestContext}; also builds the case's {@link EvalResult}. */ +export class TrajectoryContext implements EvalTestContext { + private readonly results: ExecutionResult[] = []; + private readonly calls: EvalToolCall[] = []; + private readonly recorded: AssertionResult[] = []; + private agent: SimpleAgent | undefined; + + /** + * @param agentSource the agent, or a factory called once, on the first + * `send()`, so every case gets its own instance + */ + constructor( + private readonly agentSource: AgentSource, + private readonly judgeConfig?: EvalJudgeConfig + ) {} + + get result(): ExecutionResult | undefined { + return this.results.at(-1); + } + + get reply(): string { + return this.result?.text ?? ''; + } + + get toolCalls(): readonly EvalToolCall[] { + return this.calls; + } + + get assertions(): readonly AssertionResult[] { + return this.recorded; + } + + async send(message: string): Promise { + this.agent ??= typeof this.agentSource === 'function' ? await this.agentSource() : this.agentSource; + const result = await this.agent.send(message); + this.results.push(result); + this.calls.push(...parseToolCalls(result.toolCalls ?? [])); + return result; + } + + completed(): void { + const reason = this.result?.finishReason; + const passed = reason === 'stop'; + const detail = + this.result === undefined + ? 'no run yet - call t.send() first' + : `finishReason was '${reason}'${reason === 'tool_calls' ? ' (the run hit maxSteps while still calling tools)' : ''}`; + this.gate('completed()', passed, passed ? undefined : `completed() failed: ${detail}`); + } + + calledTool(name: string, options: CalledToolOptions = {}): void { + const label = options.args || options.times !== undefined ? `calledTool('${name}', ${JSON.stringify(options)})` : `calledTool('${name}')`; + const problem = this.calledToolProblem(name, options); + this.gate(label, problem === undefined, problem === undefined ? undefined : `${label} failed: ${problem}`); + } + + notCalledTool(name: string): void { + const count = this.calls.filter((call) => call.name === name).length; + const label = `notCalledTool('${name}')`; + this.gate(label, count === 0, count === 0 ? undefined : `${label} failed: '${name}' was called ${count} time(s); tools called were ${describeCalled(this.calls)}`); + } + + toolOrder(names: readonly string[]): void { + const passed = isSubsequence(this.calls, names); + const label = `toolOrder([${names.join(', ')}])`; + this.gate(label, passed, passed ? undefined : `${label} failed: tools called were ${describeCalled(this.calls)}`); + } + + maxSteps(limit: number): void { + const steps = this.totalSteps(); + this.gate(`maxSteps(${limit})`, steps <= limit, `maxSteps(${limit}) failed: the agent took ${steps} steps`); + } + + maxTokens(limit: number): void { + const tokens = sumUsage(this.results)?.totalTokens ?? 0; + this.gate(`maxTokens(${limit})`, tokens <= limit, `maxTokens(${limit}) failed: the agent used ${tokens} tokens`); + } + + maxCostUsd(limit: number): void { + const label = `maxCostUsd(${limit})`; + const costs = this.results.map((r) => (r.usage as { costUsd?: number } | undefined)?.costUsd); + if (costs.length === 0 || costs.some((cost) => typeof cost !== 'number')) { + this.recorded.push({ + name: label, + kind: 'gate', + passed: true, + score: 1, + skipped: true, + message: `${label} skipped: this run reported no cost (usage.costUsd)`, + }); + return; + } + const total = (costs as number[]).reduce((sum, cost) => sum + cost, 0); + this.gate(label, total <= limit, `${label} failed: the agent cost $${total}`); + } + + check(name: string, value: T, check: Check): void { + this.scored('gate', name, value, check); + } + + soft(name: string, value: T, check: Check): void { + this.scored('soft', name, value, check); + } + + async judge(rubric: string): Promise { + if (!this.judgeConfig) throw new Error(NO_JUDGE_MESSAGE); + if (!this.result) throw new Error('t.judge() grades the latest reply - call t.send() first.'); + const grade = llmJudge({ ...this.judgeConfig, rubric, allowOutsideJudgeRunner: true }); + return grade(this.result); + } + + /** Builds the case's result from everything recorded so far. */ + toResult(base: Pick, durationMs: number, error?: string): EvalResult { + const result: EvalResult = { + ...base, + passed: false, + assertions: [...this.recorded], + durationMs, + steps: this.totalSteps(), + toolCalls: [...this.calls], + usage: sumUsage(this.results), + error, + }; + result.passed = error === undefined && gateFailures(result).length === 0; + return result; + } + + private totalSteps(): number { + return this.results.reduce((sum, result) => sum + (result.steps ?? 0), 0); + } + + private gate(name: string, passed: boolean, message?: string): void { + this.recorded.push({ name, kind: 'gate', passed, score: passed ? 1 : 0, threshold: 1, message: passed ? undefined : message }); + } + + private scored(kind: AssertionKind, name: string, value: T, check: Check): void { + const outcome = check.evaluate(value); + const message = outcome.passed ? undefined : `${kind === 'soft' ? 'soft ' : ''}check '${name}' failed: ${outcome.message ?? 'check did not pass'}`; + this.recorded.push({ name, kind, passed: outcome.passed, score: outcome.score, threshold: outcome.threshold, message }); + } + + private calledToolProblem(name: string, options: CalledToolOptions): string | undefined { + const named = this.calls.filter((call) => call.name === name); + if (named.length === 0) return `tools called were ${describeCalled(this.calls)}`; + const matching = options.args ? named.filter((call) => diffArgs(options.args, call.args).length === 0) : named; + if (options.args && matching.length === 0) return argMismatch(name, named, options.args); + if (options.times !== undefined && matching.length !== options.times) { + return `'${name}' was called ${matching.length} time(s), expected ${options.times}`; + } + return undefined; + } +} + +/** What {@link runTrajectoryCase} needs from a `defineEval()` trajectory config. */ +export interface TrajectoryCaseSpec { + name: string; + tags?: string[]; + agent: AgentSource; + judge?: EvalJudgeConfig; + test(t: EvalTestContext, c: unknown): void | Promise; +} + +/** + * Runs one case of a trajectory eval and returns its result. Never throws: + * an error from `test` (a failed `send()`, an unconfigured `t.judge()`) is + * recorded as the result's `error`. + */ +export async function runTrajectoryCase( + spec: TrajectoryCaseSpec, + c: unknown, + label: string | undefined, + file: string | undefined +): Promise { + const started = Date.now(); + const context = new TrajectoryContext(spec.agent, spec.judge); + let error: string | undefined; + try { + await spec.test(context, c); + } catch (caught) { + error = caught instanceof Error ? caught.message : String(caught); + } + const base = { name: spec.name, case: label, tags: spec.tags ?? [], file }; + return context.toResult(base, Date.now() - started, error); +} + +/** The error message a failed case throws into vitest: every failed gate, one per line. */ +export function describeFailure(result: EvalResult): string { + const label = result.case ? `${result.name} [${result.case}]` : result.name; + const lines = gateFailures(result).map((a) => ` - ${a.message ?? a.name}`); + if (result.error) lines.push(` - ${result.error}`); + return `${label} failed:\n${lines.join('\n')}`; +} + +function argMismatch(name: string, named: readonly EvalToolCall[], expected: Record): string { + const diffs = named.map((call) => diffArgs(expected, call.args)); + const closest = diffs.reduce((best, current) => (current.length < best.length ? current : best)); + return `'${name}' was called ${named.length} time(s) but no call matched the expected args; closest call differs in ${closest.join('; ')}`; +} diff --git a/tsup.config.ts b/tsup.config.ts index 06b88c252..4a029ad47 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -32,6 +32,7 @@ export default defineConfig({ 'cli/studio': 'src/cli/studio.ts', 'cli/mcp': 'src/cli/mcp.ts', 'cli/doctor': 'src/cli/doctor.ts', + 'cli/eval': 'src/cli/eval.ts', 'execution/otel': 'src/execution/otel.ts', 'execution/hooks': 'src/execution/hooks.ts', 'triggers/index': 'src/triggers/index.ts', diff --git a/vitest.config.ts b/vitest.config.ts index 7b4a68470..651416b25 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -26,6 +26,8 @@ export default defineConfig({ '**/node_modules/**', '**/dist/**', 'src/execution/__fixtures__/**', + // Fixture evals that src/cli/eval.test.ts runs through `loushy eval`. + 'src/cli/__fixtures__/**', '**/*.judge.eval.ts', // apps/* are separate npm workspaces (LOU-L1) with their own Vite/ // vitest config and browser-only test environment (jsdom) - without From 5be332e69b48f019be10a909f7572606f3191df7 Mon Sep 17 00:00:00 2001 From: Ali Mohammad Date: Thu, 1 Oct 2026 10:23:48 +0300 Subject: [PATCH 2/2] LOU-D7, LOU-D8: trajectory evals and the loushy eval command Trajectory evals: defineEval() gains a test(t, c) form (agent or per-case factory, tags, cases dataset, gate vs soft assertions, t.judge with an explicit judge provider only) producing a structured EvalResult per case with diagnostic failure messages. The classic score/threshold form is unchanged and also reports a result. loushy eval: spawns the project's vitest with a generated config, collects results through a JSON-lines file (LOUSHY_EVAL_RESULTS), prints a summary table, writes JUnit XML and JSON, and exits 1 on gate failures (soft with --strict), 2 when it cannot run. docs/evals.md added; README, api-overview and llms files updated. Co-Authored-By: Claude Sonnet 5.5 --- llms-full.txt | 283 ++++++++++++++++++++++++++++++++++++++-- llms.txt | 1 + src/cli/eval.test.ts | 7 +- src/cli/eval.ts | 17 ++- src/evals/trajectory.ts | 2 +- 5 files changed, 291 insertions(+), 19 deletions(-) diff --git a/llms-full.txt b/llms-full.txt index cc3654646..7d45c5455 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -214,6 +214,7 @@ console.log(result.usage.totalTokens, result.finishReason, result.steps); - [Workspace tools](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/workspace-tools.md) - file system and shell tools for coding agents, and their security model - [Tracing and observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md) - OpenTelemetry GenAI spans, attribute table, content opt-in - [Testing](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md) - unit-test agents deterministically with the scripted `mockModel` +- [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md) - trajectory evals with `defineEval()`, datasets, `mockModel`, judge evals, `loushy eval` with JUnit/JSON reports - [Skills](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/skills.md) - on-demand instructions: `defineSkill()`, `loadSkills()`, how they save context - [Agent Forge](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/agent-forge.md) - the visual dashboard (`loushy studio`): quickstart, first-agent walkthrough, hook authoring - Full guides site: [linuxdevil.github.io/agent-sdk-docs](https://linuxdevil.github.io/agent-sdk-docs/) @@ -459,22 +460,51 @@ and real OpenTelemetry bridges). ### Evals -Agent-behavior regression tests, run via `vitest run` alongside your normal -test suite — no new test runner: +Agent-behavior regression tests: assert on the tools an agent called, their +order and arguments, its steps and its reply. Deterministic with `mockModel`, +datasets via `cases`, soft vs gate assertions, and `loushy eval` for a summary +table plus JUnit/JSON reports in CI: ```typescript // support-agent.eval.ts -import { defineEval, exactMatch, toolCallOrder } from '@loushy/build-ai-agent'; +import { z } from 'zod'; +import { createAgent, defineEval, defineTool, includes } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'shipped' }), +}); defineEval({ name: 'looks up the order before replying', - agent, provider, toolRegistry, - input: 'Where is order #123?', - score: toolCallOrder([{ tool: 'lookupOrder' }]), - threshold: 1, + agent: () => + createAgent({ + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '123' } }] }, + { text: 'Order 123 has shipped.' }, + ]), + }), + async test(t) { + await t.send('Where is order #123?'); + t.completed(); + t.calledTool('lookup_order', { args: { orderId: '123' } }); + t.check('says shipped', t.reply, includes('shipped')); + }, }); ``` +```bash +npx loushy eval --junit reports/evals.xml +``` + +See [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md) for the assertions table, datasets, judge evals and +a CI example. The original `{ agent, input, provider, score, threshold }` form +of `defineEval()` keeps working. + ### MCP tools & sandboxing ```typescript @@ -1354,8 +1384,9 @@ The built-in context windows and prices are a dated snapshot (see the retrieval ## Flows, evals, observability and security - `FlowBuilder` / `FlowExecutor` - multi-step workflow graphs. -- `defineEval()`, scorers such as `exactMatch` and `toolCallOrder`, and - `llmJudge()` - agent evals run under vitest. +- `defineEval()`, scorers such as `exactMatch` and `toolCallOrder`, checks such + as `includes` and `atLeast`, and `llmJudge()` - agent evals run under vitest + or `loushy eval`; see [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md). - `withSpan()` and `TraceExporter` - tracing for `AgentExecutor.execute()`. `TraceExporter` is a bring-your-own-exporter interface (no exporter ships by default); for real OpenTelemetry spans, import @@ -2178,6 +2209,240 @@ Targets are `DeploymentAdapter` objects (`scaffold`, `build`, `describe`) registered by name with `registerAdapter()`; both are exported from the package root. +# Evals + +Source: docs/evals.md + +Evals are regression tests for agent *behaviour*: which tools the agent called, +in what order, with which arguments, how many steps it took, and what it said. +They are written with `defineEval()`, live in `*.eval.ts` files, run under +[vitest](https://vitest.dev), and are best run in CI with `loushy eval`, which +prints a summary and writes JUnit and JSON reports. + +```bash +npm install --save-dev vitest +npx loushy eval +``` + +## Writing a trajectory eval + +Give `defineEval()` an agent (from `createAgent()`) and a `test` function. Send +messages with `t.send()`, then assert on the run. Using `mockModel` as the +agent's provider makes the eval deterministic: no network, no API key, no +flakiness. **This is the default way to write an eval**; use a real model only +for the [judge evals](#judge-evals) below. + +```ts +// refund.eval.ts +import { z } from 'zod'; +import { createAgent, defineEval, defineTool, includes } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order by id', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId, status: 'delivered' }), +}); + +defineEval({ + name: 'refund flow', + tags: ['smoke'], + // A factory builds a fresh agent (and a fresh script) for every case. + agent: () => + createAgent({ + prompt: 'You handle refund requests.', + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '42' } }] }, + { text: 'Order 42 can be refunded within 30 days.' }, + ]), + }), + async test(t) { + await t.send('Refund order 42'); + t.completed(); + t.calledTool('lookup_order', { args: { orderId: '42' } }); + t.notCalledTool('issue_refund'); + t.check('mentions policy', t.reply, includes('30 days')); + t.maxSteps(4); + }, +}); +``` + +`defineEval()` also still accepts the original form +(`{ name, agent, input, provider, score, threshold }`): it keeps working +unchanged, and its result shows up in `loushy eval` as one `score` assertion. + +### Assertions + +Every assertion is recorded; the case fails at the end and lists **every** +failed gate, so one run shows all that is wrong. + +| Call | Kind | Passes when | +| --- | --- | --- | +| `t.completed()` | gate | the latest run ended with a normal `stop` (not an error, abort, pending approval, or `maxSteps` cut-off) | +| `t.calledTool(name, { args?, times? })` | gate | the tool was called; `args` is a partial deep match (the call's arguments must contain these keys with equal values); `times` is an exact count of matching calls | +| `t.notCalledTool(name)` | gate | the tool was never called | +| `t.toolOrder([a, b])` | gate | the tools were called in this order (other calls may sit in between) | +| `t.maxSteps(n)` | gate | the agent took at most `n` model steps | +| `t.maxTokens(n)` | gate | the agent used at most `n` tokens | +| `t.maxCostUsd(n)` | gate | the run's reported cost is at most `n` USD. Skipped (counts as passed, marked `skipped`) when the SDK reports no cost | +| `t.check(name, value, check)` | gate | `check` passes for `value` | +| `t.soft(name, value, check)` | soft | never fails the run (unless `--strict`); the score is recorded and reported | +| `await t.judge(rubric)` | n/a | returns a 0 to 1 score from an LLM judge (see [Judge evals](#judge-evals)) | + +Checks, for `t.check()` and `t.soft()`: `includes(text)`, `matches(regex)`, +`equals(value)` (yes/no, score 1 or 0), and `atLeast(n)`, `atMost(n)` (the +number is the score). + +Failure messages say what was actually seen: + +```text +calledTool('lookup_order') failed: tools called were [search_docs, issue_refund] +calledTool('lookup_order', {"args":{"orderId":"41"}}) failed: 'lookup_order' was called 1 time(s) but no call matched the expected args; closest call differs in orderId: expected "41", got "42" +completed() failed: finishReason was 'tool_calls' (the run hit maxSteps while still calling tools) +``` + +Other context members: `t.reply` (latest reply text), `t.result` (latest +`ExecutionResult`), `t.toolCalls` (every call so far, with parsed arguments). +You can `send()` more than once; steps, tokens and tool calls add up. + +### Datasets + +Pass `cases` and `test` runs once per case, each reported separately. A case is +any object; its `label` (or `name`, or its `input`) names it. + +```ts +import { z } from 'zod'; +import { createAgent, defineEval, defineTool } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +const lookupOrder = defineTool({ + name: 'lookup_order', + description: 'Look up an order by id', + input: z.object({ orderId: z.string() }), + execute: ({ orderId }) => ({ orderId }), +}); + +defineEval({ + name: 'order lookups', + agent: () => + createAgent({ + tools: [lookupOrder], + provider: mockModel([ + { toolCalls: [{ name: 'lookup_order', args: { orderId: '7' } }] }, + { text: 'Found it.' }, + ]), + }), + cases: [ + { input: 'Where is order 7?', orderId: '7' }, + { input: 'Status of #7', orderId: '7', label: 'hash syntax' }, + ], + async test(t, c) { + await t.send(c.input); + t.calledTool('lookup_order', { args: { orderId: c.orderId } }); + }, +}); +``` + +Pass a **factory** (`agent: () => createAgent(...)`) so each case gets its own +agent and its own `mockModel` script. With a single shared agent, a scripted +model runs out of turns on the second case. + +## Judge evals + +`t.judge(rubric)` grades the latest reply with an LLM and returns a score from +0 to 1. It needs a judge provider, and loushy never calls a real LLM unless you +configured one: without `judge`, `t.judge()` throws an error saying how to fix +it. + +```ts +// tone.judge.eval.ts: run with `loushy eval --judge` +import { createAgent, defineEval, atLeast } from '@loushy/build-ai-agent'; +import { mockModel } from '@loushy/build-ai-agent/testing'; + +defineEval({ + name: 'tone', + agent: createAgent({ provider: mockModel(['Happy to help with that refund!']) }), + // A real provider in practice; mockModel keeps this example offline. + judge: { provider: mockModel(['0.9']), model: 'judge-model' }, + async test(t) { + await t.send('I want my money back'); + t.soft('polite', await t.judge('Is the reply polite?'), atLeast(0.7)); + }, +}); +``` + +Files named `*.judge.eval.ts` are never picked up by a normal run (and +`npm test`); only `loushy eval --judge` (or `npm run test:evals:judge` in this +repository) runs them. + +## `loushy eval` + +```text +loushy eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] +``` + +| Option | Meaning | +| --- | --- | +| `globs...` | eval files to run (default: every `**/*.eval.ts`, or `**/*.judge.eval.ts` with `--judge`) | +| `--tag t` | only run evals whose `tags` include `t` (repeat or comma-separate for several) | +| `--junit path` | write a JUnit XML report | +| `--json path` | write the summary and every structured result as JSON | +| `--strict` | soft failures fail the run | +| `--judge` | run `*.judge.eval.ts` files instead of the normal ones | +| `--config path` | use your own vitest config instead of the generated one | + +It spawns the vitest installed in your project (vitest is *your* dependency; if +it is missing the command prints `npm install --save-dev vitest` and exits 2), +then prints one row per case with its scores and duration, totals, and the gate +failures and soft failures listed separately. + +**Exit code:** `1` on any gate failure (or soft failure with `--strict`, or when +vitest itself fails, for example a file that does not load), `2` when it cannot +run at all (bad arguments, vitest missing), otherwise `0`. + +How results are collected: each case appends one JSON line to a file named by +the `LOUSHY_EVAL_RESULTS` environment variable, which `loushy eval` sets and +reads back. This is more robust than a custom vitest reporter: it works across +vitest versions and worker pools, and needs no module loaded from your project. + +### JUnit + +The report is standard JUnit: one `testsuite` per eval, one `testcase` per case. +A failed gate is a `` carrying the diagnostic message; a +case that threw is an ``; soft failures are `` notes (or +failures under `--strict`). + +### In CI + +```yaml no-verify +# .github/workflows/evals.yml +name: evals +on: [pull_request] +jobs: + evals: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: 22 } + - run: npm ci + - run: npx loushy eval --junit reports/evals.xml --json reports/evals.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: eval-reports + path: reports/ + - uses: mikepenz/action-junit-report@v4 + if: always() + with: + report_paths: reports/evals.xml +``` + +Run `--tag smoke` on every pull request and the full set nightly; run +`--judge` only where you accept real LLM calls and their cost. + # Tracing and observability Source: docs/observability.md diff --git a/llms.txt b/llms.txt index 0993d06be..bfb06b643 100644 --- a/llms.txt +++ b/llms.txt @@ -9,6 +9,7 @@ - [Quick Start](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/quick-start.md): Every TypeScript snippet on this page is a complete, standalone ES module (`.mts`). - [API Overview](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/api-overview.md): The package root (`@loushy/build-ai-agent`) exports everything below. - [Configuration](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/configuration.md): A declarative agent is a YAML (`.yaml`/`.yml`) or JSON (`.json`) file, validated with zod by `loadSpec()` (`src/spec/schema.ts`). +- [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md): Evals are regression tests for agent *behaviour*: which tools the agent called, in what order, with which arguments, how many steps it took, and what it said. - [Tracing and observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md): `AgentExecutor.execute()` and `FlowExecutor.execute()` emit spans through a `TraceExporter`. - [Skills](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/skills.md): A skill is a named chunk of instructions (a markdown document) that the model loads only when it needs it. - [Testing agents](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/testing.md): `@loushy/build-ai-agent/testing` ships `mockModel`: a scripted, deterministic `LLMProvider` for unit tests. diff --git a/src/cli/eval.test.ts b/src/cli/eval.test.ts index 37254fd55..ea70f0b32 100644 --- a/src/cli/eval.test.ts +++ b/src/cli/eval.test.ts @@ -4,6 +4,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { buildVitestConfig, + createVitestSpawner, parseEvalArgs, resolveVitestBin, runEval, @@ -173,10 +174,12 @@ describe('runEval with a fake vitest', () => { }); describe('loushy eval end to end (real vitest, mockModel fixtures)', () => { + // The real vitest, with its own console output discarded to keep this suite's log readable. + const quietVitest = createVitestSpawner('ignore'); + const cli = (args: string[]) => { const log = vi.fn(); - vi.spyOn(console, 'log').mockImplementation(() => {}); - return runEval(args, { log }).then((code) => ({ code, output: log.mock.calls.map((c) => String(c[0])).join('\n') })); + return runEval(args, { log, spawnVitest: quietVitest }).then((code) => ({ code, output: log.mock.calls.map((c) => String(c[0])).join('\n') })); }; it('runs a passing eval: exit 0, JUnit and JSON written, soft failure reported but not failing', async () => { diff --git a/src/cli/eval.ts b/src/cli/eval.ts index b11a803b5..359452007 100644 --- a/src/cli/eval.ts +++ b/src/cli/eval.ts @@ -84,12 +84,15 @@ export function resolveVitestBin(cwd: string): string | undefined { } } -const defaultSpawn: VitestSpawner = (bin, args, env) => - new Promise((resolve, reject) => { - const child = spawn(process.execPath, [bin, ...args], { stdio: 'inherit', env }); - child.on('error', reject); - child.on('close', (code) => resolve(code ?? 1)); - }); +/** Spawns vitest as a child process; `stdio: 'ignore'` discards its console output. */ +export function createVitestSpawner(stdio: 'inherit' | 'ignore' = 'inherit'): VitestSpawner { + return (bin, args, env) => + new Promise((resolve, reject) => { + const child = spawn(process.execPath, [bin, ...args], { stdio, env }); + child.on('error', reject); + child.on('close', (code) => resolve(code ?? 1)); + }); +} /** * The generated vitest config: a plain object (no imports, so it loads from @@ -179,7 +182,7 @@ async function runAndReport(args: EvalCliArgs, deps: Required, vitestB export async function runEval(rest: string[], deps: EvalDeps = {}): Promise { const resolved: Required = { resolveVitest: deps.resolveVitest ?? resolveVitestBin, - spawnVitest: deps.spawnVitest ?? defaultSpawn, + spawnVitest: deps.spawnVitest ?? createVitestSpawner(), cwd: deps.cwd ?? process.cwd(), log: deps.log ?? ((message) => console.log(message)), }; diff --git a/src/evals/trajectory.ts b/src/evals/trajectory.ts index 35053a50c..1dd53081e 100644 --- a/src/evals/trajectory.ts +++ b/src/evals/trajectory.ts @@ -99,7 +99,7 @@ function sumUsage(results: readonly ExecutionResult[]): EvalResult['usage'] { } /** Implementation of {@link EvalTestContext}; also builds the case's {@link EvalResult}. */ -export class TrajectoryContext implements EvalTestContext { +class TrajectoryContext implements EvalTestContext { private readonly results: ExecutionResult[] = []; private readonly calls: EvalToolCall[] = []; private readonly recorded: AssertionResult[] = [];