diff --git a/api/index.api.md b/api/index.api.md index 2cb45bba..528fbd71 100644 --- a/api/index.api.md +++ b/api/index.api.md @@ -1550,6 +1550,15 @@ interface CustomWebhookAuth { verify(req: IncomingMessage, rawBody: Buffer): boolean | Promise; } +// @public +export interface DatasetCase { + example_id: string; + expected: unknown | null; + input: unknown; + label: string; + source_trace_id: string | null; +} + // @public export const dayNameTool: ToolDescriptor; @@ -1638,6 +1647,12 @@ export interface DefinedTool & { + dataset: string; + cases?: undefined; +}): void; + // @public (undocumented) export function defineEval>(config: TrajectoryEvalConfig): void; @@ -1938,6 +1953,7 @@ export const ERROR_CODES: { readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example."; readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target."; readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose."; + readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it."; readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured)."; readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation."; readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names."; @@ -7144,6 +7160,7 @@ export interface TrajectoryComparison { export interface TrajectoryEvalConfig> extends RepeatConfig { agent?: AgentSource; cases?: readonly C[]; + dataset?: string; judge?: EvalJudgeConfig; name: string; tags?: string[]; diff --git a/api/mcp.api.md b/api/mcp.api.md index 3ba8774d..82c1b59c 100644 --- a/api/mcp.api.md +++ b/api/mcp.api.md @@ -575,6 +575,7 @@ const ERROR_CODES: { readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example."; readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target."; readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose."; + readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it."; readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured)."; readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation."; readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names."; diff --git a/api/testing.api.md b/api/testing.api.md index e566c00d..f19536d7 100644 --- a/api/testing.api.md +++ b/api/testing.api.md @@ -97,6 +97,7 @@ const ERROR_CODES: { readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example."; readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target."; readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose."; + readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it."; readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured)."; readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation."; readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names."; diff --git a/api/tools.api.md b/api/tools.api.md index 1629c565..6d90bac6 100644 --- a/api/tools.api.md +++ b/api/tools.api.md @@ -671,6 +671,7 @@ const ERROR_CODES: { readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example."; readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target."; readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose."; + readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it."; readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured)."; readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation."; readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names."; diff --git a/api/worker.api.md b/api/worker.api.md index 618bf831..3a00f194 100644 --- a/api/worker.api.md +++ b/api/worker.api.md @@ -1312,6 +1312,7 @@ const ERROR_CODES: { readonly LOUSHO_TRIGGER_INVALID: "Fix the trigger option the message names; the message shows a working example."; readonly LOUSHO_DEPLOY_FAILED: "Read the message: it names the missing option, file or unsupported feature; docs/deployment.md covers each target."; readonly LOUSHO_SERVER_UNPROTECTED: "Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose."; + readonly LOUSHO_DATASET_UNAVAILABLE: "Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it."; readonly LOUSHO_EVALS_INVALID: "Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured)."; readonly LOUSHO_TEST_FAILED: "Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation."; readonly LOUSHO_CASSETTE_INVALID: "Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names."; diff --git a/docs/errors.md b/docs/errors.md index 637981f7..64fb26bd 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -99,7 +99,7 @@ Find a code by area: | [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied), [`LOUSHO_SANDBOX_FAILED`](#lousho_sandbox_failed) | Asking a sandbox for something it cannot do on the current platform, writing outside its root, or a code-sandbox boundary (timeout / OOM / driver) failing a run. | | [Agent directories, skills and flows](#agent-directories-skills-and-flows) | [`LOUSHO_AGENT_DIR_INVALID`](#lousho_agent_dir_invalid), [`LOUSHO_SKILL_INVALID`](#lousho_skill_invalid), [`LOUSHO_FLOW_INVALID`](#lousho_flow_invalid), [`LOUSHO_FLOW_TOOL_DENIED`](#lousho_flow_tool_denied), [`LOUSHO_FLOW_CHANGED`](#lousho_flow_changed) | Loading an agent directory, a skill, or a flow definition, and gating a flow's tool calls. | | [Storage, deployment and integrations](#storage-deployment-and-integrations) | [`LOUSHO_STORAGE_FAILED`](#lousho_storage_failed), [`LOUSHO_STORAGE_BUSY`](#lousho_storage_busy), [`LOUSHO_TRIGGER_INVALID`](#lousho_trigger_invalid), [`LOUSHO_CHANNEL_REQUEST_FAILED`](#lousho_channel_request_failed), [`LOUSHO_DEPLOY_FAILED`](#lousho_deploy_failed), [`LOUSHO_SERVER_UNPROTECTED`](#lousho_server_unprotected) | A storage backend, a trigger, a channel request, `lousho build` or the built server's start-up. | -| [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. | +| [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_DATASET_UNAVAILABLE`](#lousho_dataset_unavailable), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. | | [General](#general) | [`LOUSHO_GENERIC_ERROR`](#lousho_generic_error), [`LOUSHO_AGENT_EXECUTION_FAILED`](#lousho_agent_execution_failed), [`LOUSHO_FLOW_EXECUTION_FAILED`](#lousho_flow_execution_failed), [`LOUSHO_VALIDATION_FAILED`](#lousho_validation_failed), [`LOUSHO_OPERATION_TIMEOUT`](#lousho_operation_timeout), [`LOUSHO_OUTPUT_INVALID`](#lousho_output_invalid), [`LOUSHO_BUDGET_EXCEEDED`](#lousho_budget_exceeded), [`LOUSHO_GUARDRAIL_TRIPPED`](#lousho_guardrail_tripped) | Run-level failures: a timeout, a budget or guardrail stop, invalid output, and the catch-all codes. | | [Auth](#auth) | [`LOUSHO_AUTH_CONFIG_INVALID`](#lousho_auth_config_invalid) | A route auth helper (`jwt()`, `oidc()`, `basic()`, `apiToken()`) got options it cannot verify with. | | [OAuth](#oauth) | [`LOUSHO_TOKEN_KEY_MISSING`](#lousho_token_key_missing), [`LOUSHO_TOKEN_DECRYPT_FAILED`](#lousho_token_decrypt_failed), [`LOUSHO_OAUTH_PRINCIPAL_REQUIRED`](#lousho_oauth_principal_required), [`LOUSHO_OAUTH_APP_SIGNIN_REQUIRED`](#lousho_oauth_app_signin_required), [`LOUSHO_OAUTH_STORE_MISSING`](#lousho_oauth_store_missing), [`LOUSHO_OAUTH_STATE_INVALID`](#lousho_oauth_state_invalid), [`LOUSHO_SIGNIN_PENDING`](#lousho_signin_pending), [`LOUSHO_OAUTH_TOKEN_EXCHANGE_FAILED`](#lousho_oauth_token_exchange_failed), [`LOUSHO_MCP_AUTH_REQUIRED`](#lousho_mcp_auth_required) | Storing or reading OAuth tokens in a file, SQLite or KV store, sign-in and token exchange for OAuth tools and MCP servers. | @@ -1141,6 +1141,23 @@ judge runner. **Example:** calling `t.judge('polite')` before `t.send('hi')`. +### LOUSHO_DATASET_UNAVAILABLE + +**Means:** `defineEval({ dataset: 'name' })` could not get the dataset's examples +from the observe daemon when vitest collected the file: `LOUSHO_OBSERVE` is not +set, the daemon is unreachable or did not answer in 10 seconds, it answered an +error (a `404` says the dataset was not found), its answer had no `examples`, +or the dataset is empty. The eval then fails as one test named after it, so it +never passes green having run nothing. The message names the dataset and the +daemon URL. + +**Fix:** start the daemon (`lousho observe`) and set `LOUSHO_OBSERVE` to its URL +(and `LOUSHO_OBSERVE_TOKEN` for a `--require-token` daemon), then create the +dataset and add examples to it (`POST /api/v1/datasets`, `POST +/api/v1/datasets/:name/examples`). See [Evals](./evals.md#from-the-daemon). + +**Example:** `lousho eval` with `dataset: 'refunds'` and no `LOUSHO_OBSERVE` set. + ### LOUSHO_TEST_FAILED **Means:** a check a test helper makes did not hold: an eval did not pass, or a diff --git a/docs/evals.md b/docs/evals.md index 8f4e5a16..f0368828 100644 --- a/docs/evals.md +++ b/docs/evals.md @@ -202,6 +202,51 @@ Pass a **factory** (`agent: () => createAgent(...)`) so each case gets its own agent and its own `mockModel` script. With a single shared agent, a scripted model runs out of turns on the second case. +#### From the daemon + +The cases can also live in the observe daemon's dataset registry (see +[Observability](./observability.md), `/api/v1/datasets`): create a dataset, add examples to it +by hand or copy them from a trace you liked, then pass `dataset: 'name'` in +place of `cases`. + +```bash +curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets" -H 'content-type: application/json' -d '{"name":"refunds"}' +curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets/refunds/examples" -H 'content-type: application/json' \ + -d '{"input":"Refund order 42","expected":"refunded"}' +curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets/refunds/examples" -H 'content-type: application/json' \ + -d '{"from_trace_id":""}' +``` + +```ts no-run +import { createAgent, defineEval, includes } from '@lousho/build-ai-agent'; + +defineEval({ + name: 'refunds', + dataset: 'refunds', + agent: () => createAgent({ instructions: 'You handle refunds.' }), + async test(t, c) { + await t.send(String(c.input)); + t.completed(); + if (typeof c.expected === 'string') t.soft('mentions the outcome', t.reply, includes(c.expected)); + }, +}); +``` + +When vitest collects the file, `defineEval()` fetches every example of the +dataset from the daemon `LOUSHO_OBSERVE` points at (sending +`LOUSHO_OBSERVE_TOKEN` when set) and registers one test per example, oldest +first: `refunds [#1]`, `refunds [#2]`, .... Each `c` is a `DatasetCase`: +`{ input, expected, label, example_id, source_trace_id }`, where `input` and +`expected` are whatever JSON the example holds (`expected` is `null` when +unset). `LOUSHO_OBSERVE` is read when `defineEval()` is called. + +The eval fails with +[`LOUSHO_DATASET_UNAVAILABLE`](./errors.md#lousho_dataset_unavailable) when +`LOUSHO_OBSERVE` is not set, the daemon cannot be reached in 10 seconds, the +dataset does not exist, or it has no examples: an eval that runs nothing never +passes. Passing both `cases` and `dataset` throws `LOUSHO_EVALS_INVALID`. A +dataset eval that `lousho eval --tag` filters out is skipped without a fetch. + ## Judge evals `t.judge(rubric)` grades the latest reply with an LLM and returns a score from diff --git a/llms-full.txt b/llms-full.txt index d275bfe2..643cece1 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -8525,7 +8525,7 @@ Find a code by area: | [Sandbox](#sandbox) | [`LOUSHO_SANDBOX_EGRESS_UNSUPPORTED`](#lousho_sandbox_egress_unsupported), [`LOUSHO_SANDBOX_PATH_DENIED`](#lousho_sandbox_path_denied), [`LOUSHO_SANDBOX_FAILED`](#lousho_sandbox_failed) | Asking a sandbox for something it cannot do on the current platform, writing outside its root, or a code-sandbox boundary (timeout / OOM / driver) failing a run. | | [Agent directories, skills and flows](#agent-directories-skills-and-flows) | [`LOUSHO_AGENT_DIR_INVALID`](#lousho_agent_dir_invalid), [`LOUSHO_SKILL_INVALID`](#lousho_skill_invalid), [`LOUSHO_FLOW_INVALID`](#lousho_flow_invalid), [`LOUSHO_FLOW_TOOL_DENIED`](#lousho_flow_tool_denied), [`LOUSHO_FLOW_CHANGED`](#lousho_flow_changed) | Loading an agent directory, a skill, or a flow definition, and gating a flow's tool calls. | | [Storage, deployment and integrations](#storage-deployment-and-integrations) | [`LOUSHO_STORAGE_FAILED`](#lousho_storage_failed), [`LOUSHO_STORAGE_BUSY`](#lousho_storage_busy), [`LOUSHO_TRIGGER_INVALID`](#lousho_trigger_invalid), [`LOUSHO_CHANNEL_REQUEST_FAILED`](#lousho_channel_request_failed), [`LOUSHO_DEPLOY_FAILED`](#lousho_deploy_failed), [`LOUSHO_SERVER_UNPROTECTED`](#lousho_server_unprotected) | A storage backend, a trigger, a channel request, `lousho build` or the built server's start-up. | -| [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. | +| [Tests and evals](#tests-and-evals) | [`LOUSHO_EVALS_INVALID`](#lousho_evals_invalid), [`LOUSHO_DATASET_UNAVAILABLE`](#lousho_dataset_unavailable), [`LOUSHO_TEST_FAILED`](#lousho_test_failed), [`LOUSHO_CASSETTE_INVALID`](#lousho_cassette_invalid) | `defineEval()`, `mockModel` and cassettes. | | [General](#general) | [`LOUSHO_GENERIC_ERROR`](#lousho_generic_error), [`LOUSHO_AGENT_EXECUTION_FAILED`](#lousho_agent_execution_failed), [`LOUSHO_FLOW_EXECUTION_FAILED`](#lousho_flow_execution_failed), [`LOUSHO_VALIDATION_FAILED`](#lousho_validation_failed), [`LOUSHO_OPERATION_TIMEOUT`](#lousho_operation_timeout), [`LOUSHO_OUTPUT_INVALID`](#lousho_output_invalid), [`LOUSHO_BUDGET_EXCEEDED`](#lousho_budget_exceeded), [`LOUSHO_GUARDRAIL_TRIPPED`](#lousho_guardrail_tripped) | Run-level failures: a timeout, a budget or guardrail stop, invalid output, and the catch-all codes. | | [Auth](#auth) | [`LOUSHO_AUTH_CONFIG_INVALID`](#lousho_auth_config_invalid) | A route auth helper (`jwt()`, `oidc()`, `basic()`, `apiToken()`) got options it cannot verify with. | | [OAuth](#oauth) | [`LOUSHO_TOKEN_KEY_MISSING`](#lousho_token_key_missing), [`LOUSHO_TOKEN_DECRYPT_FAILED`](#lousho_token_decrypt_failed), [`LOUSHO_OAUTH_PRINCIPAL_REQUIRED`](#lousho_oauth_principal_required), [`LOUSHO_OAUTH_APP_SIGNIN_REQUIRED`](#lousho_oauth_app_signin_required), [`LOUSHO_OAUTH_STORE_MISSING`](#lousho_oauth_store_missing), [`LOUSHO_OAUTH_STATE_INVALID`](#lousho_oauth_state_invalid), [`LOUSHO_SIGNIN_PENDING`](#lousho_signin_pending), [`LOUSHO_OAUTH_TOKEN_EXCHANGE_FAILED`](#lousho_oauth_token_exchange_failed), [`LOUSHO_MCP_AUTH_REQUIRED`](#lousho_mcp_auth_required) | Storing or reading OAuth tokens in a file, SQLite or KV store, sign-in and token exchange for OAuth tools and MCP servers. | @@ -9567,6 +9567,23 @@ judge runner. **Example:** calling `t.judge('polite')` before `t.send('hi')`. +### LOUSHO_DATASET_UNAVAILABLE + +**Means:** `defineEval({ dataset: 'name' })` could not get the dataset's examples +from the observe daemon when vitest collected the file: `LOUSHO_OBSERVE` is not +set, the daemon is unreachable or did not answer in 10 seconds, it answered an +error (a `404` says the dataset was not found), its answer had no `examples`, +or the dataset is empty. The eval then fails as one test named after it, so it +never passes green having run nothing. The message names the dataset and the +daemon URL. + +**Fix:** start the daemon (`lousho observe`) and set `LOUSHO_OBSERVE` to its URL +(and `LOUSHO_OBSERVE_TOKEN` for a `--require-token` daemon), then create the +dataset and add examples to it (`POST /api/v1/datasets`, `POST +/api/v1/datasets/:name/examples`). See [Evals](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/evals.md#from-the-daemon). + +**Example:** `lousho eval` with `dataset: 'refunds'` and no `LOUSHO_OBSERVE` set. + ### LOUSHO_TEST_FAILED **Means:** a check a test helper makes did not hold: an eval did not pass, or a @@ -10027,6 +10044,51 @@ Pass a **factory** (`agent: () => createAgent(...)`) so each case gets its own agent and its own `mockModel` script. With a single shared agent, a scripted model runs out of turns on the second case. +#### From the daemon + +The cases can also live in the observe daemon's dataset registry (see +[Observability](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/observability.md), `/api/v1/datasets`): create a dataset, add examples to it +by hand or copy them from a trace you liked, then pass `dataset: 'name'` in +place of `cases`. + +```bash +curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets" -H 'content-type: application/json' -d '{"name":"refunds"}' +curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets/refunds/examples" -H 'content-type: application/json' \ + -d '{"input":"Refund order 42","expected":"refunded"}' +curl -X POST "$LOUSHO_OBSERVE/api/v1/datasets/refunds/examples" -H 'content-type: application/json' \ + -d '{"from_trace_id":""}' +``` + +```ts no-run +import { createAgent, defineEval, includes } from '@lousho/build-ai-agent'; + +defineEval({ + name: 'refunds', + dataset: 'refunds', + agent: () => createAgent({ instructions: 'You handle refunds.' }), + async test(t, c) { + await t.send(String(c.input)); + t.completed(); + if (typeof c.expected === 'string') t.soft('mentions the outcome', t.reply, includes(c.expected)); + }, +}); +``` + +When vitest collects the file, `defineEval()` fetches every example of the +dataset from the daemon `LOUSHO_OBSERVE` points at (sending +`LOUSHO_OBSERVE_TOKEN` when set) and registers one test per example, oldest +first: `refunds [#1]`, `refunds [#2]`, .... Each `c` is a `DatasetCase`: +`{ input, expected, label, example_id, source_trace_id }`, where `input` and +`expected` are whatever JSON the example holds (`expected` is `null` when +unset). `LOUSHO_OBSERVE` is read when `defineEval()` is called. + +The eval fails with +[`LOUSHO_DATASET_UNAVAILABLE`](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/errors.md#lousho_dataset_unavailable) when +`LOUSHO_OBSERVE` is not set, the daemon cannot be reached in 10 seconds, the +dataset does not exist, or it has no examples: an eval that runs nothing never +passes. Passing both `cases` and `dataset` throws `LOUSHO_EVALS_INVALID`. A +dataset eval that `lousho eval --tag` filters out is skipped without a fetch. + ## Judge evals `t.judge(rubric)` grades the latest reply with an LLM and returns a score from diff --git a/src/cli/__fixtures__/eval/dataset.eval.ts b/src/cli/__fixtures__/eval/dataset.eval.ts new file mode 100644 index 00000000..0dd403fb --- /dev/null +++ b/src/cli/__fixtures__/eval/dataset.eval.ts @@ -0,0 +1,17 @@ +/** + * Fixture for the `defineEval({ dataset })` tests in src/cli/eval.test.ts: + * the cases come from the stub daemon the test points LOUSHO_OBSERVE at. + */ +import { createAgent } from '../../../createAgent'; +import { defineEval, equals } from '../../../evals'; +import { mockModel } from '../../../testing'; + +defineEval({ + name: 'dataset refunds', + dataset: 'refunds', + agent: () => createAgent({ provider: mockModel([(req) => `echo ${String(req.messages.at(-1)?.content)}`]) }), + async test(t, c) { + await t.send(String(c.input)); + t.check('echoes', t.reply, equals(c.expected)); + }, +}); diff --git a/src/cli/eval.test.ts b/src/cli/eval.test.ts index f7852d31..60486152 100644 --- a/src/cli/eval.test.ts +++ b/src/cli/eval.test.ts @@ -1,5 +1,7 @@ import { describe, it, expect, vi, afterEach, beforeAll, afterAll } from 'vitest'; import * as fs from 'node:fs'; +import * as http from 'node:http'; +import type { AddressInfo } from 'node:net'; import * as os from 'node:os'; import * as path from 'node:path'; import { @@ -331,6 +333,52 @@ describe('lousho eval end to end (real vitest, mockModel fixtures)', () => { }, 60_000); }); +describe('lousho eval over a daemon dataset (real vitest, stub daemon)', () => { + const fixture = path.join(FIXTURES, 'dataset.eval.ts'); + const quietVitest = createVitestSpawner('ignore'); + const cli = (args: string[]) => { + const log = vi.fn(); + return runEval(args, { log, spawnVitest: quietVitest }).then((code) => ({ code, output: log.mock.calls.map((c) => String(c[0])).join('\n') })); + }; + const examples = [ + { example_id: 'ex_1', dataset_id: 'ds', input: 'refund 42', expected: 'echo refund 42', source_trace_id: null, created_at: 1 }, + { example_id: 'ex_2', dataset_id: 'ds', input: 'refund 7', expected: 'echo refund 7', source_trace_id: null, created_at: 2 }, + ]; + // Serves `refunds` at GET /api/v1/datasets/refunds; every other request (trace/score emission) gets `{}`. + const daemon = http.createServer((req, res) => { + res.writeHead(200, { 'content-type': 'application/json' }); + const body = req.method === 'GET' && req.url?.startsWith('/api/v1/datasets/refunds') ? { examples, total: examples.length, limit: 1000, offset: 0 } : {}; + res.end(JSON.stringify(body)); + }); + + beforeAll(() => new Promise((resolve) => daemon.listen(0, '127.0.0.1', resolve))); + afterAll(() => new Promise((resolve) => daemon.close(() => resolve()))); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('runs one case per example, reported as #1 and #2', async () => { + vi.stubEnv('LOUSHO_OBSERVE', `http://127.0.0.1:${(daemon.address() as AddressInfo).port}`); + const json = tempFile('results.json'); + const { code, output } = await cli([fixture, '--json', json]); + expect(code).toBe(0); + expect(output).toContain('2 eval(s): 2 passed, 0 failed'); + const parsed = JSON.parse(fs.readFileSync(json, 'utf8')) as { results: EvalResult[] }; + expect(parsed.results.map((r) => [r.name, r.case, r.passed])).toEqual([ + ['dataset refunds', '#1', true], + ['dataset refunds', '#2', true], + ]); + }, 60_000); + + it('fails the eval with LOUSHO_DATASET_UNAVAILABLE when LOUSHO_OBSERVE is not set', async () => { + vi.stubEnv('LOUSHO_OBSERVE', ''); + const { code, output } = await cli([fixture]); + expect(code).toBe(1); + expect(output).toContain('1 eval(s): 0 passed, 1 failed'); + expect(output).toContain("defineEval({ dataset: 'refunds' }) needs the observe daemon"); + }, 60_000); +}); + describe('lousho eval --record / --replay / --drift (real vitest, mockModel as the "real" provider)', () => { const fixture = path.join(FIXTURES, 'replay.eval.ts'); const cassettes = path.join(FIXTURES, '__cassettes__'); diff --git a/src/evals/dataset.ts b/src/evals/dataset.ts new file mode 100644 index 00000000..880690ed --- /dev/null +++ b/src/evals/dataset.ts @@ -0,0 +1,85 @@ +/** + * The cases of a `defineEval({ dataset: 'name' })` eval: the examples of a + * dataset in the observe daemon's registry (`GET /api/v1/datasets/:name`), + * fetched page by page at vitest collection. Every failure is + * `LOUSHO_DATASET_UNAVAILABLE`, and so is an empty dataset: an eval that runs + * no case must never pass green. + */ + +import { SDKError } from '../execution/errors'; +import { observeRequest } from '../observe/httpClient'; + +/** The `c` a dataset eval's `test(t, c)` receives: one daemon example. */ +export interface DatasetCase { + /** The example's input (any JSON value). */ + input: unknown; + /** The example's expected value, `null` when the example has none. */ + expected: unknown | null; + /** `#1`, `#2`, ... in the daemon's order (oldest first); names the test ` [#1]`. */ + label: string; + /** The daemon's id of the example. */ + example_id: string; + /** The trace the example was copied from (`from_trace_id`), else `null`. */ + source_trace_id: string | null; +} + +interface ExampleRow { + example_id: string; + input: unknown; + expected?: unknown; + source_trace_id?: string | null; +} + +interface DatasetPage { + examples?: unknown; + total?: unknown; +} + +/** Where the daemon is: `LOUSHO_OBSERVE`'s URL (undefined when unset) and its token. */ +export interface DatasetSource { + url: string | undefined; + token?: string; +} + +const PAGE_LIMIT = 1000; +const REQUEST_TIMEOUT_MS = 10_000; + +function unavailable(message: string, cause?: unknown): SDKError { + return new SDKError(message, 'LOUSHO_DATASET_UNAVAILABLE', cause === undefined ? {} : { cause }); +} + +async function fetchPage(name: string, url: string, token: string | undefined, offset: number): Promise { + const path = `/api/v1/datasets/${encodeURIComponent(name)}?limit=${PAGE_LIMIT}&offset=${offset}`; + try { + return await observeRequest(url, path, { method: 'GET' }, REQUEST_TIMEOUT_MS, token ? { token } : {}); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const notFound = /answered 404/.test(detail) ? `dataset '${name}' was not found on the observe daemon at ${url}` : undefined; + throw unavailable(notFound ?? `Could not fetch dataset '${name}' from the observe daemon at ${url}: ${detail}`, error); + } +} + +/** Every example of dataset `name`, oldest first, as eval cases labelled `#1`, `#2`, .... */ +export async function fetchDatasetCases(name: string, source: DatasetSource): Promise { + const { url, token } = source; + if (!url) throw unavailable(`defineEval({ dataset: '${name}' }) needs the observe daemon: set LOUSHO_OBSERVE to its URL`); + const rows: ExampleRow[] = []; + for (;;) { + const page = await fetchPage(name, url, token, rows.length); + if (!page || !Array.isArray(page.examples)) { + throw unavailable(`The observe daemon at ${url} answered dataset '${name}' without an examples array`); + } + rows.push(...(page.examples as ExampleRow[])); + const total = typeof page.total === 'number' ? page.total : rows.length; + // An empty page ends the loop even if `total` says more, so a shrinking dataset cannot spin it. + if (rows.length >= total || page.examples.length === 0) break; + } + if (rows.length === 0) throw unavailable(`dataset '${name}' on the observe daemon at ${url} is empty; add examples before running the eval`); + return rows.map((row, index) => ({ + input: row.input, + expected: row.expected ?? null, + label: `#${index + 1}`, + example_id: row.example_id, + source_trace_id: row.source_trace_id ?? null, + })); +} diff --git a/src/evals/datasetEval.test.ts b/src/evals/datasetEval.test.ts new file mode 100644 index 00000000..9831a669 --- /dev/null +++ b/src/evals/datasetEval.test.ts @@ -0,0 +1,183 @@ +/** + * defineEval({ dataset: 'name' }): the cases come from the observe daemon's + * dataset registry (`GET /api/v1/datasets/:name`), fetched at collection. + * A stub daemon (node:http on 127.0.0.1:0) stands in for `lousho observe`. + */ +import { describe, it, expect } from 'vitest'; +import * as http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { createAgent } from '../createAgent'; +import { mockModel } from '../testing'; +import { SDKError } from '../execution/errors'; +import { equals } from './checks'; +import { defineEval } from './defineEval'; +import { fetchDatasetCases, type DatasetCase } from './dataset'; +import { TAGS_ENV } from './recorder'; + +const EXAMPLES = [ + { example_id: 'ex_1', dataset_id: 'ds_1', input: 'refund order 42', expected: 'echo refund order 42', source_trace_id: 'tr_9', created_at: 1 }, + { example_id: 'ex_2', dataset_id: 'ds_1', input: 'refund order 7', expected: 'echo refund order 7', source_trace_id: null, created_at: 2 }, +]; + +const requests: Array<{ url: string; auth?: string }> = []; + +/** Serves `refunds` one example per page (so paging is exercised), `blank` with none, `broken` without `examples`, `html` not JSON. */ +const server = http.createServer((req, res) => { + requests.push({ url: req.url ?? '', auth: req.headers.authorization }); + const url = new URL(req.url ?? '/', 'http://stub'); + const offset = Number(url.searchParams.get('offset') ?? 0); + const send = (status: number, body: unknown) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }; + const dataset = { dataset_id: 'ds_1', name: 'refunds', description: null, created_at: 1, example_count: EXAMPLES.length }; + if (url.pathname === '/api/v1/datasets/refunds') { + return send(200, { dataset, examples: EXAMPLES.slice(offset, offset + 1), total: EXAMPLES.length, limit: 1, offset }); + } + if (url.pathname === '/api/v1/datasets/blank') return send(200, { dataset: { ...dataset, name: 'blank', example_count: 0 }, examples: [], total: 0, limit: 1000, offset }); + if (url.pathname === '/api/v1/datasets/broken') return send(200, { dataset }); + if (url.pathname === '/api/v1/datasets/html') { + res.writeHead(200, { 'content-type': 'text/html' }); + return res.end('not the daemon'); + } + if (url.pathname.startsWith('/api/v1/datasets/')) return send(404, { error: 'dataset not found', code: 'LOUSHO_REQUEST_INVALID' }); + send(200, {}); +}); +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); +const daemonUrl = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + +/** A port nothing listens on. */ +async function deadUrl(): Promise { + const probe = http.createServer(); + await new Promise((resolve) => probe.listen(0, '127.0.0.1', resolve)); + const { port } = probe.address() as AddressInfo; + await new Promise((resolve) => probe.close(() => resolve())); + return `http://127.0.0.1:${port}`; +} + +async function rejection(promise: Promise): Promise { + const error = await promise.then( + () => undefined, + (e: unknown) => e + ); + expect(error).toBeInstanceOf(SDKError); + return error as SDKError; +} + +// --- real registrations (LOUSHO_OBSERVE is read when defineEval() is called) --- + +const seen: DatasetCase[] = []; +process.env.LOUSHO_OBSERVE = daemonUrl; +process.env.LOUSHO_OBSERVE_TOKEN = 'tok'; + +defineEval({ + name: 'refunds', + dataset: 'refunds', + agent: () => createAgent({ provider: mockModel([(req) => `echo ${String(req.messages.at(-1)?.content)}`]) }), + async test(t, c) { + seen.push(c); + await t.send(String(c.input)); + t.check('echoes', t.reply, equals(c.expected)); + }, +}); + +process.env[TAGS_ENV] = 'nightly'; +defineEval({ + name: 'filtered out', + dataset: 'never-fetched', + tags: ['smoke'], + async test() { + throw new Error('a tag-filtered eval must not run'); + }, +}); +delete process.env[TAGS_ENV]; +delete process.env.LOUSHO_OBSERVE; +delete process.env.LOUSHO_OBSERVE_TOKEN; + +describe('defineEval({ dataset })', () => { + it('ran one case per daemon example, labelled #1, #2, with the example fields', () => { + expect(seen).toEqual([ + { input: 'refund order 42', expected: 'echo refund order 42', label: '#1', example_id: 'ex_1', source_trace_id: 'tr_9' }, + { input: 'refund order 7', expected: 'echo refund order 7', label: '#2', example_id: 'ex_2', source_trace_id: null }, + ]); + }); + + it('named the tests [#n], fetched every page with the token, and skipped the filtered eval without a fetch', ({ task }) => { + const names: Array<{ name: string; mode: string }> = []; + const walk = (tasks: readonly { name: string; mode: string; type: string; tasks?: unknown[] }[]) => { + for (const t of tasks) { + if (t.type === 'suite') walk((t.tasks ?? []) as typeof tasks); + else names.push({ name: t.name, mode: t.mode }); + } + }; + walk(task.file!.tasks as never); + expect(names).toEqual( + expect.arrayContaining([ + { name: 'refunds [#1]', mode: 'run' }, + { name: 'refunds [#2]', mode: 'run' }, + { name: 'filtered out', mode: 'skip' }, + ]) + ); + const refunds = requests.filter((r) => r.url.startsWith('/api/v1/datasets/refunds')); + expect(refunds.map((r) => new URL(r.url, 'http://stub').searchParams.get('offset'))).toEqual(['0', '1']); + expect(refunds.every((r) => r.auth === 'Bearer tok')).toBe(true); + expect(requests.some((r) => r.url.includes('never-fetched'))).toBe(false); + }); + + it('rejects `cases` and `dataset` together with LOUSHO_EVALS_INVALID', () => { + let error: unknown; + try { + defineEval({ name: 'both', dataset: 'refunds', cases: [{ input: 'x' }], test() {} } as never); + } catch (e) { + error = e; + } + expect(error).toBeInstanceOf(SDKError); + expect((error as SDKError).code).toBe('LOUSHO_EVALS_INVALID'); + }); +}); + +describe('fetchDatasetCases', () => { + it('fails with LOUSHO_DATASET_UNAVAILABLE when the daemon is unreachable', async () => { + const url = await deadUrl(); + const error = await rejection(fetchDatasetCases('refunds', { url })); + expect(error.code).toBe('LOUSHO_DATASET_UNAVAILABLE'); + expect(error.message).toContain('refunds'); + expect(error.message).toContain(url); + }); + + it('says "not found" and names the dataset and URL for a 404', async () => { + const error = await rejection(fetchDatasetCases('refundz', { url: daemonUrl })); + expect(error.code).toBe('LOUSHO_DATASET_UNAVAILABLE'); + expect(error.message).toContain("'refundz'"); + expect(error.message).toContain('not found'); + expect(error.message).toContain(daemonUrl); + }); + + it('fails when LOUSHO_OBSERVE is not set', async () => { + const error = await rejection(fetchDatasetCases('refunds', { url: undefined })); + expect(error.code).toBe('LOUSHO_DATASET_UNAVAILABLE'); + expect(error.message).toContain('LOUSHO_OBSERVE'); + }); + + it('fails for an empty dataset, so an eval that runs nothing cannot pass', async () => { + const error = await rejection(fetchDatasetCases('blank', { url: daemonUrl })); + expect(error.code).toBe('LOUSHO_DATASET_UNAVAILABLE'); + expect(error.message).toContain('empty'); + }); + + it('fails for a body without an examples array', async () => { + const error = await rejection(fetchDatasetCases('broken', { url: daemonUrl })); + expect(error.code).toBe('LOUSHO_DATASET_UNAVAILABLE'); + expect(error.message).toContain('examples'); + }); + + it('fails for a 200 answer that is not JSON', async () => { + const error = await rejection(fetchDatasetCases('html', { url: daemonUrl })); + expect(error.code).toBe('LOUSHO_DATASET_UNAVAILABLE'); + expect(error.message).toContain('examples'); + }); + + it('closes the stub daemon', async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); +}); diff --git a/src/evals/defineEval.ts b/src/evals/defineEval.ts index 0ac8a879..ed901857 100644 --- a/src/evals/defineEval.ts +++ b/src/evals/defineEval.ts @@ -16,6 +16,8 @@ import { scoreAssertion, type EvalResult } from './evalResult'; import { isJudgeScorer } from './llmJudge'; import { evalCaseObservation } from './observe'; import { matchesTagFilter, recordEvalResult } from './recorder'; +import { fetchDatasetCases, type DatasetCase } from './dataset'; +import { observeUrlFromEnv } from '../observe/reporter'; import { remoteTargetFromEnv } from './remoteTarget'; import { SDKError } from '../execution/errors'; import { @@ -108,6 +110,14 @@ export interface TrajectoryEvalConfig> extends RepeatC tags?: string[]; /** Dataset: `test` runs once per case. A case's `label` (or `name`, or its `input`) names it in reports. */ cases?: readonly C[]; + /** + * A dataset in the observe daemon's registry (`LOUSHO_OBSERVE`), by name: + * its examples are fetched when vitest collects the file and `test` runs + * once per example, each a {@link DatasetCase} labelled `#1`, `#2`, .... + * An unreachable daemon, an unknown name or an empty dataset fails the + * eval with `LOUSHO_DATASET_UNAVAILABLE`. Not together with `cases`. + */ + dataset?: string; /** Judge provider for `t.judge()`. Without it `t.judge()` throws; no LLM is ever called implicitly. */ judge?: EvalJudgeConfig; /** @@ -231,20 +241,65 @@ const missingAgent: AgentSource = () => { throw new SDKError("defineEval() needs an `agent` (or a `target`) to run the cases against", 'LOUSHO_CONFIG_MISSING_AGENT'); }; +/** `test` or `test.skip`: what a case registers with. */ +type Register = (name: string, fn: (context: Vitest.TestContext) => Promise, timeout?: number) => void; + +function registerCases(config: TrajectoryEvalConfig, cases: readonly unknown[], register: Register, file: () => string | undefined): void { + const tags = config.tags ?? []; + cases.forEach((c, index) => { + const { label, key } = caseLabel(c, index); + const ref = { name: config.name, tags, label, key, index }; + register(`${config.name} [${label}]`, (context) => runTrajectory(config, c, ref, file(), context), config.timeoutMs); + }); +} + function defineTrajectoryEval(config: TrajectoryEvalConfig): void { - const { test, expect } = currentVitest(); + if (config.cases !== undefined && config.dataset !== undefined) { + throw new SDKError(`defineEval({ name: '${config.name}' }) takes \`cases\` or \`dataset\`, not both`, 'LOUSHO_EVALS_INVALID'); + } + const vitest = currentVitest(); // expect.getState().testPath is only set while a test runs, not at collection. - const file = () => currentTestPath(expect); - const register = matchesTagFilter(config.tags ?? []) ? test : test.skip; - const tags = config.tags ?? []; + const file = () => currentTestPath(vitest.expect); + const selected = matchesTagFilter(config.tags ?? []); + const register = selected ? vitest.test : vitest.test.skip; + if (config.dataset !== undefined) { + // A tag-filtered-out dataset eval needs no daemon: one skipped test, no fetch. + if (selected) defineDatasetEval(config, config.dataset, vitest, file); + else register(config.name, () => undefined); + return; + } if (config.cases === undefined) { - register(config.name, (context) => runTrajectory(config, NO_CASE, { name: config.name, tags }, file(), context), config.timeoutMs); + register(config.name, (context) => runTrajectory(config, NO_CASE, { name: config.name, tags: config.tags ?? [] }, file(), context), config.timeoutMs); return; } - config.cases.forEach((c, index) => { - const { label, key } = caseLabel(c, index); - const ref = { name: config.name, tags, label, key, index }; - register(`${config.name} [${label}]`, (context) => runTrajectory(config, c, ref, file(), context), config.timeoutMs); + registerCases(config, config.cases, register, file); +} + +/** + * Registers a `dataset` eval's cases inside an async suite: vitest awaits the + * factory at collection, so each daemon example becomes its own test. The + * daemon URL and token are read when `defineEval()` is called. A failed fetch + * registers one failing test (and a failed result row for `lousho eval`), + * never zero tests. + */ +function defineDatasetEval(config: TrajectoryEvalConfig, dataset: string, vitest: VitestApi, file: () => string | undefined): void { + const source = { + url: observeUrlFromEnv(), + ...(process.env.LOUSHO_OBSERVE_TOKEN ? { token: process.env.LOUSHO_OBSERVE_TOKEN } : {}), + }; + vitest.describe(config.name, async () => { + let cases: DatasetCase[]; + try { + cases = await fetchDatasetCases(dataset, source); + } catch (error) { + vitest.test(config.name, () => { + const message = error instanceof Error ? error.message : String(error); + recordEvalResult({ name: config.name, tags: config.tags ?? [], passed: false, assertions: [], durationMs: 0, steps: 0, toolCalls: [], error: message, file: file() }); + throw error; + }); + return; + } + registerCases(config, cases, vitest.test, file); }); } @@ -261,12 +316,14 @@ function defineTrajectoryEval(config: TrajectoryEvalConfig): void { * exactly the instance the running test file registers with; with vitest * `globals` enabled, the global test/expect are used as a fallback. */ -function currentVitest(): Pick { +type VitestApi = Pick; + +function currentVitest(): VitestApi { const g = globalThis as Record; const api = g.__vitest_index__ as typeof Vitest | undefined; if (api && typeof api.test === 'function') return api; if (typeof g.test === 'function' && typeof g.expect === 'function') { - return { test: g.test as typeof Vitest.test, expect: g.expect as typeof Vitest.expect }; + return { test: g.test as typeof Vitest.test, expect: g.expect as typeof Vitest.expect, describe: g.describe as typeof Vitest.describe }; } throw new SDKError('defineEval() must be called from a test file running under vitest', 'LOUSHO_EVALS_INVALID'); } @@ -323,6 +380,8 @@ function defineClassicEval(config: EvalConfig): void { * `score >= threshold`. * - Trajectory: `{ name, agent, cases?, test(t, c) }` drives a * `createAgent()` agent and asserts on its tool calls, steps and reply. + * `{ dataset: 'name' }` in place of `cases` runs it over a dataset stored in + * the observe daemon (`LOUSHO_OBSERVE`), one {@link DatasetCase} per example. * * Run evals with `lousho eval` for a summary table and JUnit/JSON reports. * @@ -340,6 +399,7 @@ function defineClassicEval(config: EvalConfig): void { * ``` */ export function defineEval(config: EvalConfig): void; +export function defineEval(config: TrajectoryEvalConfig & { dataset: string; cases?: undefined }): void; export function defineEval>(config: TrajectoryEvalConfig): void; export function defineEval(config: EvalConfig | TrajectoryEvalConfig): void { if ('test' in config) defineTrajectoryEval(config); diff --git a/src/evals/index.ts b/src/evals/index.ts index 22e2a59d..325f099a 100644 --- a/src/evals/index.ts +++ b/src/evals/index.ts @@ -13,4 +13,5 @@ export * from './checks'; export * from './evalResult'; export { compareTrajectories } from './drift'; export type { TrajectoryComparison, TrajectoryStep, DriftEntry } from './drift'; +export type { DatasetCase } from './dataset'; export type { EvalJudgeConfig, EvalTestContext, CalledToolOptions, ToolScopeOptions, MaxCostOptions, AgentSource } from './trajectory'; diff --git a/src/utils/errorCodes.ts b/src/utils/errorCodes.ts index 80c7acf3..83f3fa06 100644 --- a/src/utils/errorCodes.ts +++ b/src/utils/errorCodes.ts @@ -66,6 +66,8 @@ export const ERROR_CODES = { /** The built server was asked to listen on a non-loopback host with no route auth. */ LOUSHO_SERVER_UNPROTECTED: 'Set LOUSHO_API_TOKEN, add an auth.ts to the agent directory, bind to 127.0.0.1, or set LOUSHO_ALLOW_PUBLIC=1 to serve an open agent on purpose.', + /** A `defineEval({ dataset })` dataset could not be fetched from the observe daemon, or is empty. */ + LOUSHO_DATASET_UNAVAILABLE: 'Start the observe daemon and set LOUSHO_OBSERVE to its URL, and create the dataset the message names with examples in it.', LOUSHO_EVALS_INVALID: 'Call the eval helper the way the message says (inside a vitest file, after t.send(), with a judge configured).', LOUSHO_TEST_FAILED: 'Read the message: an eval or a mockModel() script did not hold; fix the agent, or the expectation.', LOUSHO_CASSETTE_INVALID: 'Record the cassette again (lousho eval --record, or recordReplay mode: record), or fix the file the message names.',